예제 #1
0
파일: helpers.py 프로젝트: SEJeff/taskflow
def load(flow,
         store=None,
         flow_detail=None,
         book=None,
         engine_conf=None,
         backend=None,
         namespace=ENGINES_NAMESPACE):
    """Load flow into engine

    This function creates and prepares engine to run the
    flow. All that is left is to run the engine with 'run()' method.

    Which engine to load is specified in 'engine_conf' parameter. It
    can be a string that names engine type or a dictionary which holds
    engine type (with 'engine' key) and additional engine-specific
    configuration (for example, executor for multithreaded engine).

    Which storage backend to use is defined by backend parameter. It
    can be backend itself, or a dictionary that is passed to
    taskflow.persistence.backends.fetch to obtain backend.

    :param flow: flow to load
    :param store: dict -- data to put to storage to satisfy flow requirements
    :param flow_detail: FlowDetail that holds state of the flow
    :param book: LogBook to create flow detail in if flow_detail is None
    :param engine_conf: engine type and configuration configuration
    :param backend: storage backend to use or configuration
    :param namespace: driver namespace for stevedore (default is fine
       if you don't know what is it)
    :returns: engine
    """

    if engine_conf is None:
        engine_conf = {'engine': 'default'}

    # NOTE(imelnikov): this allows simpler syntax
    if isinstance(engine_conf, six.string_types):
        engine_conf = {'engine': engine_conf}

    if isinstance(backend, dict):
        backend = p_backends.fetch(backend)

    if flow_detail is None:
        flow_detail = p_utils.create_flow_detail(flow,
                                                 book=book,
                                                 backend=backend)

    mgr = stevedore.driver.DriverManager(namespace,
                                         engine_conf['engine'],
                                         invoke_on_load=True,
                                         invoke_kwds={
                                             'conf': engine_conf.copy(),
                                             'flow': flow,
                                             'flow_detail': flow_detail,
                                             'backend': backend
                                         })
    engine = mgr.driver
    if store:
        engine.storage.inject(store)
    return engine
    def test_graph_translate(self):
        wf = graph_flow.Flow("graph")
        one = self.someTask(provides=['data'])
        two = self.someTask(requires=['data'])
        three = self.someTask()
        wf.add(one, two, three)
        wf.link(two, three)

        backend = impl_memory.MemoryBackend({})
        lb = logbook.LogBook("my-log-book")
        fd = persistence_utils.create_flow_detail(wf, book=lb, backend=backend)
        storage = task_storage.Storage(fd)

        engine = distributed_engine.DistributedEngine(wf, storage)
        trans = dist_translator.DistributedTranslator(engine)
        trans.translate(wf)

        self.assertEqual(3, len(engine.tasks))
        self.assertEqual(1, len(engine.roots))

        dict_listeners = engine.client._listeners
        actual = set([])

        for listeners in dict_listeners.values():
            for listener in listeners:
                actual.add(listener[1].flow_task)

        self.assertEquals(actual, set([two, three]))
예제 #3
0
 def _create_engine(**kwargs):
     flow = lf.Flow('test-flow').add(utils.DummyTask())
     backend = backends.fetch({'connection': 'memory'})
     flow_detail = pu.create_flow_detail(flow, backend=backend)
     options = kwargs.copy()
     return engine.WorkerBasedActionEngine(flow, flow_detail, backend,
                                           options)
예제 #4
0
def load_from_factory(flow_factory, factory_args=None, factory_kwargs=None,
                      store=None, book=None, engine_conf=None, backend=None,
                      namespace=ENGINES_NAMESPACE, engine=ENGINE_DEFAULT,
                      **kwargs):
    """Loads a flow from a factory function into an engine.

    Gets flow factory function (or name of it) and creates flow with
    it. Then, the flow is loaded into an engine with the :func:`load() <load>`
    function, and the factory function fully qualified name is saved to flow
    metadata so that it can be later resumed.

    :param flow_factory: function or string: function that creates the flow
    :param factory_args: list or tuple of factory positional arguments
    :param factory_kwargs: dict of factory keyword arguments

    Further arguments are interpreted as for :func:`load() <load>`.

    :returns: engine
    """

    _factory_name, factory_fun = _fetch_validate_factory(flow_factory)
    if not factory_args:
        factory_args = []
    if not factory_kwargs:
        factory_kwargs = {}
    flow = factory_fun(*factory_args, **factory_kwargs)
    if isinstance(backend, dict):
        backend = p_backends.fetch(backend)
    flow_detail = p_utils.create_flow_detail(flow, book=book, backend=backend)
    save_factory_details(flow_detail,
                         flow_factory, factory_args, factory_kwargs,
                         backend=backend)
    return load(flow=flow, store=store, flow_detail=flow_detail, book=book,
                engine_conf=engine_conf, backend=backend, namespace=namespace,
                engine=engine, **kwargs)
예제 #5
0
def load(flow, store=None, flow_detail=None, book=None,
         engine_conf=None, backend=None,
         namespace=ENGINES_NAMESPACE, engine=ENGINE_DEFAULT, **kwargs):
    """Load a flow into an engine.

    This function creates and prepares an engine to run the provided flow. All
    that is left after this returns is to run the engine with the
    engines :py:meth:`~taskflow.engines.base.Engine.run` method.

    Which engine to load is specified via the ``engine`` parameter. It
    can be a string that names the engine type to use, or a string that
    is a URI with a scheme that names the engine type to use and further
    options contained in the URI's host, port, and query parameters...

    Which storage backend to use is defined by the backend parameter. It
    can be backend itself, or a dictionary that is passed to
    :py:func:`~taskflow.persistence.backends.fetch` to obtain a
    viable backend.

    :param flow: flow to load
    :param store: dict -- data to put to storage to satisfy flow requirements
    :param flow_detail: FlowDetail that holds the state of the flow (if one is
        not provided then one will be created for you in the provided backend)
    :param book: LogBook to create flow detail in if flow_detail is None
    :param engine_conf: engine type or URI and options (**deprecated**)
    :param backend: storage backend to use or configuration that defines it
    :param namespace: driver namespace for stevedore (or empty for default)
    :param engine: string engine type or URI string with scheme that contains
                   the engine type and any URI specific components that will
                   become part of the engine options.
    :param kwargs: arbitrary keyword arguments passed as options (merged with
                   any extracted ``engine`` and ``engine_conf`` options),
                   typically used for any engine specific options that do not
                   fit as any of the existing arguments.
    :returns: engine
    """

    kind, options = _extract_engine(engine_conf=engine_conf,
                                    engine=engine, **kwargs)

    if isinstance(backend, dict):
        backend = p_backends.fetch(backend)

    if flow_detail is None:
        flow_detail = p_utils.create_flow_detail(flow, book=book,
                                                 backend=backend)

    LOG.debug('Looking for %r engine driver in %r', kind, namespace)
    try:
        mgr = stevedore.driver.DriverManager(
            namespace, kind,
            invoke_on_load=True,
            invoke_args=(flow, flow_detail, backend, options))
        engine = mgr.driver
    except RuntimeError as e:
        raise exc.NotFound("Could not find engine '%s'" % (kind), e)
    else:
        if store:
            engine.storage.inject(store)
        return engine
예제 #6
0
 def __init__(self, flow, flow_detail=None, book=None, backend=None):
     if flow_detail is None:
         flow_detail = p_utils.create_flow_detail(flow,
                                                  book=book,
                                                  backend=backend)
     ActionEngine.__init__(self, flow,
                           storage=t_storage.Storage(flow_detail, backend))
예제 #7
0
def load_from_factory(flow_factory, factory_args=None, factory_kwargs=None,
                      store=None, book=None, engine_conf=None, backend=None,
                      namespace=ENGINES_NAMESPACE, engine=ENGINE_DEFAULT,
                      **kwargs):
    """Loads a flow from a factory function into an engine.

    Gets flow factory function (or name of it) and creates flow with
    it. Then, the flow is loaded into an engine with the :func:`load() <load>`
    function, and the factory function fully qualified name is saved to flow
    metadata so that it can be later resumed.

    :param flow_factory: function or string: function that creates the flow
    :param factory_args: list or tuple of factory positional arguments
    :param factory_kwargs: dict of factory keyword arguments

    Further arguments are interpreted as for :func:`load() <load>`.

    :returns: engine
    """

    _factory_name, factory_fun = _fetch_validate_factory(flow_factory)
    if not factory_args:
        factory_args = []
    if not factory_kwargs:
        factory_kwargs = {}
    flow = factory_fun(*factory_args, **factory_kwargs)
    if isinstance(backend, dict):
        backend = p_backends.fetch(backend)
    flow_detail = p_utils.create_flow_detail(flow, book=book, backend=backend)
    save_factory_details(flow_detail,
                         flow_factory, factory_args, factory_kwargs,
                         backend=backend)
    return load(flow=flow, store=store, flow_detail=flow_detail, book=book,
                engine_conf=engine_conf, backend=backend, namespace=namespace,
                engine=engine, **kwargs)
예제 #8
0
 def _create_engine(**kwargs):
     flow = lf.Flow('test-flow').add(utils.DummyTask())
     backend = backends.fetch({'connection': 'memory'})
     flow_detail = pu.create_flow_detail(flow, backend=backend)
     options = kwargs.copy()
     return engine.WorkerBasedActionEngine(flow, flow_detail,
                                           backend, options)
 def test_broken_flow(self):
     backend = impl_memory.MemoryBackend({})
     flow = self.make_broken_flow()
     lb = logbook.LogBook("my-log-book")
     fd = persistence_utils.create_flow_detail(flow,
                                               book=lb, backend=backend)
     storage = task_storage.Storage(fd)
     distributed_engine.DistributedEngine(flow, storage)
예제 #10
0
def load(flow, store=None, flow_detail=None, book=None,
         engine_conf=None, backend=None,
         namespace=ENGINES_NAMESPACE, engine=ENGINE_DEFAULT, **kwargs):
    """Load a flow into an engine.

    This function creates and prepares an engine to run the provided flow. All
    that is left after this returns is to run the engine with the
    engines ``run()`` method.

    Which engine to load is specified via the ``engine`` parameter. It
    can be a string that names the engine type to use, or a string that
    is a URI with a scheme that names the engine type to use and further
    options contained in the URI's host, port, and query parameters...

    Which storage backend to use is defined by the backend parameter. It
    can be backend itself, or a dictionary that is passed to
    ``taskflow.persistence.backends.fetch()`` to obtain a viable backend.

    :param flow: flow to load
    :param store: dict -- data to put to storage to satisfy flow requirements
    :param flow_detail: FlowDetail that holds the state of the flow (if one is
        not provided then one will be created for you in the provided backend)
    :param book: LogBook to create flow detail in if flow_detail is None
    :param engine_conf: engine type or URI and options (**deprecated**)
    :param backend: storage backend to use or configuration that defines it
    :param namespace: driver namespace for stevedore (or empty for default)
    :param engine: string engine type or URI string with scheme that contains
                   the engine type and any URI specific components that will
                   become part of the engine options.
    :param kwargs: arbitrary keyword arguments passed as options (merged with
                   any extracted ``engine`` and ``engine_conf`` options),
                   typically used for any engine specific options that do not
                   fit as any of the existing arguments.
    :returns: engine
    """

    kind, options = _extract_engine(engine_conf=engine_conf,
                                    engine=engine, **kwargs)

    if isinstance(backend, dict):
        backend = p_backends.fetch(backend)

    if flow_detail is None:
        flow_detail = p_utils.create_flow_detail(flow, book=book,
                                                 backend=backend)

    try:
        mgr = stevedore.driver.DriverManager(
            namespace, kind,
            invoke_on_load=True,
            invoke_args=(flow, flow_detail, backend, options))
        engine = mgr.driver
    except RuntimeError as e:
        raise exc.NotFound("Could not find engine '%s'" % (kind), e)
    else:
        if store:
            engine.storage.inject(store)
        return engine
예제 #11
0
def load(flow, store=None, flow_detail=None, book=None,
         engine_conf=None, backend=None, namespace=ENGINES_NAMESPACE):
    """Load flow into engine.

    This function creates and prepares engine to run the
    flow. All that is left is to run the engine with 'run()' method.

    Which engine to load is specified in 'engine_conf' parameter. It
    can be a string that names engine type or a dictionary which holds
    engine type (with 'engine' key) and additional engine-specific
    configuration (for example, executor for multithreaded engine).

    Which storage backend to use is defined by backend parameter. It
    can be backend itself, or a dictionary that is passed to
    taskflow.persistence.backends.fetch to obtain backend.

    :param flow: flow to load
    :param store: dict -- data to put to storage to satisfy flow requirements
    :param flow_detail: FlowDetail that holds the state of the flow (if one is
        not provided then one will be created for you in the provided backend)
    :param book: LogBook to create flow detail in if flow_detail is None
    :param engine_conf: engine type and configuration configuration
    :param backend: storage backend to use or configuration
    :param namespace: driver namespace for stevedore (default is fine
       if you don't know what is it)
    :returns: engine
    """

    if engine_conf is None:
        engine_conf = {'engine': 'default'}

    # NOTE(imelnikov): this allows simpler syntax.
    if isinstance(engine_conf, six.string_types):
        engine_conf = {'engine': engine_conf}

    if isinstance(backend, dict):
        backend = p_backends.fetch(backend)

    if flow_detail is None:
        flow_detail = p_utils.create_flow_detail(flow, book=book,
                                                 backend=backend)

    mgr = stevedore.driver.DriverManager(
        namespace, engine_conf['engine'],
        invoke_on_load=True,
        invoke_kwds={
            'conf': engine_conf.copy(),
            'flow': flow,
            'flow_detail': flow_detail,
            'backend': backend
        })
    engine = mgr.driver
    if store:
        engine.storage.inject(store)
    return engine
예제 #12
0
파일: helpers.py 프로젝트: SEJeff/taskflow
def load_from_factory(flow_factory,
                      factory_args=None,
                      factory_kwargs=None,
                      store=None,
                      book=None,
                      engine_conf=None,
                      backend=None):
    """Load flow from factory function into engine

    Gets flow factory function (or name of it) and creates flow with
    it. Then, flow is loaded into engine with load(), and factory
    function fully qualified name is saved to flow metadata so that
    it can be later resumed with resume.

    :param flow_factory: function or string: function that creates the flow
    :param factory_args: list or tuple of factory positional arguments
    :param factory_kwargs: dict of factory keyword arguments
    :param store: dict -- data to put to storage to satisfy flow requirements
    :param book: LogBook to create flow detail in
    :param engine_conf: engine type and configuration configuration
    :param backend: storage backend to use or configuration
    :returns: engine
    """

    if isinstance(flow_factory, six.string_types):
        factory_fun = importutils.import_class(flow_factory)
        factory_name = flow_factory
    else:
        factory_fun = flow_factory
        factory_name = reflection.get_callable_name(flow_factory)
        try:
            reimported = importutils.import_class(factory_name)
            assert reimported == factory_fun
        except (ImportError, AssertionError):
            raise ValueError('Flow factory %r is not reimportable by name %s' %
                             (factory_fun, factory_name))

    args = factory_args or []
    kwargs = factory_kwargs or {}
    flow = factory_fun(*args, **kwargs)
    factory_data = dict(name=factory_name, args=args, kwargs=kwargs)

    if isinstance(backend, dict):
        backend = p_backends.fetch(backend)
    flow_detail = p_utils.create_flow_detail(flow,
                                             book=book,
                                             backend=backend,
                                             meta={'factory': factory_data})
    return load(flow=flow,
                flow_detail=flow_detail,
                store=store,
                book=book,
                engine_conf=engine_conf,
                backend=backend)
예제 #13
0
 def _make_runtime(self, flow, initial_state=None):
     compilation = compiler.PatternCompiler(flow).compile()
     flow_detail = pu.create_flow_detail(flow)
     store = storage.SingleThreadedStorage(flow_detail)
     # This ensures the tasks exist in storage...
     for task in compilation.execution_graph:
         store.ensure_atom(task)
     if initial_state:
         store.set_flow_state(initial_state)
     task_notifier = misc.Notifier()
     task_executor = executor.SerialTaskExecutor()
     task_executor.start()
     self.addCleanup(task_executor.stop)
     return runtime.Runtime(compilation, store, task_notifier, task_executor)
예제 #14
0
 def _make_runtime(self, flow, initial_state=None):
     compilation = compiler.PatternCompiler(flow).compile()
     flow_detail = pu.create_flow_detail(flow)
     store = storage.SingleThreadedStorage(flow_detail)
     # This ensures the tasks exist in storage...
     for task in compilation.execution_graph:
         store.ensure_atom(task)
     if initial_state:
         store.set_flow_state(initial_state)
     task_notifier = misc.Notifier()
     task_executor = executor.SerialTaskExecutor()
     task_executor.start()
     self.addCleanup(task_executor.stop)
     return runtime.Runtime(compilation, store,
                            task_notifier, task_executor)
예제 #15
0
def load_from_factory(flow_factory,
                      factory_args=None,
                      factory_kwargs=None,
                      store=None,
                      book=None,
                      engine_conf=None,
                      backend=None,
                      namespace=ENGINES_NAMESPACE):
    """Loads a flow from a factory function into an engine.

    Gets flow factory function (or name of it) and creates flow with
    it. Then, flow is loaded into engine with load(), and factory
    function fully qualified name is saved to flow metadata so that
    it can be later resumed with resume.

    :param flow_factory: function or string: function that creates the flow
    :param factory_args: list or tuple of factory positional arguments
    :param factory_kwargs: dict of factory keyword arguments
    :param store: dict -- data to put to storage to satisfy flow requirements
    :param book: LogBook to create flow detail in
    :param engine_conf: engine type and configuration configuration
    :param backend: storage backend to use or configuration
    :param namespace: driver namespace for stevedore (default is fine
       if you don't know what is it)
    :returns: engine
    """

    _factory_name, factory_fun = _fetch_validate_factory(flow_factory)
    if not factory_args:
        factory_args = []
    if not factory_kwargs:
        factory_kwargs = {}
    flow = factory_fun(*factory_args, **factory_kwargs)
    if isinstance(backend, dict):
        backend = p_backends.fetch(backend)
    flow_detail = p_utils.create_flow_detail(flow, book=book, backend=backend)
    save_factory_details(flow_detail,
                         flow_factory,
                         factory_args,
                         factory_kwargs,
                         backend=backend)
    return load(flow=flow,
                store=store,
                flow_detail=flow_detail,
                book=book,
                engine_conf=engine_conf,
                backend=backend,
                namespace=namespace)
예제 #16
0
 def _make_runtime(self, flow, initial_state=None):
     compilation = compiler.PatternCompiler(flow).compile()
     flow_detail = pu.create_flow_detail(flow)
     store = storage.Storage(flow_detail)
     nodes_iter = compilation.execution_graph.nodes_iter(data=True)
     for node, node_attrs in nodes_iter:
         if node_attrs['kind'] in ('task', 'retry'):
             store.ensure_atom(node)
     if initial_state:
         store.set_flow_state(initial_state)
     atom_notifier = notifier.Notifier()
     task_executor = executor.SerialTaskExecutor()
     retry_executor = executor.SerialRetryExecutor()
     task_executor.start()
     self.addCleanup(task_executor.stop)
     r = runtime.Runtime(compilation, store, atom_notifier, task_executor,
                         retry_executor)
     r.compile()
     return r
    def test_multiple_root_translate(self):
        wf = graph_flow.Flow("mult-root")
        one = self.someTask()
        two = self.someTask()
        three = self.someOtherTask()

        wf.add(one, two, three)
        wf.link(one, three)
        wf.link(two, three)

        backend = impl_memory.MemoryBackend({})
        lb = logbook.LogBook("my-log-book")
        fd = persistence_utils.create_flow_detail(wf, book=lb, backend=backend)
        storage = task_storage.Storage(fd)

        engine = distributed_engine.DistributedEngine(wf, storage)
        trans = dist_translator.DistributedTranslator(engine)
        trans.translate(wf)

        self.assertEqual(2, len(engine.roots))
예제 #18
0
 def _make_runtime(self, flow, initial_state=None):
     compilation = compiler.PatternCompiler(flow).compile()
     flow_detail = pu.create_flow_detail(flow)
     store = storage.Storage(flow_detail)
     nodes_iter = compilation.execution_graph.nodes_iter(data=True)
     for node, node_attrs in nodes_iter:
         if node_attrs['kind'] in ('task', 'retry'):
             store.ensure_atom(node)
     if initial_state:
         store.set_flow_state(initial_state)
     atom_notifier = notifier.Notifier()
     task_executor = executor.SerialTaskExecutor()
     retry_executor = executor.SerialRetryExecutor()
     task_executor.start()
     self.addCleanup(task_executor.stop)
     r = runtime.Runtime(compilation, store,
                         atom_notifier, task_executor,
                         retry_executor)
     r.compile()
     return r
예제 #19
0
파일: helpers.py 프로젝트: ntt-sic/taskflow
def load_from_factory(
    flow_factory, factory_args=None, factory_kwargs=None, store=None, book=None, engine_conf=None, backend=None
):
    """Load flow from factory function into engine

    Gets flow factory function (or name of it) and creates flow with
    it. Then, flow is loaded into engine with load(), and factory
    function fully qualified name is saved to flow metadata so that
    it can be later resumed with resume.

    :param flow_factory: function or string: function that creates the flow
    :param factory_args: list or tuple of factory positional arguments
    :param factory_kwargs: dict of factory keyword arguments
    :param store: dict -- data to put to storage to satisfy flow requirements
    :param book: LogBook to create flow detail in
    :param engine_conf: engine type and configuration configuration
    :param backend: storage backend to use or configuration
    :returns: engine
    """

    if isinstance(flow_factory, six.string_types):
        factory_fun = importutils.import_class(flow_factory)
        factory_name = flow_factory
    else:
        factory_fun = flow_factory
        factory_name = reflection.get_callable_name(flow_factory)
        try:
            reimported = importutils.import_class(factory_name)
            assert reimported == factory_fun
        except (ImportError, AssertionError):
            raise ValueError("Flow factory %r is not reimportable by name %s" % (factory_fun, factory_name))

    args = factory_args or []
    kwargs = factory_kwargs or {}
    flow = factory_fun(*args, **kwargs)
    factory_data = dict(name=factory_name, args=args, kwargs=kwargs)

    if isinstance(backend, dict):
        backend = p_backends.fetch(backend)
    flow_detail = p_utils.create_flow_detail(flow, book=book, backend=backend, meta={"factory": factory_data})
    return load(flow=flow, flow_detail=flow_detail, store=store, book=book, engine_conf=engine_conf, backend=backend)
    def test_linear_translate(self):
        wf = linear_flow.Flow("linear")
        one = self.someTask()
        two = self.someOtherTask()
        wf.add(one, two)

        backend = impl_memory.MemoryBackend({})
        lb = logbook.LogBook("my-log-book")
        fd = persistence_utils.create_flow_detail(wf, book=lb, backend=backend)
        storage = task_storage.Storage(fd)

        engine = distributed_engine.DistributedEngine(wf, storage)
        trans = dist_translator.DistributedTranslator(engine)
        trans.translate(wf)

        listeners = engine.client._listeners
        hybrid = listeners[listeners.keys()[0]][0][1]

        self.assertEqual(1, len(engine.roots))
        self.assertEqual(hybrid.name,
                         "taskflow.tests.unit.test_distributed_translator"
                         ".someOtherTask")
예제 #21
0
def load_from_factory(flow_factory, factory_args=None, factory_kwargs=None,
                      store=None, book=None, engine_conf=None, backend=None,
                      namespace=ENGINES_NAMESPACE, **kwargs):
    """Loads a flow from a factory function into an engine.

    Gets flow factory function (or name of it) and creates flow with
    it. Then, flow is loaded into engine with load(), and factory
    function fully qualified name is saved to flow metadata so that
    it can be later resumed with resume.

    :param flow_factory: function or string: function that creates the flow
    :param factory_args: list or tuple of factory positional arguments
    :param factory_kwargs: dict of factory keyword arguments
    :param store: dict -- data to put to storage to satisfy flow requirements
    :param book: LogBook to create flow detail in
    :param engine_conf: engine type and configuration configuration
    :param backend: storage backend to use or configuration
    :param namespace: driver namespace for stevedore (default is fine
       if you don't know what is it)
    :returns: engine
    """

    _factory_name, factory_fun = _fetch_validate_factory(flow_factory)
    if not factory_args:
        factory_args = []
    if not factory_kwargs:
        factory_kwargs = {}
    flow = factory_fun(*factory_args, **factory_kwargs)
    if isinstance(backend, dict):
        backend = p_backends.fetch(backend)
    flow_detail = p_utils.create_flow_detail(flow, book=book, backend=backend)
    save_factory_details(flow_detail,
                         flow_factory, factory_args, factory_kwargs,
                         backend=backend)
    return load(flow=flow, store=store, flow_detail=flow_detail, book=book,
                engine_conf=engine_conf, backend=backend, namespace=namespace,
                **kwargs)
예제 #22
0
 def __init__(self, flow, flow_detail=None, book=None, backend=None,
              executor=None):
     if flow_detail is None:
         flow_detail = p_utils.create_flow_detail(flow,
                                                  book=book,
                                                  backend=backend)
     ActionEngine.__init__(self, flow,
                           storage=t_storage.ThreadSafeStorage(flow_detail,
                                                               backend))
     if executor is not None:
         self._executor = executor
         self._owns_executor = False
         self._thread_count = -1
     else:
         self._executor = None
         self._owns_executor = True
         # TODO(harlowja): allow this to be configurable??
         try:
             self._thread_count = multiprocessing.cpu_count() + 1
         except NotImplementedError:
             # NOTE(harlowja): apparently may raise so in this case we will
             # just setup two threads since its hard to know what else we
             # should do in this situation.
             self._thread_count = 2
예제 #23
0
    return lf.Flow('resume from backend example').add(
        TestTask(name='first'),
        InterruptTask(name='boom'),
        TestTask(name='second'))


### INITIALIZE PERSISTENCE ####################################

backend = get_backend()
logbook = p_utils.temporary_log_book(backend)


### CREATE AND RUN THE FLOW: FIRST ATTEMPT ####################

flow = flow_factory()
flowdetail = p_utils.create_flow_detail(flow, logbook, backend)
engine = taskflow.engines.load(flow, flow_detail=flowdetail,
                               backend=backend)

print_task_states(flowdetail, "At the beginning, there is no state")
print_wrapped("Running")
engine.run()
print_task_states(flowdetail, "After running")


### RE-CREATE, RESUME, RUN ####################################

print_wrapped("Resuming and running again")

# NOTE(harlowja): reload the flow detail from backend, this will allow us to
# resume the flow from its suspended state, but first we need to search for
예제 #24
0
from taskflow.engines.action_engine import engine
from taskflow.patterns import linear_flow as lf
from taskflow.persistence.backends import impl_memory
from taskflow import task
from taskflow.utils import persistence_utils

# INTRO: This examples shows how to run a engine using the engine iteration
# capability, in between iterations other activities occur (in this case a
# value is output to stdout); but more complicated actions can occur at the
# boundary when a engine yields its current state back to the caller.


class EchoNameTask(task.Task):
    def execute(self):
        print(self.name)


f = lf.Flow("counter")
for i in range(0, 10):
    f.add(EchoNameTask("echo_%s" % (i + 1)))

be = impl_memory.MemoryBackend()
book = persistence_utils.temporary_log_book(be)
fd = persistence_utils.create_flow_detail(f, book, be)
e = engine.SingleThreadedActionEngine(f, fd, be, {})
e.compile()
e.prepare()

for i, st in enumerate(e.run_iter(), 1):
    print("Transition %s: %s" % (i, st))
예제 #25
0
def flow_factory():
    return lf.Flow('resume from backend example').add(
        TestTask(name='first'), InterruptTask(name='boom'),
        TestTask(name='second'))


### INITIALIZE PERSISTENCE ####################################

backend = get_backend()
logbook = p_utils.temporary_log_book(backend)

### CREATE AND RUN THE FLOW: FIRST ATTEMPT ####################

flow = flow_factory()
flowdetail = p_utils.create_flow_detail(flow, logbook, backend)
engine = taskflow.engines.load(flow, flow_detail=flowdetail, backend=backend)

print_task_states(flowdetail, "At the beginning, there is no state")
print_wrapped("Running")
engine.run()
print_task_states(flowdetail, "After running")

### RE-CREATE, RESUME, RUN ####################################

print_wrapped("Resuming and running again")

# NOTE(harlowja): reload the flow detail from backend, this will allow us to
# resume the flow from its suspended state, but first we need to search for
# the right flow details in the correct logbook where things are stored.
#
 def _make_engine(self, flow, flow_detail=None):
     if flow_detail is None:
         flow_detail = p_utils.create_flow_detail(flow, self.book, self.backend)
     return eng.SingleThreadedActionEngine(flow, backend=self.backend, flow_detail=flow_detail)
예제 #27
0
 def _create_engine(**kwargs):
     flow = lf.Flow("test-flow").add(utils.DummyTask())
     backend = backends.fetch({"connection": "memory"})
     flow_detail = pu.create_flow_detail(flow, backend=backend)
     options = kwargs.copy()
     return engine.ParallelActionEngine(flow, flow_detail, backend, options)
예제 #28
0
        else:
            f.add(
                EchoTask(name="echoer_%s" % curr_value,
                         rebind={'value': curr_value}))
        curr_value = next_value
    return f


# Adjust this number to change how many engines/flows run at once.
flow_count = 1
flows = []
for i in range(0, flow_count):
    f = make_alphabet_flow(i + 1)
    flows.append(make_alphabet_flow(i + 1))
be = persistence_backends.fetch(conf={'connection': 'memory'})
book = persistence_utils.temporary_log_book(be)
engine_iters = []
for f in flows:
    fd = persistence_utils.create_flow_detail(f, book, be)
    e = engines.load(f, flow_detail=fd, backend=be, book=book)
    e.compile()
    e.storage.inject({'A': 'A'})
    e.prepare()
    engine_iters.append(e.run_iter())
while engine_iters:
    for it in list(engine_iters):
        try:
            print(six.next(it))
        except StopIteration:
            engine_iters.remove(it)
예제 #29
0
def load(flow, store=None, flow_detail=None, book=None,
         engine_conf=None, backend=None, namespace=ENGINES_NAMESPACE,
         **kwargs):
    """Load a flow into an engine.

    This function creates and prepares engine to run the
    flow. All that is left is to run the engine with 'run()' method.

    Which engine to load is specified in 'engine_conf' parameter. It
    can be a string that names engine type or a dictionary which holds
    engine type (with 'engine' key) and additional engine-specific
    configuration.

    Which storage backend to use is defined by backend parameter. It
    can be backend itself, or a dictionary that is passed to
    taskflow.persistence.backends.fetch to obtain backend.

    :param flow: flow to load
    :param store: dict -- data to put to storage to satisfy flow requirements
    :param flow_detail: FlowDetail that holds the state of the flow (if one is
        not provided then one will be created for you in the provided backend)
    :param book: LogBook to create flow detail in if flow_detail is None
    :param engine_conf: engine type and configuration configuration
    :param backend: storage backend to use or configuration
    :param namespace: driver namespace for stevedore (default is fine
       if you don't know what is it)
    :returns: engine
    """

    if engine_conf is None:
        engine_conf = {'engine': 'default'}

    # NOTE(imelnikov): this allows simpler syntax.
    if isinstance(engine_conf, six.string_types):
        engine_conf = {'engine': engine_conf}

    engine_name = engine_conf['engine']
    try:
        pieces = misc.parse_uri(engine_name)
    except (TypeError, ValueError):
        pass
    else:
        engine_name = pieces['scheme']
        engine_conf = misc.merge_uri(pieces, engine_conf.copy())

    if isinstance(backend, dict):
        backend = p_backends.fetch(backend)

    if flow_detail is None:
        flow_detail = p_utils.create_flow_detail(flow, book=book,
                                                 backend=backend)

    try:
        mgr = stevedore.driver.DriverManager(
            namespace, engine_name,
            invoke_on_load=True,
            invoke_args=(flow, flow_detail, backend, engine_conf),
            invoke_kwds=kwargs)
        engine = mgr.driver
    except RuntimeError as e:
        raise exc.NotFound("Could not find engine %s" % (engine_name), e)
    else:
        if store:
            engine.storage.inject(store)
        return engine
 def _make_engine(self, flow, flow_detail=None, executor=None):
     if flow_detail is None:
         flow_detail = p_utils.create_flow_detail(flow, self.book, self.backend)
     return eng.MultiThreadedActionEngine(flow, backend=self.backend, flow_detail=flow_detail, executor=executor)