Beispiel #1
0
def load_from_factory(flow_factory,
                      factory_args=None,
                      factory_kwargs=None,
                      store=None,
                      book=None,
                      engine_conf=None,
                      backend=None):
    """Load flow from factory function into engine

    Gets flow factory function (or name of it) and creates flow with
    it. Then, flow is loaded into engine with load(), and factory
    function fully qualified name is saved to flow metadata so that
    it can be later resumed with resume.

    :param flow_factory: function or string: function that creates the flow
    :param factory_args: list or tuple of factory positional arguments
    :param factory_kwargs: dict of factory keyword arguments
    :param store: dict -- data to put to storage to satisfy flow requirements
    :param book: LogBook to create flow detail in
    :param engine_conf: engine type and configuration configuration
    :param backend: storage backend to use or configuration
    :returns: engine
    """

    if isinstance(flow_factory, six.string_types):
        factory_fun = importutils.import_class(flow_factory)
        factory_name = flow_factory
    else:
        factory_fun = flow_factory
        factory_name = reflection.get_callable_name(flow_factory)
        try:
            reimported = importutils.import_class(factory_name)
            assert reimported == factory_fun
        except (ImportError, AssertionError):
            raise ValueError('Flow factory %r is not reimportable by name %s' %
                             (factory_fun, factory_name))

    args = factory_args or []
    kwargs = factory_kwargs or {}
    flow = factory_fun(*args, **kwargs)
    factory_data = dict(name=factory_name, args=args, kwargs=kwargs)

    if isinstance(backend, dict):
        backend = p_backends.fetch(backend)
    flow_detail = p_utils.create_flow_detail(flow,
                                             book=book,
                                             backend=backend,
                                             meta={'factory': factory_data})
    return load(flow=flow,
                flow_detail=flow_detail,
                store=store,
                book=book,
                engine_conf=engine_conf,
                backend=backend)
Beispiel #2
0
def _fetch_validate_factory(flow_factory):
    if isinstance(flow_factory, six.string_types):
        factory_fun = importutils.import_class(flow_factory)
        factory_name = flow_factory
    else:
        factory_fun = flow_factory
        factory_name = reflection.get_callable_name(flow_factory)
        try:
            reimported = importutils.import_class(factory_name)
            assert reimported == factory_fun
        except (ImportError, AssertionError):
            raise ValueError('Flow factory %r is not reimportable by name %s'
                             % (factory_fun, factory_name))
    return (factory_name, factory_fun)
Beispiel #3
0
def _fetch_validate_factory(flow_factory):
    if isinstance(flow_factory, six.string_types):
        factory_fun = importutils.import_class(flow_factory)
        factory_name = flow_factory
    else:
        factory_fun = flow_factory
        factory_name = reflection.get_callable_name(flow_factory)
        try:
            reimported = importutils.import_class(factory_name)
            assert reimported == factory_fun
        except (ImportError, AssertionError):
            raise ValueError('Flow factory %r is not reimportable by name %s' %
                             (factory_fun, factory_name))
    return (factory_name, factory_fun)
Beispiel #4
0
    def _derive_endpoints(tasks):
        """Derive endpoints from list of strings, classes or packages."""
        derived_tasks = set()
        for item in tasks:
            module = None
            if isinstance(item, six.string_types):
                try:
                    pkg, cls = item.split(":")
                except ValueError:
                    module = importutils.import_module(item)
                else:
                    obj = importutils.import_class("%s.%s" % (pkg, cls))
                    if not reflection.is_subclass(obj, t_task.BaseTask):
                        raise TypeError("Item %s is not a BaseTask subclass" % item)
                    derived_tasks.add(obj)
            elif isinstance(item, types.ModuleType):
                module = item
            elif reflection.is_subclass(item, t_task.BaseTask):
                derived_tasks.add(item)
            else:
                raise TypeError("Item %s unexpected type: %s" % (item, type(item)))

            # derive tasks
            if module is not None:
                for name, obj in inspect.getmembers(module):
                    if reflection.is_subclass(obj, t_task.BaseTask):
                        derived_tasks.add(obj)

        return [endpoint.Endpoint(task) for task in derived_tasks]
Beispiel #5
0
    def _derive_endpoints(tasks):
        """Derive endpoints from list of strings, classes or packages."""
        derived_tasks = set()
        for item in tasks:
            module = None
            if isinstance(item, six.string_types):
                try:
                    pkg, cls = item.split(':')
                except ValueError:
                    module = importutils.import_module(item)
                else:
                    obj = importutils.import_class('%s.%s' % (pkg, cls))
                    if not reflection.is_subclass(obj, t_task.BaseTask):
                        raise TypeError("Item %s is not a BaseTask subclass" %
                                        item)
                    derived_tasks.add(obj)
            elif isinstance(item, types.ModuleType):
                module = item
            elif reflection.is_subclass(item, t_task.BaseTask):
                derived_tasks.add(item)
            else:
                raise TypeError("Item %s unexpected type: %s" %
                                (item, type(item)))

            # derive tasks
            if module is not None:
                for name, obj in inspect.getmembers(module):
                    if reflection.is_subclass(obj, t_task.BaseTask):
                        derived_tasks.add(obj)

        return [endpoint.Endpoint(task) for task in derived_tasks]
Beispiel #6
0
def flow_from_detail(flow_detail):
    """Reloads a flow previously saved.

    Gets the flow factories name and any arguments and keyword arguments from
    the flow details metadata, and then calls that factory to recreate the
    flow.

    :param flow_detail: FlowDetail that holds state of the flow to load
    """
    try:
        factory_data = flow_detail.meta['factory']
    except (KeyError, AttributeError, TypeError):
        raise ValueError('Cannot reconstruct flow %s %s: '
                         'no factory information saved.'
                         % (flow_detail.name, flow_detail.uuid))

    try:
        factory_fun = importutils.import_class(factory_data['name'])
    except (KeyError, ImportError):
        raise ImportError('Could not import factory for flow %s %s'
                          % (flow_detail.name, flow_detail.uuid))

    args = factory_data.get('args', ())
    kwargs = factory_data.get('kwargs', {})
    return factory_fun(*args, **kwargs)
Beispiel #7
0
def load_from_factory(
    flow_factory, factory_args=None, factory_kwargs=None, store=None, book=None, engine_conf=None, backend=None
):
    """Load flow from factory function into engine

    Gets flow factory function (or name of it) and creates flow with
    it. Then, flow is loaded into engine with load(), and factory
    function fully qualified name is saved to flow metadata so that
    it can be later resumed with resume.

    :param flow_factory: function or string: function that creates the flow
    :param factory_args: list or tuple of factory positional arguments
    :param factory_kwargs: dict of factory keyword arguments
    :param store: dict -- data to put to storage to satisfy flow requirements
    :param book: LogBook to create flow detail in
    :param engine_conf: engine type and configuration configuration
    :param backend: storage backend to use or configuration
    :returns: engine
    """

    if isinstance(flow_factory, six.string_types):
        factory_fun = importutils.import_class(flow_factory)
        factory_name = flow_factory
    else:
        factory_fun = flow_factory
        factory_name = reflection.get_callable_name(flow_factory)
        try:
            reimported = importutils.import_class(factory_name)
            assert reimported == factory_fun
        except (ImportError, AssertionError):
            raise ValueError("Flow factory %r is not reimportable by name %s" % (factory_fun, factory_name))

    args = factory_args or []
    kwargs = factory_kwargs or {}
    flow = factory_fun(*args, **kwargs)
    factory_data = dict(name=factory_name, args=args, kwargs=kwargs)

    if isinstance(backend, dict):
        backend = p_backends.fetch(backend)
    flow_detail = p_utils.create_flow_detail(flow, book=book, backend=backend, meta={"factory": factory_data})
    return load(flow=flow, flow_detail=flow_detail, store=store, book=book, engine_conf=engine_conf, backend=backend)
Beispiel #8
0
def find_subclasses(locations, base_cls, exclude_hidden=True):
    """Finds subclass types in the given locations.

    This will examines the given locations for types which are subclasses of
    the base class type provided and returns the found subclasses (or fails
    with exceptions if this introspection can not be accomplished).

    If a string is provided as one of the locations it will be imported and
    examined if it is a subclass of the base class. If a module is given,
    all of its members will be examined for attributes which are subclasses of
    the base class. If a type itself is given it will be examined for being a
    subclass of the base class.
    """
    derived = set()
    for item in locations:
        module = None
        if isinstance(item, six.string_types):
            try:
                pkg, cls = item.split(':')
            except ValueError:
                module = importutils.import_module(item)
            else:
                obj = importutils.import_class('%s.%s' % (pkg, cls))
                if not is_subclass(obj, base_cls):
                    raise TypeError("Item %s is not a %s subclass" %
                                    (item, base_cls))
                derived.add(obj)
        elif isinstance(item, types.ModuleType):
            module = item
        elif is_subclass(item, base_cls):
            derived.add(item)
        else:
            raise TypeError("Item %s unexpected type: %s" %
                            (item, type(item)))
        # If it's a module derive objects from it if we can.
        if module is not None:
            for (_name, obj) in _get_members(module, exclude_hidden):
                if is_subclass(obj, base_cls):
                    derived.add(obj)
    return derived
Beispiel #9
0
def flow_from_detail(flow_detail):
    """Recreate flow previously loaded with load_form_factory

    Gets flow factory name from metadata, calls it to recreate the flow

    :param flow_detail: FlowDetail that holds state of the flow to load
    """
    try:
        factory_data = flow_detail.meta["factory"]
    except (KeyError, AttributeError, TypeError):
        raise ValueError(
            "Cannot reconstruct flow %s %s: " "no factory information saved." % (flow_detail.name, flow_detail.uuid)
        )

    try:
        factory_fun = importutils.import_class(factory_data["name"])
    except (KeyError, ImportError):
        raise ImportError("Could not import factory for flow %s %s" % (flow_detail.name, flow_detail.uuid))

    args = factory_data.get("args", ())
    kwargs = factory_data.get("kwargs", {})
    return factory_fun(*args, **kwargs)
Beispiel #10
0
def flow_from_detail(flow_detail):
    """Recreate flow previously loaded with load_form_factory

    Gets flow factory name from metadata, calls it to recreate the flow

    :param flow_detail: FlowDetail that holds state of the flow to load
    """
    try:
        factory_data = flow_detail.meta['factory']
    except (KeyError, AttributeError, TypeError):
        raise ValueError('Cannot reconstruct flow %s %s: '
                         'no factory information saved.' %
                         (flow_detail.name, flow_detail.uuid))

    try:
        factory_fun = importutils.import_class(factory_data['name'])
    except (KeyError, ImportError):
        raise ImportError('Could not import factory for flow %s %s' %
                          (flow_detail.name, flow_detail.uuid))

    args = factory_data.get('args', ())
    kwargs = factory_data.get('kwargs', {})
    return factory_fun(*args, **kwargs)