def load_from_factory(flow_factory, factory_args=None, factory_kwargs=None, store=None, book=None, engine_conf=None, backend=None): """Load flow from factory function into engine Gets flow factory function (or name of it) and creates flow with it. Then, flow is loaded into engine with load(), and factory function fully qualified name is saved to flow metadata so that it can be later resumed with resume. :param flow_factory: function or string: function that creates the flow :param factory_args: list or tuple of factory positional arguments :param factory_kwargs: dict of factory keyword arguments :param store: dict -- data to put to storage to satisfy flow requirements :param book: LogBook to create flow detail in :param engine_conf: engine type and configuration configuration :param backend: storage backend to use or configuration :returns: engine """ if isinstance(flow_factory, six.string_types): factory_fun = importutils.import_class(flow_factory) factory_name = flow_factory else: factory_fun = flow_factory factory_name = reflection.get_callable_name(flow_factory) try: reimported = importutils.import_class(factory_name) assert reimported == factory_fun except (ImportError, AssertionError): raise ValueError('Flow factory %r is not reimportable by name %s' % (factory_fun, factory_name)) args = factory_args or [] kwargs = factory_kwargs or {} flow = factory_fun(*args, **kwargs) factory_data = dict(name=factory_name, args=args, kwargs=kwargs) if isinstance(backend, dict): backend = p_backends.fetch(backend) flow_detail = p_utils.create_flow_detail(flow, book=book, backend=backend, meta={'factory': factory_data}) return load(flow=flow, flow_detail=flow_detail, store=store, book=book, engine_conf=engine_conf, backend=backend)
def _fetch_validate_factory(flow_factory): if isinstance(flow_factory, six.string_types): factory_fun = importutils.import_class(flow_factory) factory_name = flow_factory else: factory_fun = flow_factory factory_name = reflection.get_callable_name(flow_factory) try: reimported = importutils.import_class(factory_name) assert reimported == factory_fun except (ImportError, AssertionError): raise ValueError('Flow factory %r is not reimportable by name %s' % (factory_fun, factory_name)) return (factory_name, factory_fun)
def _derive_endpoints(tasks): """Derive endpoints from list of strings, classes or packages.""" derived_tasks = set() for item in tasks: module = None if isinstance(item, six.string_types): try: pkg, cls = item.split(":") except ValueError: module = importutils.import_module(item) else: obj = importutils.import_class("%s.%s" % (pkg, cls)) if not reflection.is_subclass(obj, t_task.BaseTask): raise TypeError("Item %s is not a BaseTask subclass" % item) derived_tasks.add(obj) elif isinstance(item, types.ModuleType): module = item elif reflection.is_subclass(item, t_task.BaseTask): derived_tasks.add(item) else: raise TypeError("Item %s unexpected type: %s" % (item, type(item))) # derive tasks if module is not None: for name, obj in inspect.getmembers(module): if reflection.is_subclass(obj, t_task.BaseTask): derived_tasks.add(obj) return [endpoint.Endpoint(task) for task in derived_tasks]
def _derive_endpoints(tasks): """Derive endpoints from list of strings, classes or packages.""" derived_tasks = set() for item in tasks: module = None if isinstance(item, six.string_types): try: pkg, cls = item.split(':') except ValueError: module = importutils.import_module(item) else: obj = importutils.import_class('%s.%s' % (pkg, cls)) if not reflection.is_subclass(obj, t_task.BaseTask): raise TypeError("Item %s is not a BaseTask subclass" % item) derived_tasks.add(obj) elif isinstance(item, types.ModuleType): module = item elif reflection.is_subclass(item, t_task.BaseTask): derived_tasks.add(item) else: raise TypeError("Item %s unexpected type: %s" % (item, type(item))) # derive tasks if module is not None: for name, obj in inspect.getmembers(module): if reflection.is_subclass(obj, t_task.BaseTask): derived_tasks.add(obj) return [endpoint.Endpoint(task) for task in derived_tasks]
def flow_from_detail(flow_detail): """Reloads a flow previously saved. Gets the flow factories name and any arguments and keyword arguments from the flow details metadata, and then calls that factory to recreate the flow. :param flow_detail: FlowDetail that holds state of the flow to load """ try: factory_data = flow_detail.meta['factory'] except (KeyError, AttributeError, TypeError): raise ValueError('Cannot reconstruct flow %s %s: ' 'no factory information saved.' % (flow_detail.name, flow_detail.uuid)) try: factory_fun = importutils.import_class(factory_data['name']) except (KeyError, ImportError): raise ImportError('Could not import factory for flow %s %s' % (flow_detail.name, flow_detail.uuid)) args = factory_data.get('args', ()) kwargs = factory_data.get('kwargs', {}) return factory_fun(*args, **kwargs)
def load_from_factory( flow_factory, factory_args=None, factory_kwargs=None, store=None, book=None, engine_conf=None, backend=None ): """Load flow from factory function into engine Gets flow factory function (or name of it) and creates flow with it. Then, flow is loaded into engine with load(), and factory function fully qualified name is saved to flow metadata so that it can be later resumed with resume. :param flow_factory: function or string: function that creates the flow :param factory_args: list or tuple of factory positional arguments :param factory_kwargs: dict of factory keyword arguments :param store: dict -- data to put to storage to satisfy flow requirements :param book: LogBook to create flow detail in :param engine_conf: engine type and configuration configuration :param backend: storage backend to use or configuration :returns: engine """ if isinstance(flow_factory, six.string_types): factory_fun = importutils.import_class(flow_factory) factory_name = flow_factory else: factory_fun = flow_factory factory_name = reflection.get_callable_name(flow_factory) try: reimported = importutils.import_class(factory_name) assert reimported == factory_fun except (ImportError, AssertionError): raise ValueError("Flow factory %r is not reimportable by name %s" % (factory_fun, factory_name)) args = factory_args or [] kwargs = factory_kwargs or {} flow = factory_fun(*args, **kwargs) factory_data = dict(name=factory_name, args=args, kwargs=kwargs) if isinstance(backend, dict): backend = p_backends.fetch(backend) flow_detail = p_utils.create_flow_detail(flow, book=book, backend=backend, meta={"factory": factory_data}) return load(flow=flow, flow_detail=flow_detail, store=store, book=book, engine_conf=engine_conf, backend=backend)
def find_subclasses(locations, base_cls, exclude_hidden=True): """Finds subclass types in the given locations. This will examines the given locations for types which are subclasses of the base class type provided and returns the found subclasses (or fails with exceptions if this introspection can not be accomplished). If a string is provided as one of the locations it will be imported and examined if it is a subclass of the base class. If a module is given, all of its members will be examined for attributes which are subclasses of the base class. If a type itself is given it will be examined for being a subclass of the base class. """ derived = set() for item in locations: module = None if isinstance(item, six.string_types): try: pkg, cls = item.split(':') except ValueError: module = importutils.import_module(item) else: obj = importutils.import_class('%s.%s' % (pkg, cls)) if not is_subclass(obj, base_cls): raise TypeError("Item %s is not a %s subclass" % (item, base_cls)) derived.add(obj) elif isinstance(item, types.ModuleType): module = item elif is_subclass(item, base_cls): derived.add(item) else: raise TypeError("Item %s unexpected type: %s" % (item, type(item))) # If it's a module derive objects from it if we can. if module is not None: for (_name, obj) in _get_members(module, exclude_hidden): if is_subclass(obj, base_cls): derived.add(obj) return derived
def flow_from_detail(flow_detail): """Recreate flow previously loaded with load_form_factory Gets flow factory name from metadata, calls it to recreate the flow :param flow_detail: FlowDetail that holds state of the flow to load """ try: factory_data = flow_detail.meta["factory"] except (KeyError, AttributeError, TypeError): raise ValueError( "Cannot reconstruct flow %s %s: " "no factory information saved." % (flow_detail.name, flow_detail.uuid) ) try: factory_fun = importutils.import_class(factory_data["name"]) except (KeyError, ImportError): raise ImportError("Could not import factory for flow %s %s" % (flow_detail.name, flow_detail.uuid)) args = factory_data.get("args", ()) kwargs = factory_data.get("kwargs", {}) return factory_fun(*args, **kwargs)
def flow_from_detail(flow_detail): """Recreate flow previously loaded with load_form_factory Gets flow factory name from metadata, calls it to recreate the flow :param flow_detail: FlowDetail that holds state of the flow to load """ try: factory_data = flow_detail.meta['factory'] except (KeyError, AttributeError, TypeError): raise ValueError('Cannot reconstruct flow %s %s: ' 'no factory information saved.' % (flow_detail.name, flow_detail.uuid)) try: factory_fun = importutils.import_class(factory_data['name']) except (KeyError, ImportError): raise ImportError('Could not import factory for flow %s %s' % (flow_detail.name, flow_detail.uuid)) args = factory_data.get('args', ()) kwargs = factory_data.get('kwargs', {}) return factory_fun(*args, **kwargs)