def exec_pipeline(self, pstr): md = MDRepository() p = yaml.safe_load(six.StringIO(pstr)) print("\n{}".format(yaml.dump(p))) pl = Plumbing(p, pid="test") res = pl.process(md, state={'batch': True, 'stats': {}}) return res, md
def _get_metadata_stream(self, load_streams): try: load = [] select = [] count = 1 for stream in load_streams: curid = "%s%d" % (self.slug, count) load.append("%s as %s" % (stream[0], curid)) if stream[1] == 'SP' or stream[1] == 'IDP': select.append( "%s!//md:EntityDescriptor[md:%sSSODescriptor]" % (curid, stream[1])) else: select.append("%s" % curid) count = count + 1 if len(select) > 0: pipeline = [{'load': load}, {'select': select}] else: pipeline = [{'load': load}, 'select'] md = MDRepository() entities = Plumbing(pipeline=pipeline, id=self.slug).process(md, state={ 'batch': True, 'stats': {} }) return etree.tostring(entities) except Exception, e: raise Exception('Getting metadata from %s failed.\nError: %s' % (load_streams, e))
def when(req, condition, *values): """ Conditionally execute part of the pipeline. :param req: The request :param condition: The condition key :param values: The condition values :param opts: More Options (unused) :return: None The inner pipeline is executed if the at least one of the condition values is present for the specified key in the request state. **Examples** .. code-block:: yaml - when foo - something - when bar bill - other The condition operates on the state: if 'foo' is present in the state (with any value), then the something branch is followed. If 'bar' is present in the state with the value 'bill' then the other branch is followed. """ # log.debug("condition key: %s" % repr(condition)) c = req.state.get(condition, None) # log.debug("condition %s" % repr(c)) if c is not None: if not values or _any(values, c): return Plumbing(pipeline=req.args, pid="%s.when" % req.plumbing.id)._process(req) return req.t
def exec_pipeline(self, pstr): md = MDRepository() p = yaml.load(StringIO(pstr)) print(p) res = Plumbing(p, pid="test").process(md, state={ 'batch': True, 'stats': {} }) return res, md
def _pipe(req, *opts): """ Run the argument list as a pipleine. :param req: The request :param opts: Options (unused) :return: None Unlike fork, pipe does not copy the working document but instead operates on the current active document. The done request property is reset to False after the pipeline has been processed. This allows for a classical switch/case flow using the following construction: .. code-block:: yaml - pipe: - when a: - one - break - when b: - two - break In this case if 'a' is present in the request state, then 'one' will be executed and the 'when b' condition will not be tested at all. Note that at the topmost level the pipe is implicit and may be left out. .. code-block:: yaml - pipe: - one - two is equivalent to .. code-block:: yaml - one - two """ # req.process(Plumbing(pipeline=req.args, pid="%s.pipe" % req.plumbing.pid)) ot = Plumbing(pipeline=req.args, pid="{}.pipe".format(req.plumbing.id))._process(req) req.done = False return ot
class PipelineCallback(): """ A delayed pipeline callback used as a post for parse_metadata """ def __init__(self, entry_point, req, stats): self.entry_point = entry_point self.plumbing = Plumbing(req.plumbing.pipeline, "%s-via-%s" % (req.plumbing.id, entry_point)) self.req = req self.stats = stats def __call__(self, *args, **kwargs): t = args[0] if t is None: raise ValueError("PipelineCallback must be called with a parse-tree argument") try: return self.plumbing.process(self.req.md, state={self.entry_point: True, 'stats': self.stats}, t=t) except Exception, ex: traceback.print_exc(ex) raise ex
def fork(req, *opts): """ Make a copy of the working tree and process the arguments as a pipleline. This essentially resets the working tree and allows a new plumbing to run. Useful for producing multiple outputs from a single source. :param req: The request :param opts: Options (unused) :return: None **Examples** .. code-block:: yaml - select # select all entities - fork: - certreport - publish: output: "/tmp/annotated.xml" - fork: - xslt: stylesheet: tidy.xml - publish: output: "/tmp/clean.xml" The second fork in this example is strictly speaking not necessary since the main plumbing is still active but it may help to structure your plumbings this way. **Merging** Normally the result of the "inner" plumbing is disgarded - unless published or emit:ed to a calling client in the case of the MDX server - but by adding 'merge' to the options with an optional 'merge strategy' the behaviour can be changed to merge the result of the inner pipeline back to the parent working document. The default merge strategy is 'replace_existing' which replaces each EntityDescriptor found in the resulting document in the parent document (using the entityID as a pointer). Any python module path ('a.mod.u.le.callable') ending in a callable is accepted. If the path doesn't contain a '.' then it is assumed to reference one of the standard merge strategies in pyff.merge_strategies. For instance the following block can be used to set an attribute on a single entity: .. code-block:: yaml - fork merge: - select: http://sp.example.com/shibboleth-sp - setattr: attribute: value Note that unless you have a select statement before your fork merge you'll be merging into an empty active document which with the default merge strategy of replace_existing will result in an empty active document. To avoid this do a select before your fork, thus: .. code-block:: yaml - select - fork merge: - select: http://sp.example.com/shibboleth-sp - setattr: attribute: value """ nt = None if req.t is not None: nt = deepcopy(req.t) ip = Plumbing(pipeline=req.args, pid="%s.fork" % req.plumbing.pid) #ip.process(req.md,t=nt) ireq = Plumbing.Request(ip, req.md, nt) ip._process(ireq) if req.t is not None and ireq.t is not None and len(root(ireq.t)) > 0: if 'merge' in opts: sn = "pyff.merge_strategies.replace_existing" if opts[-1] != 'merge': sn = opts[-1] req.md.merge(req.t, ireq.t, strategy_name=sn)
def __init__(self, entry_point, req, stats): self.entry_point = entry_point self.plumbing = Plumbing(req.plumbing.pipeline, "%s-via-%s" % (req.plumbing.id, entry_point)) self.req = req self.stats = stats
def fork(req, *opts): """ Make a copy of the working tree and process the arguments as a pipleline. This essentially resets the working tree and allows a new plumbing to run. Useful for producing multiple outputs from a single source. :param req: The request :param opts: Options (unused) :return: None **Examples** .. code-block:: yaml - select # select all entities - fork: - certreport - publish: output: "/tmp/annotated.xml" - fork: - xslt: stylesheet: tidy.xml - publish: output: "/tmp/clean.xml" The second fork in this example is strictly speaking not necessary since the main plumbing is still active but it may help to structure your plumbings this way. **Merging** Normally the result of the "inner" plumbing is disgarded - unless published or emit:ed to a calling client in the case of the MDX server - but by adding 'merge' to the options with an optional 'merge strategy' the behaviour can be changed to merge the result of the inner pipeline back to the parent working document. The default merge strategy is 'replace_existing' which replaces each EntityDescriptor found in the resulting document in the parent document (using the entityID as a pointer). Any python module path ('a.mod.u.le.callable') ending in a callable is accepted. If the path doesn't contain a '.' then it is assumed to reference one of the standard merge strategies in pyff.merge_strategies. For instance the following block can be used to set an attribute on a single entity: .. code-block:: yaml - fork merge: - select: http://sp.example.com/shibboleth-sp - setattr: attribute: value Note that unless you have a select statement before your fork merge you'll be merging into an empty active document which with the default merge strategy of replace_existing will result in an empty active document. To avoid this do a select before your fork, thus: .. code-block:: yaml - select - fork merge: - select: http://sp.example.com/shibboleth-sp - setattr: attribute: value """ nt = None if req.t is not None: nt = deepcopy(req.t) ip = Plumbing(pipeline=req.args, pid="{}.fork".format(req.plumbing.pid)) # ip.process(req.md,t=nt) ireq = Plumbing.Request(ip, req.md, nt) ip._process(ireq) if req.t is not None and ireq.t is not None and len(root(ireq.t)) > 0: if 'merge' in opts: sn = "pyff.merge_strategies.replace_existing" if opts[-1] != 'merge': sn = opts[-1] req.md.merge(req.t, ireq.t, strategy_name=sn)