def exec_pipeline(self, pstr):
md = MDRepository()
p = yaml.safe_load(six.StringIO(pstr))
print("\n{}".format(yaml.dump(p)))
pl = Plumbing(p, pid="test")
res = pl.process(md, state={'batch': True, 'stats': {}})
return res, md
def _get_metadata_stream(self, load_streams):
try:
load = []
select = []
count = 1
for stream in load_streams:
curid = "%s%d" % (self.slug, count)
load.append("%s as %s" % (stream[0], curid))
if stream[1] == 'SP' or stream[1] == 'IDP':
select.append(
"%s!//md:EntityDescriptor[md:%sSSODescriptor]" %
(curid, stream[1]))
else:
select.append("%s" % curid)
count = count + 1
if len(select) > 0:
pipeline = [{'load': load}, {'select': select}]
else:
pipeline = [{'load': load}, 'select']
md = MDRepository()
entities = Plumbing(pipeline=pipeline,
id=self.slug).process(md,
state={
'batch': True,
'stats': {}
})
return etree.tostring(entities)
except Exception, e:
raise Exception('Getting metadata from %s failed.\nError: %s' %
(load_streams, e))
def when(req, condition, *values):
"""
Conditionally execute part of the pipeline.
:param req: The request
:param condition: The condition key
:param values: The condition values
:param opts: More Options (unused)
:return: None
The inner pipeline is executed if the at least one of the condition values is present for the specified key in
the request state.
**Examples**
.. code-block:: yaml
- when foo
- something
- when bar bill
- other
The condition operates on the state: if 'foo' is present in the state (with any value), then the something branch is
followed. If 'bar' is present in the state with the value 'bill' then the other branch is followed.
"""
# log.debug("condition key: %s" % repr(condition))
c = req.state.get(condition, None)
# log.debug("condition %s" % repr(c))
if c is not None:
if not values or _any(values, c):
return Plumbing(pipeline=req.args,
pid="%s.when" % req.plumbing.id)._process(req)
return req.t
def exec_pipeline(self, pstr):
md = MDRepository()
p = yaml.load(StringIO(pstr))
print(p)
res = Plumbing(p, pid="test").process(md,
state={
'batch': True,
'stats': {}
})
return res, md
def _pipe(req, *opts):
"""
Run the argument list as a pipleine.
:param req: The request
:param opts: Options (unused)
:return: None
Unlike fork, pipe does not copy the working document but instead operates on the current active document. The done
request property is reset to False after the pipeline has been processed. This allows for a classical switch/case
flow using the following construction:
.. code-block:: yaml
- pipe:
- when a:
- one
- break
- when b:
- two
- break
In this case if 'a' is present in the request state, then 'one' will be executed and the 'when b' condition will not
be tested at all. Note that at the topmost level the pipe is implicit and may be left out.
.. code-block:: yaml
- pipe:
- one
- two
is equivalent to
.. code-block:: yaml
- one
- two
"""
# req.process(Plumbing(pipeline=req.args, pid="%s.pipe" % req.plumbing.pid))
ot = Plumbing(pipeline=req.args,
pid="{}.pipe".format(req.plumbing.id))._process(req)
req.done = False
return ot
class PipelineCallback():
"""
A delayed pipeline callback used as a post for parse_metadata
"""
def __init__(self, entry_point, req, stats):
self.entry_point = entry_point
self.plumbing = Plumbing(req.plumbing.pipeline, "%s-via-%s" % (req.plumbing.id, entry_point))
self.req = req
self.stats = stats
def __call__(self, *args, **kwargs):
t = args[0]
if t is None:
raise ValueError("PipelineCallback must be called with a parse-tree argument")
try:
return self.plumbing.process(self.req.md, state={self.entry_point: True, 'stats': self.stats}, t=t)
except Exception, ex:
traceback.print_exc(ex)
raise ex
def fork(req, *opts):
"""
Make a copy of the working tree and process the arguments as a pipleline. This essentially resets the working
tree and allows a new plumbing to run. Useful for producing multiple outputs from a single source.
:param req: The request
:param opts: Options (unused)
:return: None
**Examples**
.. code-block:: yaml
- select # select all entities
- fork:
- certreport
- publish:
output: "/tmp/annotated.xml"
- fork:
- xslt:
stylesheet: tidy.xml
- publish:
output: "/tmp/clean.xml"
The second fork in this example is strictly speaking not necessary since the main plumbing is still active
but it may help to structure your plumbings this way.
**Merging**
Normally the result of the "inner" plumbing is disgarded - unless published or emit:ed to a calling client
in the case of the MDX server - but by adding 'merge' to the options with an optional 'merge strategy' the
behaviour can be changed to merge the result of the inner pipeline back to the parent working document.
The default merge strategy is 'replace_existing' which replaces each EntityDescriptor found in the resulting
document in the parent document (using the entityID as a pointer). Any python module path ('a.mod.u.le.callable')
ending in a callable is accepted. If the path doesn't contain a '.' then it is assumed to reference one of the
standard merge strategies in pyff.merge_strategies.
For instance the following block can be used to set an attribute on a single entity:
.. code-block:: yaml
- fork merge:
- select: http://sp.example.com/shibboleth-sp
- setattr:
attribute: value
Note that unless you have a select statement before your fork merge you'll be merging into an empty
active document which with the default merge strategy of replace_existing will result in an empty
active document. To avoid this do a select before your fork, thus:
.. code-block:: yaml
- select
- fork merge:
- select: http://sp.example.com/shibboleth-sp
- setattr:
attribute: value
"""
nt = None
if req.t is not None:
nt = deepcopy(req.t)
ip = Plumbing(pipeline=req.args, pid="%s.fork" % req.plumbing.pid)
#ip.process(req.md,t=nt)
ireq = Plumbing.Request(ip, req.md, nt)
ip._process(ireq)
if req.t is not None and ireq.t is not None and len(root(ireq.t)) > 0:
if 'merge' in opts:
sn = "pyff.merge_strategies.replace_existing"
if opts[-1] != 'merge':
sn = opts[-1]
req.md.merge(req.t, ireq.t, strategy_name=sn)
def __init__(self, entry_point, req, stats):
self.entry_point = entry_point
self.plumbing = Plumbing(req.plumbing.pipeline, "%s-via-%s" % (req.plumbing.id, entry_point))
self.req = req
self.stats = stats
def fork(req, *opts):
"""
Make a copy of the working tree and process the arguments as a pipleline. This essentially resets the working
tree and allows a new plumbing to run. Useful for producing multiple outputs from a single source.
:param req: The request
:param opts: Options (unused)
:return: None
**Examples**
.. code-block:: yaml
- select # select all entities
- fork:
- certreport
- publish:
output: "/tmp/annotated.xml"
- fork:
- xslt:
stylesheet: tidy.xml
- publish:
output: "/tmp/clean.xml"
The second fork in this example is strictly speaking not necessary since the main plumbing is still active
but it may help to structure your plumbings this way.
**Merging**
Normally the result of the "inner" plumbing is disgarded - unless published or emit:ed to a calling client
in the case of the MDX server - but by adding 'merge' to the options with an optional 'merge strategy' the
behaviour can be changed to merge the result of the inner pipeline back to the parent working document.
The default merge strategy is 'replace_existing' which replaces each EntityDescriptor found in the resulting
document in the parent document (using the entityID as a pointer). Any python module path ('a.mod.u.le.callable')
ending in a callable is accepted. If the path doesn't contain a '.' then it is assumed to reference one of the
standard merge strategies in pyff.merge_strategies.
For instance the following block can be used to set an attribute on a single entity:
.. code-block:: yaml
- fork merge:
- select: http://sp.example.com/shibboleth-sp
- setattr:
attribute: value
Note that unless you have a select statement before your fork merge you'll be merging into an empty
active document which with the default merge strategy of replace_existing will result in an empty
active document. To avoid this do a select before your fork, thus:
.. code-block:: yaml
- select
- fork merge:
- select: http://sp.example.com/shibboleth-sp
- setattr:
attribute: value
"""
nt = None
if req.t is not None:
nt = deepcopy(req.t)
ip = Plumbing(pipeline=req.args, pid="{}.fork".format(req.plumbing.pid))
# ip.process(req.md,t=nt)
ireq = Plumbing.Request(ip, req.md, nt)
ip._process(ireq)
if req.t is not None and ireq.t is not None and len(root(ireq.t)) > 0:
if 'merge' in opts:
sn = "pyff.merge_strategies.replace_existing"
if opts[-1] != 'merge':
sn = opts[-1]
req.md.merge(req.t, ireq.t, strategy_name=sn)
