def process(self, baton):
groups = collections.defaultdict(list)
items = util.dict_get_path(baton, self.input_path, list())
for item in items:
value = util.dict_get_path(item, self.key_path, self.fallback)
groups[value].append(item)
util.dict_set_path(baton, self.output_path, groups)
return baton
def process(self, baton):
value = util.dict_get_path(baton, self.counter_path)
delta = util.dict_get_path(baton, self.delta_path)
rate = value / delta
log_string = self.format % dict(rate=rate, delta=delta, value=value)
if(rate != 0 or self.report_zero):
log.info(log_string)
return baton
def test_getting_attribute(self):
class Bar(object):
def __init__(self, wrapped):
self.wrapped = wrapped
d = dict(a=Bar(dict(b='c')))
self.assertEquals(util.dict_get_path(d, 'a.wrapped.b'), 'c')
self.assertEquals(util.dict_get_path(d, 'a.nonexistent', 'fallback'), 'fallback')
def process(self, baton):
deferred = util.dict_get_path(baton, self.deferred_path)
result = self.result
if result is Ellipsis:
result = util.dict_get_path(baton, self.result_path)
deferred.callback(result)
return baton
def test_setdefault_through_attributes(self):
class Bar(object):
def __init__(self, wrapped):
self.wrapped = wrapped
d = dict(a=Bar(dict(b='c')))
self.assertEquals(util.dict_setdefault_path(d, 'a.wrapped.b', 'd'), 'c')
self.assertEquals(util.dict_get_path(d, 'a.wrapped.b'), 'c')
self.assertEquals(util.dict_setdefault_path(d, 'a.wrapped.c', 'd'), 'd')
self.assertEquals(util.dict_get_path(d, 'a.wrapped.c'), 'd')
def test_not_a_dictionary(self):
class Foo(object):
def __getitem__(self, key):
if key == 'foo':
return self
return key
foo = Foo()
self.assertEquals(util.dict_get_path(foo, 'foo.foo'), foo)
self.assertEquals(util.dict_get_path(foo, 'foo.foo.bar'), 'bar')
def process(self, baton):
root = yield self.root_dependency.wait_for_resource()
args = util.dict_get_path(baton, self.args_path, list()) if self.args_path is not None else list()
kwargs = util.dict_get_path(baton, self.kwargs_path, dict()) if self.kwargs_path is not None else dict()
result = yield root.callRemote(self.remote_name, *args, **kwargs)
if self.output_path == "":
defer.returnValue(result)
util.dict_set_path(baton, self.output_path, result)
defer.returnValue(baton)
def test_setting_attribute(self):
class Bar(object):
def __init__(self, wrapped):
self.wrapped = wrapped
d = dict(a=Bar(dict(b=Bar('c'))))
util.dict_set_path(d, 'a.wrapped.b.wrapped', 'not c')
self.assertEquals(util.dict_get_path(d, 'a.wrapped.b.wrapped'), 'not c')
util.dict_set_path(d, 'a.wrapped', 'not a bar')
self.assertEquals(util.dict_get_path(d, 'a.wrapped'), 'not a bar')
self.assertRaises(AttributeError, util.dict_set_path, d, 'a.no-such-attribute', 'whatever')
def process(self, baton):
message = util.dict_get_path(baton, self.message_path)
from_addr = self.from_addr
if not from_addr:
from_addr = util.dict_get_path(baton, self.from_path)
to_addr = self.to_addr
if not to_addr:
to_addr = util.dict_get_path(baton, self.to_path)
yield smtp_provider.send_mail(from_addr, to_addr, message, **self.configuration)
defer.returnValue(baton)
def process(self, baton):
# For every specified attribute path, check if there's
# anything processable. If there is, process it and store the
# result in the supplied attribute.
ds = []
for mapped_attribute in self.mapping:
input_path = self.input_path_prefix + mapped_attribute.get('input_path', '')
input_baton = util.dict_get_path(baton, input_path, Ellipsis)
if input_baton is Ellipsis:
continue
pipeline_name = mapped_attribute['pipeline']
output_path = self.output_path_prefix + mapped_attribute.get('output_path', pipeline_name)
pipeline = self.get_pipeline(pipeline_name)
input_baton = self.preprocess_baton(self._maybe_copy(input_baton))
d = defer.maybeDeferred(pipeline.process, input_baton)
d.addCallback(lambda _, resulting_baton=input_baton, output_path=output_path: util.dict_set_path(baton, output_path, resulting_baton))
ds.append(d)
results = yield defer.DeferredList(ds)
for success, result in results:
if not success:
# It's a failure. Reraise it, so the resulting
# stack-trace is correct. Note that if there are
# failures in multiple pipelines, only the first one
# is raised.
result.raiseException()
defer.returnValue(baton)
def get_consumers(self, baton):
input = util.dict_get_path(baton, self.input_path)
should_stop = self.decider(input)
if should_stop:
return []
else:
return self.consumers
def __init__(self, site_name, site_configuration):
service.MultiService.__init__(self)
self.site_name = site_name
self.site_configuration = site_configuration
self.log_exceptions_level = site_configuration.get("log_exceptions", None)
self.debug_configuration = util.dict_get_path(self.site_configuration, "debug", dict())
def process(self, baton):
input_for_conditional = util.dict_get_path(baton, self.condition_input_path)
should_invoke_pipeline = self.lambda_(input_for_conditional)
if not should_invoke_pipeline:
return baton
return super(ConditionalPipelineRunner, self).process(baton)
def process(self, baton):
input = util.dict_get_path(baton, self.input_path)
if input:
output = self.format % self.formatter(input)
if self.encode:
output = output.encode(self.encode)
self.file.write(output)
return baton
def process(self, baton):
reporter = util.dict_get_path(baton, self.reporter_path)
loader = statustest.StatusTestLoader(self.get_namespace(baton))
for suite in self.get_suites(baton, loader):
yield suite.run(reporter)
defer.returnValue(baton)
def process(self, baton):
xml = util.dict_get_path(baton, self.xml_path)
e = etree.XML(xml)
for xpath, output_path in self.xpaths.items():
result = e.xpath(xpath)
util.dict_setdefault_path(baton, output_path, []).extend(result)
return baton
def process(self, baton):
reporter = util.dict_get_path(baton, self.reporter_path)
yield reporter.wait_for_result_processing()
if self.done:
reporter.done()
defer.returnValue(baton)
def get_consumers(self, baton):
input = util.dict_get_path(baton, self.input_path)
index_or_indexes = self.lambda_(input)
if isinstance(index_or_indexes, int):
return [self.consumers[index_or_indexes]]
else:
return [self.consumers[index] for index in index_or_indexes]
def process(self, baton):
message = util.dict_get_path(baton, self.message_path)
for key, value in self.headers.items():
if key in message:
message.replace_header(key, value)
else:
message.add_header(key, value)
return baton
def process(self, baton):
if self.message_path:
message = util.dict_get_path(baton, self.message_path)
else:
message = self.message
if message:
self.logger(message)
return baton
def get(self, path_or_paths, fallback=None):
""" Return the first value piped in the configuration for the
key paths given in *path_or_paths*. If no paths result in a
value, *fallback* is returned.
.. seealso::
:func:`~piped.util.dict_get_path`
for more on the behaviour of getting with key paths.
"""
return util.dict_get_path(self._config, path_or_paths, fallback)
def process(self, baton):
to_request = util.dict_get_path(baton, self.to_request)
from_request = util.dict_get_path(baton, self.from_request)
# write the headers
for key, values in from_request.responseHeaders.getAllRawHeaders():
to_request.responseHeaders.setRawHeaders(key, values)
# copy the response code
to_request.setResponseCode(from_request.code, from_request.code_message)
# copy the written data if they're available
if hasattr(from_request, 'written'):
for data in from_request.written:
to_request.write(data)
if self.finish and from_request.finished:
to_request.finish()
return baton
def process_input(self, input, baton):
replacements = []
for dictionary in input:
value = util.dict_get_path(dictionary, self.key_path)
if value:
replacements.append(value)
if self.uniquify:
replacements = list(set(replacements))
if self.sort:
replacements.sort()
return replacements
def process_mapping(self, input, input_path, output_path, baton, **additional_kwargs):
output = input
if self.deep_copy:
output = copy.deepcopy(output)
elif self.copy:
output = copy.copy(output)
if self.extend:
output = util.dict_get_path(baton, output_path, list()) + [output]
return output
def process(self, baton):
response = util.dict_get_path(baton, self.response_path)
assert response is not None, "provide a response if you expect something sensible from this processor"
message = self._make_http_response(response)
self.dependencies.socket.send(message, flags=zmq.NOBLOCK)
if self.close:
response = dict(response) # Don't empty the body of the original response dict.
response['body'] = ''
close_message = self._make_close_response(response)
self.dependencies.socket.send(close_message)
return baton
def process_mapping(self, input, input_path, output_path, baton, only_first=True, load_json=False):
# we have to recheck if the input_path is in the request arguments, otherwise we don't know
# whether the input is a default provided by our configuration or an actual argument.
request = util.dict_get_path(baton, self.request_path)
if input_path not in request.args:
return input
if load_json:
for i, value in enumerate(input):
input[i] = json.loads(value)
if only_first:
return input[0]
return input
def process(self, baton):
reporter_arguments = dict()
for key, value in self.arguments.items():
if key.endswith('_path'):
key = key[:len('_path')+1]
value = util.dict_get_path(baton, value)
reporter_arguments[key] = value
reporter = self.reporter_class(self.pipeline_dependency, **reporter_arguments)
if self.output_path == '':
return reporter
util.dict_set_path(baton, self.output_path, reporter)
return baton
def process(self, baton):
for path in self.remove:
util.dict_remove_path(baton, path)
if self.keep:
keepers = dict()
for path in self.keep:
value = util.dict_get_path(baton, path, Ellipsis)
if value is Ellipsis:
continue
util.dict_set_path(keepers, path, value)
# Modify baton in-place
baton.clear()
baton.update(keepers)
return baton
def process(self, baton):
input = util.dict_get_path(baton, self.input_path, Ellipsis)
if input is Ellipsis:
if self.skip_if_nonexistent:
defer.returnValue(baton)
input = self.input_fallback
output = yield self.process_input(input=input, baton=baton)
if self.output_path == '':
defer.returnValue(output) # the output replaces the baton
if self.output_path is None:
defer.returnValue(baton) # discard the output
util.dict_set_path(baton, self.output_path, output)
defer.returnValue(baton)
def process_request(self, request, baton):
content = util.dict_get_path(baton, self.content_path, self.fallback_content)
if self.response_code:
request.setResponseCode(self.response_code)
if self.content_type:
request.setHeader('content-type', self.content_type)
if isinstance(content, unicode):
content = content.encode(self.encoding)
if content is not Ellipsis:
request.write(content)
if self.finish:
request.finish()
return baton