def process(self, baton):
markov = self.context_dependency.get_resource()
word_1, word_2 = None, None
sentence = list()
length = 0
# we must at least have room for a space in order to
while length < 140:
# pick a random word from our model:
word = random.choice(markov[(word_1, word_2)])
sentence.append(word)
length += len(word or '')
# we used None to signify the end of a sentence:
if word is None or length > 140:
sentence.pop() # remove the None from the sentence
break
word_1, word_2 = word_2, word
util.dict_set_path(baton, self.output_path, ' '.join(sentence))
return baton
def get_resulting_baton(self, baton, path, value):
""" Returns the new baton after optionally setting an output value.
Example usage::
def process(self, baton):
value = self._calculate(baton)
return self.get_resulting_baton(baton, self.output_path, value)
:param baton: The baton in which the output value may be set.
:param path: The path in the baton to set the value. Some paths have special meanings:
None:
The value should be discarded and the baton returned unchanged.
An empty string:
The value should completely replace the baton.
:return: The new baton.
"""
if path is None:
return baton
elif path == '':
return value
util.dict_set_path(baton, path, value)
return baton
def process(self, baton):
groups = collections.defaultdict(list)
items = util.dict_get_path(baton, self.input_path, list())
for item in items:
value = util.dict_get_path(item, self.key_path, self.fallback)
groups[value].append(item)
util.dict_set_path(baton, self.output_path, groups)
return baton
def set(self, path, value):
""" Sets the configuration key specified by *path* to *value*.
.. seealso::
:func:`~piped.util.dict_set_path`
for more on the behaviour of setting with key paths.
"""
util.dict_set_path(self._config, path, value)
return value
def process_input_using_pipeline(self, input, baton, pipeline):
if self.trace_path:
results, trace = yield pipeline.traced_process(input)
util.dict_set_path(baton, self.trace_path, trace)
else:
results = yield pipeline.process(input)
if hasattr(results, '__getitem__') and self.only_last_result:
results = results[-1]
defer.returnValue(results)
def process(self, baton):
any = reflect.namedAny(self.name)
result = any(*self.args, **self.kwargs)
if self.output_path == '':
return result
util.dict_set_path(baton, self.output_path, result)
return baton
def process(self, baton):
pipeline_provider = yield self.pipeline_provider_dependency.wait_for_resource()
subgraphs = []
for pipeline_name, pipeline in pipeline_provider.pipeline_by_name.items():
dot = pipeline.processor_graph.get_dot()
dot = dot.replace('digraph G {', 'subgraph "cluster%s" { label="%s"; ' % (pipeline_name, pipeline_name))
subgraphs.append(dot)
dot = u'digraph G {\n%s\n}' % '\n'.join(subgraphs)
util.dict_set_path(baton, self.output_path, dot)
defer.returnValue(baton)
def process(self, baton):
root = yield self.root_dependency.wait_for_resource()
args = util.dict_get_path(baton, self.args_path, list()) if self.args_path is not None else list()
kwargs = util.dict_get_path(baton, self.kwargs_path, dict()) if self.kwargs_path is not None else dict()
result = yield root.callRemote(self.remote_name, *args, **kwargs)
if self.output_path == "":
defer.returnValue(result)
util.dict_set_path(baton, self.output_path, result)
defer.returnValue(baton)
def test_setting_attribute(self):
class Bar(object):
def __init__(self, wrapped):
self.wrapped = wrapped
d = dict(a=Bar(dict(b=Bar('c'))))
util.dict_set_path(d, 'a.wrapped.b.wrapped', 'not c')
self.assertEquals(util.dict_get_path(d, 'a.wrapped.b.wrapped'), 'not c')
util.dict_set_path(d, 'a.wrapped', 'not a bar')
self.assertEquals(util.dict_get_path(d, 'a.wrapped'), 'not a bar')
self.assertRaises(AttributeError, util.dict_set_path, d, 'a.no-such-attribute', 'whatever')
def process_request(self, request, baton):
proxy_url = self.get_input(baton, self.url)
remaining_path = '/'.join(urllib2.quote(segment, safe='') for segment in request.postpath)
base_url_here = self._construct_base_url_here(request, proxy_url, remaining_path)
proxied_url_parsed = urlparse.urlparse(proxy_url)
proxied_path = self._create_proxied_request_path(proxied_url_parsed, remaining_path)
proxied_query_string = self._create_query_string(request)
rest = proxied_path + proxied_query_string
# TODO: support for streaming large files etc.
# create the proxied request object and add it to the baton
proxied_request = web_provider.DummyRequest(request.postpath)
util.dict_set_path(baton, self.proxied_request_path, proxied_request)
# rewind the original requests content buffer before reading the data from it, since another processor
# might have read its contents.
request.content.seek(0, 0)
data = request.content.read()
headers = request.getAllHeaders()
# we need to set the 'host' header, as our host can differ from the proxied host
headers['host'] = proxied_url_parsed.netloc
clientFactory = self.proxy_client_factory(request.method, rest, request.clientproto, headers, data, proxied_request)
clientFactory.noisy = self.noisy
reactor.connectTCP(proxied_url_parsed.hostname, proxied_url_parsed.port or 80, factory=clientFactory)
# wait until the proxied request is finished:
if not proxied_request.finished:
yield proxied_request.notifyFinish()
# We attempt to rewrite redirects coming from the proxied server in order to try to get the redirected request
# to use this proxy too.
if proxied_request.code in (301, 302, 303, 307):
if self.rewrite_redirects:
location = proxied_request.responseHeaders.getRawHeaders('location')[-1]
proxied_request.responseHeaders.setRawHeaders('location', [location.replace(self.url, base_url_here)])
if self.stop_if_redirected:
for key, values in proxied_request.responseHeaders.getAllRawHeaders():
request.responseHeaders.setRawHeaders(key, values)
request.setResponseCode(proxied_request.code, proxied_request.code_message)
defer.returnValue(Ellipsis)
defer.returnValue(baton)
def process(self, baton):
reporter_arguments = dict()
for key, value in self.arguments.items():
if key.endswith('_path'):
key = key[:len('_path')+1]
value = util.dict_get_path(baton, value)
reporter_arguments[key] = value
reporter = self.reporter_class(self.pipeline_dependency, **reporter_arguments)
if self.output_path == '':
return reporter
util.dict_set_path(baton, self.output_path, reporter)
return baton
def process(self, baton):
for path in self.remove:
util.dict_remove_path(baton, path)
if self.keep:
keepers = dict()
for path in self.keep:
value = util.dict_get_path(baton, path, Ellipsis)
if value is Ellipsis:
continue
util.dict_set_path(keepers, path, value)
# Modify baton in-place
baton.clear()
baton.update(keepers)
return baton
def process(self, baton):
input = util.dict_get_path(baton, self.input_path, Ellipsis)
if input is Ellipsis:
if self.skip_if_nonexistent:
defer.returnValue(baton)
input = self.input_fallback
output = yield self.process_input(input=input, baton=baton)
if self.output_path == '':
defer.returnValue(output) # the output replaces the baton
if self.output_path is None:
defer.returnValue(baton) # discard the output
util.dict_set_path(baton, self.output_path, output)
defer.returnValue(baton)
def process(self, baton):
msg = message.Message()
for key, value in self.headers.items():
msg.add_header(key, value)
if self.payload_path:
payload = util.dict_get_path(baton, self.payload_path)
if hasattr(payload, 'read'):
payload = payload.read()
msg.set_payload(payload)
if self.output_path == '':
return msg
util.dict_set_path(baton, self.output_path, msg)
return baton
def _set_threshold(self, module_name, threshold):
""" Sets a special loglevel for the given module. """
util.dict_set_path(self.modules, module_name, dict(__threshold__=threshold))
def process(self, baton):
logger.info('Loading tables.')
util.dict_set_path(baton, self.output_path, (yield self._load_every_jumprun_table()))
defer.returnValue(baton)
def process(self, baton):
input = util.dict_get_path(baton, self.input_path)
util.dict_set_path(baton, self.output_path, self.schema.to_python(input))
return baton
def process(self, baton):
context = self.context_dependency.get_resource()
if self.output_path == '':
return context
util.dict_set_path(baton, self.output_path, context)
return baton
def test_setting_simple_unnested(self):
d = dict(foo='bar')
util.dict_set_path(d, 'foo', 'baz')
self.assertEquals(d, dict(foo='baz'))
def process(self, baton):
util.dict_set_path(baton, self.path, self.value)
return baton
def process(self, baton):
util.dict_set_path(baton, self.output_path, (yield self._get_changes()))
defer.returnValue(baton)
def set_output(self, baton, output, output_path, **kwargs):
""" Set the output. """
util.dict_set_path(baton, output_path, output)
def test_setting_nonexistent(self):
d = dict(a='b')
util.dict_set_path(d, 'b.c', 'd')
self.assertEquals(d, dict(a='b', b=dict(c='d')))
def test_setting_nested(self):
d = dict(a=dict(b='c'))
util.dict_set_path(d, 'a.b', 'd')
self.assertEquals(d, dict(a=dict(b='d')))
util.dict_set_path(d, ('a', 'b'), 'e')
self.assertEquals(d, dict(a=dict(b='e')))
def process(self, baton):
for path in self.paths:
value = util.dict_get_path(baton, path, None)
if isinstance(value, list):
util.dict_set_path(baton, path, util.flatten(value))
return baton
def process(self, baton):
dot = self.dependency_manager.get_dot()
util.dict_set_path(baton, self.output_path, dot)
return baton
def process(self, baton):
for path, value in self.mapping.items():
path = self.path_prefix + path
util.dict_set_path(baton, path, value)
return baton
def process(self, baton):
# For every specified attribute path, check if there's
# anything processable. If there is, process it and store the
# result in the supplied attribute.
ds = []
for mapped_attribute in self.mapping:
input_path = self.input_path_prefix + mapped_attribute.get('input_path', '')
input_baton = util.dict_get_path(baton, input_path, Ellipsis)
if input_baton is Ellipsis:
continue
pipeline_name = mapped_attribute['pipeline']
output_path = self.output_path_prefix + mapped_attribute.get('output_path', pipeline_name)
pipeline = self.get_pipeline(pipeline_name)
input_baton = self.preprocess_baton(self._maybe_copy(input_baton))
d = defer.maybeDeferred(pipeline.process, input_baton)
d.addCallback(lambda _, resulting_baton=input_baton, output_path=output_path: util.dict_set_path(baton, output_path, resulting_baton))
ds.append(d)
results = yield defer.DeferredList(ds)
for success, result in results:
if not success:
# It's a failure. Reraise it, so the resulting
# stack-trace is correct. Note that if there are
# failures in multiple pipelines, only the first one
# is raised.
result.raiseException()
defer.returnValue(baton)
def process(self, baton):
value = util.dict_get_path(baton, self.counter_path)
util.dict_set_path(baton, self.counter_path, value + self.increment)
return baton
def process_request(self, request, baton):
ip = self._determine_ip(request)
util.dict_set_path(baton, self.output_path, ip)
return baton