def process(self, baton): markov = self.context_dependency.get_resource() word_1, word_2 = None, None sentence = list() length = 0 # we must at least have room for a space in order to while length < 140: # pick a random word from our model: word = random.choice(markov[(word_1, word_2)]) sentence.append(word) length += len(word or '') # we used None to signify the end of a sentence: if word is None or length > 140: sentence.pop() # remove the None from the sentence break word_1, word_2 = word_2, word util.dict_set_path(baton, self.output_path, ' '.join(sentence)) return baton
def get_resulting_baton(self, baton, path, value): """ Returns the new baton after optionally setting an output value. Example usage:: def process(self, baton): value = self._calculate(baton) return self.get_resulting_baton(baton, self.output_path, value) :param baton: The baton in which the output value may be set. :param path: The path in the baton to set the value. Some paths have special meanings: None: The value should be discarded and the baton returned unchanged. An empty string: The value should completely replace the baton. :return: The new baton. """ if path is None: return baton elif path == '': return value util.dict_set_path(baton, path, value) return baton
def process(self, baton): groups = collections.defaultdict(list) items = util.dict_get_path(baton, self.input_path, list()) for item in items: value = util.dict_get_path(item, self.key_path, self.fallback) groups[value].append(item) util.dict_set_path(baton, self.output_path, groups) return baton
def set(self, path, value): """ Sets the configuration key specified by *path* to *value*. .. seealso:: :func:`~piped.util.dict_set_path` for more on the behaviour of setting with key paths. """ util.dict_set_path(self._config, path, value) return value
def process_input_using_pipeline(self, input, baton, pipeline): if self.trace_path: results, trace = yield pipeline.traced_process(input) util.dict_set_path(baton, self.trace_path, trace) else: results = yield pipeline.process(input) if hasattr(results, '__getitem__') and self.only_last_result: results = results[-1] defer.returnValue(results)
def process(self, baton): any = reflect.namedAny(self.name) result = any(*self.args, **self.kwargs) if self.output_path == '': return result util.dict_set_path(baton, self.output_path, result) return baton
def process(self, baton): pipeline_provider = yield self.pipeline_provider_dependency.wait_for_resource() subgraphs = [] for pipeline_name, pipeline in pipeline_provider.pipeline_by_name.items(): dot = pipeline.processor_graph.get_dot() dot = dot.replace('digraph G {', 'subgraph "cluster%s" { label="%s"; ' % (pipeline_name, pipeline_name)) subgraphs.append(dot) dot = u'digraph G {\n%s\n}' % '\n'.join(subgraphs) util.dict_set_path(baton, self.output_path, dot) defer.returnValue(baton)
def process(self, baton): root = yield self.root_dependency.wait_for_resource() args = util.dict_get_path(baton, self.args_path, list()) if self.args_path is not None else list() kwargs = util.dict_get_path(baton, self.kwargs_path, dict()) if self.kwargs_path is not None else dict() result = yield root.callRemote(self.remote_name, *args, **kwargs) if self.output_path == "": defer.returnValue(result) util.dict_set_path(baton, self.output_path, result) defer.returnValue(baton)
def test_setting_attribute(self): class Bar(object): def __init__(self, wrapped): self.wrapped = wrapped d = dict(a=Bar(dict(b=Bar('c')))) util.dict_set_path(d, 'a.wrapped.b.wrapped', 'not c') self.assertEquals(util.dict_get_path(d, 'a.wrapped.b.wrapped'), 'not c') util.dict_set_path(d, 'a.wrapped', 'not a bar') self.assertEquals(util.dict_get_path(d, 'a.wrapped'), 'not a bar') self.assertRaises(AttributeError, util.dict_set_path, d, 'a.no-such-attribute', 'whatever')
def process_request(self, request, baton): proxy_url = self.get_input(baton, self.url) remaining_path = '/'.join(urllib2.quote(segment, safe='') for segment in request.postpath) base_url_here = self._construct_base_url_here(request, proxy_url, remaining_path) proxied_url_parsed = urlparse.urlparse(proxy_url) proxied_path = self._create_proxied_request_path(proxied_url_parsed, remaining_path) proxied_query_string = self._create_query_string(request) rest = proxied_path + proxied_query_string # TODO: support for streaming large files etc. # create the proxied request object and add it to the baton proxied_request = web_provider.DummyRequest(request.postpath) util.dict_set_path(baton, self.proxied_request_path, proxied_request) # rewind the original requests content buffer before reading the data from it, since another processor # might have read its contents. request.content.seek(0, 0) data = request.content.read() headers = request.getAllHeaders() # we need to set the 'host' header, as our host can differ from the proxied host headers['host'] = proxied_url_parsed.netloc clientFactory = self.proxy_client_factory(request.method, rest, request.clientproto, headers, data, proxied_request) clientFactory.noisy = self.noisy reactor.connectTCP(proxied_url_parsed.hostname, proxied_url_parsed.port or 80, factory=clientFactory) # wait until the proxied request is finished: if not proxied_request.finished: yield proxied_request.notifyFinish() # We attempt to rewrite redirects coming from the proxied server in order to try to get the redirected request # to use this proxy too. if proxied_request.code in (301, 302, 303, 307): if self.rewrite_redirects: location = proxied_request.responseHeaders.getRawHeaders('location')[-1] proxied_request.responseHeaders.setRawHeaders('location', [location.replace(self.url, base_url_here)]) if self.stop_if_redirected: for key, values in proxied_request.responseHeaders.getAllRawHeaders(): request.responseHeaders.setRawHeaders(key, values) request.setResponseCode(proxied_request.code, proxied_request.code_message) defer.returnValue(Ellipsis) defer.returnValue(baton)
def process(self, baton): reporter_arguments = dict() for key, value in self.arguments.items(): if key.endswith('_path'): key = key[:len('_path')+1] value = util.dict_get_path(baton, value) reporter_arguments[key] = value reporter = self.reporter_class(self.pipeline_dependency, **reporter_arguments) if self.output_path == '': return reporter util.dict_set_path(baton, self.output_path, reporter) return baton
def process(self, baton): for path in self.remove: util.dict_remove_path(baton, path) if self.keep: keepers = dict() for path in self.keep: value = util.dict_get_path(baton, path, Ellipsis) if value is Ellipsis: continue util.dict_set_path(keepers, path, value) # Modify baton in-place baton.clear() baton.update(keepers) return baton
def process(self, baton): input = util.dict_get_path(baton, self.input_path, Ellipsis) if input is Ellipsis: if self.skip_if_nonexistent: defer.returnValue(baton) input = self.input_fallback output = yield self.process_input(input=input, baton=baton) if self.output_path == '': defer.returnValue(output) # the output replaces the baton if self.output_path is None: defer.returnValue(baton) # discard the output util.dict_set_path(baton, self.output_path, output) defer.returnValue(baton)
def process(self, baton): msg = message.Message() for key, value in self.headers.items(): msg.add_header(key, value) if self.payload_path: payload = util.dict_get_path(baton, self.payload_path) if hasattr(payload, 'read'): payload = payload.read() msg.set_payload(payload) if self.output_path == '': return msg util.dict_set_path(baton, self.output_path, msg) return baton
def _set_threshold(self, module_name, threshold): """ Sets a special loglevel for the given module. """ util.dict_set_path(self.modules, module_name, dict(__threshold__=threshold))
def process(self, baton): logger.info('Loading tables.') util.dict_set_path(baton, self.output_path, (yield self._load_every_jumprun_table())) defer.returnValue(baton)
def process(self, baton): input = util.dict_get_path(baton, self.input_path) util.dict_set_path(baton, self.output_path, self.schema.to_python(input)) return baton
def process(self, baton): context = self.context_dependency.get_resource() if self.output_path == '': return context util.dict_set_path(baton, self.output_path, context) return baton
def test_setting_simple_unnested(self): d = dict(foo='bar') util.dict_set_path(d, 'foo', 'baz') self.assertEquals(d, dict(foo='baz'))
def process(self, baton): util.dict_set_path(baton, self.path, self.value) return baton
def process(self, baton): util.dict_set_path(baton, self.output_path, (yield self._get_changes())) defer.returnValue(baton)
def set_output(self, baton, output, output_path, **kwargs): """ Set the output. """ util.dict_set_path(baton, output_path, output)
def test_setting_nonexistent(self): d = dict(a='b') util.dict_set_path(d, 'b.c', 'd') self.assertEquals(d, dict(a='b', b=dict(c='d')))
def test_setting_nested(self): d = dict(a=dict(b='c')) util.dict_set_path(d, 'a.b', 'd') self.assertEquals(d, dict(a=dict(b='d'))) util.dict_set_path(d, ('a', 'b'), 'e') self.assertEquals(d, dict(a=dict(b='e')))
def process(self, baton): for path in self.paths: value = util.dict_get_path(baton, path, None) if isinstance(value, list): util.dict_set_path(baton, path, util.flatten(value)) return baton
def process(self, baton): dot = self.dependency_manager.get_dot() util.dict_set_path(baton, self.output_path, dot) return baton
def process(self, baton): for path, value in self.mapping.items(): path = self.path_prefix + path util.dict_set_path(baton, path, value) return baton
def process(self, baton): # For every specified attribute path, check if there's # anything processable. If there is, process it and store the # result in the supplied attribute. ds = [] for mapped_attribute in self.mapping: input_path = self.input_path_prefix + mapped_attribute.get('input_path', '') input_baton = util.dict_get_path(baton, input_path, Ellipsis) if input_baton is Ellipsis: continue pipeline_name = mapped_attribute['pipeline'] output_path = self.output_path_prefix + mapped_attribute.get('output_path', pipeline_name) pipeline = self.get_pipeline(pipeline_name) input_baton = self.preprocess_baton(self._maybe_copy(input_baton)) d = defer.maybeDeferred(pipeline.process, input_baton) d.addCallback(lambda _, resulting_baton=input_baton, output_path=output_path: util.dict_set_path(baton, output_path, resulting_baton)) ds.append(d) results = yield defer.DeferredList(ds) for success, result in results: if not success: # It's a failure. Reraise it, so the resulting # stack-trace is correct. Note that if there are # failures in multiple pipelines, only the first one # is raised. result.raiseException() defer.returnValue(baton)
def process(self, baton): value = util.dict_get_path(baton, self.counter_path) util.dict_set_path(baton, self.counter_path, value + self.increment) return baton
def process_request(self, request, baton): ip = self._determine_ip(request) util.dict_set_path(baton, self.output_path, ip) return baton