def testNestedhandle(self): handle = lambda value : reduce(lambda x,y: x+y, value)\ if isinstance(value, list) else value d = MultiDeferred(handle_value=handle) d.append((a for a in range(1,11))) r = maybe_async(d.lock()) self.assertFalse(is_async(r)) self.assertEqual(r, [55]) handle = lambda value: 'c'*value d = MultiDeferred(handle_value=handle) d.append((a for a in range(1,11))) r = maybe_async(d.lock()) self.assertFalse(is_async(r)) self.assertTrue(is_failure(r[0]))
def do_stream(self, request): # stream the body body = self.body.render(request) # the body has asynchronous components # delay the header untl later if is_async(body): yield self._html(request, body) head = self.head.render(request) # # header not ready (this should never occur really) if is_async(head): yield self._html(request, body, head) else: yield self._template % (self.flatatt(), head, body)
def __iter__(self): request = self.request rpc = request['rpc'] status_code = 200 try: result = rpc.process(request) except Exception as e: result = as_failure(e) handler = rpc.handler result = maybe_async(result) while is_async(result): yield b'' result = maybe_async(result) try: if is_failure(result): e = result.trace[1] status_code = getattr(e, 'status', 400) log_failure(result) result = handler.dumps(rpc.id, rpc.version, error=e) else: result = handler.dumps(rpc.id, rpc.version, result=result) except Exception as e: LOGGER.error('Could not serialize', exc_info=True) status_code = 500 result = handler.dumps(rpc.id, rpc.version, error=e) content = to_bytes(result) response = WsgiResponse(status_code, content, content_type=handler.content_type) for c in self.start(response): yield c
def main(loop, syspath, params, stask): logger = LOGGER try: from pq import TaskApp sys.path[:] = json.loads(syspath) params = json.loads(params) params.update({'python_path': False, 'parse_console': False}) producer = TaskApp(**params).backend logger = producer.logger yield from producer.ready() task = producer._pubsub.load(stask) JobClass = producer.registry.get(task.name) if not JobClass: raise RuntimeError('%s not in registry' % task.name) job = JobClass(producer, task) result = yield from job.green_pool.submit(job, **task.kwargs) if is_async(result): result = yield from result sys.stdout.write({'cpubound_result': result}) except Exception: exc_info = sys.exc_info() error = str(exc_info[1]) stacktrace = traceback.format_tb(exc_info[2]) sys.stderr.write({'cpubound_failure': (error, stacktrace)}) msg = '%s\n%s' % (error, ''.join(stacktrace)) logger.error(msg)
def testNested(self): d = MultiDeferred() # add a generator d.append((a for a in range(1,11))) r = maybe_async(d.lock()) self.assertTrue(d.locked) self.assertFalse(is_async(r)) self.assertEqual(r, [[1,2,3,4,5,6,7,8,9,10]])
def __call__(self): try: r = self._call() except Exception as e: r = e if pulsar.is_async(r): return r.add_callback(self) else: return self.callback(r)
def stream_mapping(value, request): result = {} async = False for key, value in value.items(): if isinstance(value, String): value = value.render(request) if is_async(value): async = True result[key] = value return multi_async(result) if async else result
def __call__(self, *args, **kwargs): data, raw = self._get_data(*args, **kwargs) body = self._json.dumps(data).encode('latin-1') # Always make sure the content-type is application/json self.http.headers['content-type'] = 'application/json' resp = self.http.post(self.__url, data=body) if is_async(resp): return resp.add_callback(lambda r: self._end_call(r, raw)) else: return self._end_call(resp, raw)
def parse(self, mem_limit=None, **kw): mem_limit = mem_limit or DEFAULT_MAXSIZE if self.content_length > mem_limit: raise HttpException("Request to big. Increase MAXMEM.", status=LARGE_BODY_CODE) inp = self.environ.get("wsgi.input") or BytesIO() data = inp.read() if is_async(data): return async(self._async(data)) else: return self._ready(data)
def _html(self, request, body, head=None): '''Asynchronous rendering ''' if head is None: body = yield from body head = self.head.render(request) if is_async(head): head = yield from head return self._template % (self.flatatt(), head, body)
def rpc_server_info(self, request): '''Return a dictionary of information regarding the server and workers. It invokes the :meth:`extra_server_info` for adding custom information. ''' info = yield from pulsar.send('arbiter', 'info') info = self.extra_server_info(request, info) if is_async(info): info = yield from info return info
def testPingArbiter(self): worker = pulsar.get_actor() yield self.async.assertEqual(send('arbiter', 'ping'), 'pong') yield self.async.assertEqual(send('arbiter', 'ping'), 'pong') result = worker.send('arbiter', 'notify', worker.info()) if is_async(result): yield outcome result = outcome.result self.assertTrue(result>0) yield self.async.assertEqual(send('arbiter', 'ping'), 'pong') yield self.async.assertEqual(send('arbiter', 'echo', 'ciao'), 'ciao')
def __call__(self, client, keys=None, args=None): '''Execute the script, passing any required ``args`` ''' if self.sha not in client.store.loaded_scripts: sha = yield from client.immediate_execute('SCRIPT', 'LOAD', self.script) self.sha = sha.decode('utf-8') client.store.loaded_scripts.add(self.sha) result = client.evalsha(self.sha, keys, args) if is_async(result): result = yield from result return result
def _(*args, **kwargs): green = GreenletWorker(callable) # switch to the new greenlet result = green.switch(*args, **kwargs) # back to the parent while is_async(result): # keep on switching back to the greenlet if we get a Future try: result = green.switch((yield from result)) except Exception as exc: result = green.throw(exc) return green.switch(result)
def _green_task(self, greenlet, task): # Run in the main greenlet of the event-loop thread while task is not _DONE: # switch to the greenlet to start the task task = greenlet.switch(task) # if an asynchronous result is returned, yield from while is_async(task): try: task = yield from task except Exception as exc: # This call can return an asynchronous component task = greenlet.throw(exc)
def as_frame(self, connection, body): '''Build a websocket server frame from body.''' body = maybe_async(body) if is_async(body): return body.addBoth(lambda b: self.as_frame(connection, b)) if is_failure(body): # We have a failure. shut down connection body.log() body = Frame.close('Server error') elif not isinstance(body, Frame): # If the body is not a frame, build a final frame from it. body = Frame(body or '', version=connection.protocol.version, final=True) return body.msg
def wait_for_body_middleware(environ, start_response=None): '''Use this middleware to wait for the full body. This middleware wait for the full body to be received before letting other middleware to be processed. Useful when using synchronous web-frameworks such as :django:`django <>`. ''' if environ['wsgi.input']: stream = environ['wsgi.input'] chunk = stream.read() if is_async(chunk): chunk = yield from chunk environ['wsgi.input'] = BytesIO(chunk)
def testDeferredCallback(self): d = Deferred() d.add_callback(lambda r : Cbk(r)) self.assertFalse(d.called) result = d.callback('ciao') self.assertTrue(d.called) self.assertEqual(d.paused,1) self.assertTrue(is_async(result)) self.assertEqual(len(result._callbacks),1) self.assertFalse(result.called) result.set_result('luca') self.assertTrue(result.called) self.assertEqual(result.result,('ciao','luca')) self.assertEqual(d.paused,0)
def _green_task(self, green, task): # Coroutine executing the in main greenlet # This coroutine is executed for every task put into the queue while task is not _DONE: # switch to the greenlet to start the task task = green.switch(task) # if an asynchronous result is returned, yield from while is_async(task): try: task = yield from task except Exception as exc: # This call can return an asynchronous component task = green.throw(exc)
def response_generator(self, response, wsgi_response): response = maybe_async(response) while is_async(response): yield b'' response = maybe_async(response) stream_content = None if is_failure(response): wsgi_response.status_code = 500 else: wsgi_response.status_code = response.status_code wsgi_response.headers.update(response.headers) stream_content = response.stream() wsgi_response.start() if stream_content: for content in stream_content: yield content
def _consume(self, job, kwargs): concurrency = job.get_concurrency() if concurrency == models.ASYNC_IO: result = job(**kwargs) assert is_async(result), "ASYNC_IO tasks not asynchronous" return result elif concurrency == models.GREEN_IO: return self.green_pool.submit(job, **kwargs) elif concurrency == models.THREAD_IO: return job._loop.run_in_executor(None, lambda: job(**kwargs)) elif concurrency == models.CPUBOUND: return self._consume_in_subprocess(job, kwargs) else: raise ImproperlyConfigured('invalid concurrency "%d"' % concurrency)
def render(self, request=None, callback=None): '''Render this string. This method returns a string or a :class:`~asyncio.Future` which results in a string. On the other hand, the callable method of a :class:`.String` **always** returns a :class:`~asyncio.Future`. ''' stream = [] async = False for data in self.stream(request): if is_async(data): async = True stream.append(data) if not callback: callback = self.to_string if async: return chain_future(multi_async(stream), callback=callback) else: return callback(stream)
def wsgi_iterator(gen, encoding=None): encoding = encoding or "utf-8" for data in gen: data = maybe_async(data) while is_async(data): yield b"" data = maybe_async(data) if data is NOT_DONE: yield b"" elif data is None: continue elif is_failure(data): log_failure(data) else: if isinstance(data, bytes): yield data elif isinstance(data, string_type): yield data.encode(encoding) else: for b in wsgi_iterator(data, encoding): yield b
def start(self, worker): '''Called by the :class:`pulsar.Worker` *worker* when the task start its execution. If no timeout has occured the task will switch to a ``STARTED`` :attr:`Task.status` and invoke the :meth:`on_start` callback.''' job = registry[self.name] result = None try: if self.maybe_revoked(): yield self.on_timeout(worker) else: self.status = STARTED self.time_start = datetime.now() yield self.on_start(worker) consumer = TaskConsumer(self, worker, job) result = maybe_async(job(consumer, *self.args, **self.kwargs)) if is_async(result): yield result result = maybe_async(result) except Exception as e: result = as_failure(e) finally: yield self.finish(worker, result)
def ascoro(result=None): if is_async(result): result = yield from result return result
def _write_streamed_data(self, transport): for data in self.data: if is_async(data): data = yield from data self._write_body_data(transport, data) self._write_body_data(transport, b'', True)
def _(*args, **kwargs): args = yield from multi_async(args) result = getattr(self.test, name)(*args, **kwargs) if is_async(result): result = yield from result return result