def decode(self, pieces): ASSERT.false(self.eof) output = [] for data in pieces: if data: self._decode(data, output) return output
def poll(self, timeout): ASSERT.false(self._epoll.closed) with self._lock: if self._closed_fds: closed_fds, self._closed_fds = self._closed_fds, set() return closed_fds, closed_fds if timeout is None: pass elif timeout <= 0: timeout = 0 else: # epoll_wait() has a resolution of 1 millisecond. timeout = math.ceil(timeout * 1e3) * 1e-3 can_read = [] can_write = [] # Since Python 3.5, poll retries with a re-computed timeout # rather than raising InterruptedError (see PEP 475). for fd, events in self._epoll.poll(timeout=timeout): if events & self._EVENT_IN: can_read.append(fd) if events & self._EVENT_OUT: can_write.append(fd) return can_read, can_write
def _decode(self, data, output): def move(n): """Move ``n`` bytes from ``data`` to ``output``.""" nonlocal data ASSERT.greater_or_equal(self._chunk_remaining, n) output.append(data[:n]) data = data[n:] self._chunk_remaining -= n def expect(pattern): """Drop ``pattern`` prefix from ``data``.""" nonlocal data n = min(len(pattern), len(data)) ASSERT.equal(pattern[:n], data[:n]) data = data[n:] return n while data: if self._chunk_remaining > 0: move(min(self._chunk_remaining, len(data))) continue if self._chunk_remaining == 0: self._chunk_remaining -= expect(b'\r\n') continue if self._chunk_remaining == -1: self._chunk_remaining -= expect(b'\n') continue match = self._CRLF_PATTERN.search(data) if not match: self._append(data) match = self._CRLF_PATTERN.search(self._buffer[:self._pos]) if not match: break data = self._reset() chunk_size = data[:match.start()] if self._pos > 0: self._append(chunk_size) chunk_size = self._reset() # TODO: Handle parameters (stuff after ';'). chunk_size = int( bytes(chunk_size).split(b';', maxsplit=1)[0], base=16, ) if chunk_size == 0: # TODO: Handle trailers. self.eof = True else: ASSERT.false(self.eof) data = data[match.end():] self._chunk_remaining = chunk_size if self.eof: ASSERT.empty(data)
def notify_close(self, fd): ASSERT.false(self._epoll.closed) with self._lock: self._closed_fds.add(fd) try: self._epoll.unregister(fd) except OSError as exc: if exc.errno != errno.EBADF: raise
def _parse_version(ps): ASSERT.false(ps['//bases:build-xar-image']) return Version( *map( int, ASSERT.not_none(_VERSION_PATTERN.search(ps['archive'].url))\ .groups(), ) )
def cancel(self, task): """Cancel the task. This is a no-op is task has been completed. """ ASSERT.false(self._closed) self._assert_owner() ASSERT.is_(task._kernel, self) if not task.is_completed(): self._disrupt(task, errors.TaskCancellation)
def timeout_after(self, task, duration): ASSERT.false(self._closed) self._assert_owner() ASSERT.is_(task._kernel, self) if duration is None: return lambda: None # Even if duration <= 0, the kernel should raise ``Timeout`` at # the next blocking trap for consistency (so, don't raise here). self._timeout_after_blocker.block(time.monotonic() + duration, task) return functools.partial(self._timeout_after_blocker.cancel, task)
def spawn(self, awaitable): """Spawn a new task onto the kernel.""" ASSERT.false(self._closed) self._assert_owner() if tasks.Task.is_coroutine(awaitable): coroutine = awaitable elif inspect.isawaitable(awaitable): coroutine = awaitable.__await__() else: coroutine = awaitable() task = tasks.Task(self, coroutine) self._ready_tasks.append(TaskReady(task, None, None)) self._num_tasks += 1 return task
async def begin(self, status, headers): ASSERT.false(self._has_begun) self._has_begun = True buffer = io.BytesIO() buffer.write( b'HTTP/1.1 %d %s\r\n' % (status, self._ENCODED_REASONS[status]) ) for key, value in headers: buffer.write(b'%s: %s\r\n' % (key, value)) buffer.write(b'\r\n') await self._send_all(buffer.getvalue()) self._headers_sent.set()
def err_after_commit(self, exc): """Record exception raised after commit. This first closes the response, dropping remaining body data, and then calls start_response with HTTP 5xx and exc_info. If the WSGI server has not yet started sending response, it resets the response to HTTP 500; otherwise it re-raises the exception. """ ASSERT.false(self.is_uncommitted()) self._body.close(graceful=False) self._start_response( self._format_status(consts.Statuses.INTERNAL_SERVER_ERROR), [], (exc.__class__, exc, exc.__traceback__), )
def build(parameters): src_path = _find_project(parameters, foreman.get_relpath()) root_path = _find_root_project(src_path) ASSERT.false(src_path.samefile(root_path)) output_path = src_path / ('build/libs/%s-all.jar' % src_path.name) task = ':'.join(src_path.relative_to(root_path).parts) task = ':%s:shadowJar' % task target_dir_path = parameters[root_project + ':packages'] if (target_dir_path / output_path.name).exists(): LOG.info('skip: run task %s', task) return LOG.info('run task %s', task) with scripts.using_cwd(root_path): scripts.run(['./gradlew', task]) with scripts.using_sudo(): scripts.mkdir(target_dir_path) scripts.cp(output_path, target_dir_path)
def __init__(self, data=b'', *, msg_p=None): # In case ``__init__`` raises. self._msg_p = None ASSERT.isinstance(data, bytes) if msg_p is None: msg_p = _nng.nng_msg_p() errors.check(_nng.F.nng_msg_alloc(ctypes.byref(msg_p), len(data))) if data: ctypes.memmove(_nng.F.nng_msg_body(msg_p), data, len(data)) else: # We are taking ownership of ``msg_p`` and should not take # any initial data. ASSERT.false(data) self._msg_p = msg_p self.header = Header(self._get) self.body = Body(self._get)
def tick(self, trap_result, trap_exception): """Run coroutine through the next trap point. NOTE: ``tick`` catches ``BaseException`` raised from the coroutine. As a result, ``SystemExit`` does not bubble up to the kernel event loop. I believe this behavior is similar to Python threading library and thus more expected (``SystemExit`` raised in non- main thread does not cause CPython process to exit). If you want raising ``SystemExit`` in a task to be effective, you have to call ``Task.get_result_nonblocking`` in the main thread (or implicitly through ``Kernel.run``). """ ASSERT.false(self._completed) if trap_exception: trap = self._tick(self._coroutine.throw, trap_exception) else: trap = self._tick(self._coroutine.send, trap_result) if trap is not None: return trap ASSERT.true(self._completed) self._call_callbacks() return None
def start_response(self, status, response_headers, exc_info=None): if exc_info: try: if self._is_committed: exc = exc_info[1] if exc is None: exc = exc_info[0]() if exc.__traceback__ is not exc_info[2]: exc.with_traceback(exc_info[2]) raise exc finally: exc_info = None # Avoid dangling cyclic ref. else: ASSERT.false(self._is_committed) # Get the status code from status line like "200 OK". self._status = http.HTTPStatus(int(status.split(maxsplit=1)[0])) self._headers = [ (name.encode('iso-8859-1'), value.encode('iso-8859-1')) for name, value in response_headers ] return self.write
def _run_one_ready_task(self): task, trap_result, trap_exception = self._ready_tasks.popleft() override = self._to_raise.pop(task, None) if override is not None: trap_result = None trap_exception = override self._current_task = task try: trap = task.tick(trap_result, trap_exception) finally: self._current_task = None if trap is None: ASSERT.true(task.is_completed()) self._trap_return(self._task_completion_blocker, task) # Clear disrupter. self._to_raise.pop(task, None) self._timeout_after_blocker.cancel(task) self._num_tasks -= 1 return task ASSERT.false(task.is_completed()) override = self._to_raise.pop(task, None) if override: self._ready_tasks.append(TaskReady(task, None, override)) else: handler = self._blocking_trap_handlers[trap.kind] try: handler(task, trap) except Exception as exc: self._ready_tasks.append(TaskReady(task, None, exc)) return None
def notify_open(self, fd): ASSERT.false(self._epoll.closed) try: self._epoll.register(fd, self._EVENT_MASK) except FileExistsError: pass
parser = _get_html_parser(encoding or ASSERT.not_none(self.encoding)) # Check whether fromstring returns None because apparently # HTMLParser is more lenient than XMLParser and may cause # fromstring to return None on some malformed HTML input. return ASSERT.not_none(lxml.etree.fromstring(string, parser)) def xml(self): """Parse response as an XML document.""" return lxml.etree.fromstring(self.content, _XML_PARSER) @functools.lru_cache(maxsize=8) def _get_html_parser(encoding): return lxml.etree.HTMLParser(encoding=encoding) _XML_PARSER = lxml.etree.XMLParser() # # Monkey-patch ``requests.Response``. # # Just to make sure we do not accidentally override them. ASSERT.false(hasattr(requests.Response, 'recvfile')) requests.Response.recvfile = recvfiles.recvfile ASSERT.false(hasattr(requests.Response, 'html')) requests.Response.html = Response.html ASSERT.false(hasattr(requests.Response, 'xml')) requests.Response.xml = Response.xml
def block(self, source, task): """Record that ``task`` is joining on ``source`` task.""" ASSERT.isinstance(source, tasks.Task) ASSERT.is_not(source, task) # A task can't join on itself. ASSERT.false(source.is_completed()) return super().block(source, task)
def write_nonblocking(self, data): ASSERT.false(self._closed) self._gate.unblock() return self._buffer.write(data)
def assert_keyless(self): ASSERT.false(self.is_keyed(), message='expect keyless schema')
def set(self, value): ASSERT.false(self._have_been_read) self._value = self._validate(value)
def run(self, awaitable=None, timeout=None): """Run spawned tasks through completion. If ``awaitable`` is not ``None``, a task is spawned for it, and when the task completes, ``run`` returns its result. If ``timeout`` is non-positive, ``run`` is guarantee to iterate exactly once. """ ASSERT.false(self._closed) self._assert_owner() ASSERT.none(self._current_task) # Disallow recursive calls. main_task = self.spawn(awaitable) if awaitable else None run_timer = timers.make(timeout) while self._num_tasks > 0: # Do sanity check every ``_sanity_check_frequency`` ticks. if self._num_ticks % self._sanity_check_frequency == 0: self._sanity_check() self._num_ticks += 1 # Fire callbacks posted by other threads. with self._callbacks_lock: callbacks, self._callbacks = \ self._callbacks, collections.deque() for callback in callbacks: callback() del callbacks # Run all ready tasks. with self._managing_async_generators(): while self._ready_tasks: completed_task = self._run_one_ready_task() if completed_task and completed_task is main_task: # Return the result eagerly. If you want to run # all remaining tasks through completion, just # call ``run`` again with no arguments. return completed_task.get_result_nonblocking() if self._num_tasks > 0: # Poll I/O. now = time.monotonic() poll_timeout = min( run_timer.get_timeout(), self._sleep_blocker.get_min_timeout(now), self._timeout_after_blocker.get_min_timeout(now), key=timers.timeout_to_key, ) can_read, can_write = self._poller.poll(poll_timeout) for fd in can_read: if self._nudger.is_nudged(fd): self._nudger.ack() else: self._trap_return(self._read_blocker, fd) for fd in can_write: self._trap_return(self._write_blocker, fd) # Handle any task timeout. now = time.monotonic() self._trap_return(self._sleep_blocker, now) self._timeout_after_on_completion(now) # Break if ``run`` times out. if run_timer.is_expired(): raise errors.KernelTimeout
def notify_open(self, fd): ASSERT.false(self._closed) self._assert_owner() self._poller.notify_open(fd)
def image_path(self): ASSERT.false(self.deploy_instruction.is_zipapp()) return self.path / models.XAR_BUNDLE_IMAGE_FILENAME
def unblock(self, source): """Unblock tasks blocked by ``source``.""" ASSERT.false(self._closed) self._assert_owner() self._trap_return(self._generic_blocker, source)
async def recvfile(response, file): """Receive response body into a file. The caller must set ``stream`` to true when make the request. DANGER! This breaks the multiple levels of encapsulation, from requests.Response all the way down to http.client.HTTPResponse. As a result, the response object is most likely unusable after a recvfile call, and you should probably close it immediately. """ # requests sets _content to False initially. ASSERT.is_(response._content, False) ASSERT.false(response._content_consumed) urllib3_response = ASSERT.not_none(response.raw) chunked = urllib3_response.chunked httplib_response = ASSERT.isinstance( urllib3_response._fp, http.client.HTTPResponse ) ASSERT.false(httplib_response.closed) sock = ASSERT.isinstance(httplib_response.fp.raw._sock, socket.socket) output = DecoderChain(file) if chunked: chunk_decoder = ChunkDecoder() output.add(chunk_decoder) num_to_read = 0 eof = lambda: chunk_decoder.eof else: num_to_read = ASSERT.greater( ASSERT.not_none(httplib_response.length), 0 ) eof = lambda: num_to_read <= 0 # Use urllib3's decoder code. urllib3_response._init_decoder() if urllib3_response._decoder is not None: output.add(ContentDecoder(urllib3_response._decoder)) with contextlib.ExitStack() as stack: src = adapters.FileAdapter(httplib_response.fp) stack.callback(src.disown) sock.setblocking(False) stack.callback(sock.setblocking, True) buffer = memoryview(stack.enter_context(_BUFFER_POOL.using())) while not eof(): if chunked: # TODO: If server sends more data at the end, like # response of the next request, for now recvfile might # read them, and then err out. Maybe recvfile should # check this, and not read more than it should instead? num_read = await src.readinto1(buffer) else: num_read = await src.readinto1( buffer[:min(num_to_read, _CHUNK_SIZE)] ) if num_read == 0: break output.write(buffer[:num_read]) num_to_read -= num_read output.flush() # Sanity check. if not chunked: ASSERT.equal(num_to_read, 0) # Trick requests to release the connection back to the connection # pool, rather than closing/discarding it. response._content_consumed = True # http.client.HTTPConnection tracks the last response; so you have # to close it to make the connection object useable again. httplib_response.close() # Close the response for the caller since response is not useable # after recvfile. response.close() loggings.ONCE_PER( 1000, LOG.info, 'buffer pool stats: %r', _BUFFER_POOL.get_stats() )
def popen(args): LOG.debug('popen: args=%s, context=%s', args, _CONTEXT) # It does not seem like we can return a fake Popen object. ASSERT.false(_get(_DRY_RUN)) return subprocess.Popen(_prepare_args(args), **_prepare_kwargs())