def call_process(self, cmd, stream, address, io_loop=None): """ Calls process cmd: command in a list e.g ['ls', '-la'] stdout_callback: callback to run on stdout TODO: add some way of calling proc.kill() if the stream is closed """ stdout_stream = Subprocess.STREAM stderr_stream = Subprocess.STREAM proc = Subprocess(cmd, stdout=stdout_stream, stderr=stderr_stream) call_back = partial(self.on_exit, address) proc.set_exit_callback(call_back) pipe_stream = PipeIOStream(proc.stdout.fileno()) try: while True: str_ = yield pipe_stream.read_bytes(102400, partial=True) yield stream.write(str_) except StreamClosedError: pass print("end address: {}".format(address))
def test_pipe_iostream_big_write(self): r, w = os.pipe() rs = PipeIOStream(r, io_loop=self.io_loop) ws = PipeIOStream(w, io_loop=self.io_loop) NUM_BYTES = 1048576 # Write 1MB of data, which should fill the buffer ws.write(b"1" * NUM_BYTES) rs.read_bytes(NUM_BYTES, self.stop) data = self.wait() self.assertEqual(data, b"1" * NUM_BYTES) ws.close() rs.close()
def test_pipe_iostream(self): r, w = os.pipe() rs = PipeIOStream(r, io_loop=self.io_loop) ws = PipeIOStream(w, io_loop=self.io_loop) ws.write(b"hel") ws.write(b"lo world") rs.read_until(b' ', callback=self.stop) data = self.wait() self.assertEqual(data, b"hello ") rs.read_bytes(3, self.stop) data = self.wait() self.assertEqual(data, b"wor") ws.close() rs.read_until_close(self.stop) data = self.wait() self.assertEqual(data, b"ld") rs.close()
def test_pipe_iostream(self): r, w = os.pipe() rs = PipeIOStream(r, io_loop=self.io_loop) ws = PipeIOStream(w, io_loop=self.io_loop) ws.write(b"hel") ws.write(b"lo world") rs.read_until(b" ", callback=self.stop) data = self.wait() self.assertEqual(data, b"hello ") rs.read_bytes(3, self.stop) data = self.wait() self.assertEqual(data, b"wor") ws.close() rs.read_until_close(self.stop) data = self.wait() self.assertEqual(data, b"ld") rs.close()
def __init__( self, command, timeout=-1, stdout_chunk_callback=None, stderr_chunk_callback=None, exit_process_callback=None, stdin_bytes=None, io_loop=None, kill_on_timeout=False ): """ Initializes the subprocess with callbacks and timeout. :param command: command like ['java', '-jar', 'test.jar'] :param timeout: timeout for subprocess to complete, if negative or \ zero then no timeout :param stdout_chunk_callback: callback(bytes_data_chuck_from_stdout) :param stderr_chunk_callback: callback(bytes_data_chuck_from_stderr) :param exit_process_callback: callback(exit_code, \ was_expired_by_timeout) :param stdin_bytes: bytes data to send to stdin :param io_loop: tornado io loop on None for current :param kill_on_timeout: kill(-9) or terminate(-15)? """ self.aa_exit_process_callback = exit_process_callback self.aa_kill_on_timeout = kill_on_timeout stdin = Subprocess.STREAM if stdin_bytes else None stdout = Subprocess.STREAM if stdout_chunk_callback else None stderr = Subprocess.STREAM if stderr_chunk_callback else None Subprocess.__init__(self, command, stdin=stdin, stdout=stdout, stderr=stderr, io_loop=io_loop) self.aa_process_expired = False self.aa_terminate_timeout = self.io_loop.call_later( timeout, self.aa_timeout_callback) if timeout > 0 else None self.set_exit_callback(self.aa_exit_callback) if stdin: self.stdin.write(stdin_bytes) self.stdin.close() if stdout: output_stream = PipeIOStream(self.stdout.fileno()) def on_stdout_chunk(data): stdout_chunk_callback(data) if not output_stream.closed(): output_stream.read_bytes(102400, on_stdout_chunk, None, True) output_stream.read_bytes(102400, on_stdout_chunk, None, True) if stderr: stderr_stream = PipeIOStream(self.stderr.fileno()) def on_stderr_chunk(data): stderr_chunk_callback(data) if not stderr_stream.closed(): stderr_stream.read_bytes(102400, on_stderr_chunk, None, True) stderr_stream.read_bytes(102400, on_stderr_chunk, None, True)
class PipeStream(Stream): def __init__(self, rpipe, wpipe=None, auto_close=False): """Pipe-based stream NOTE: reading from or writing to files, use os.open to get the file descriptor instead of python's open. Socket file descriptors and others are fine. when you use os.pipe to generate one write pipe and one read pipe, you need to pass both of them into init method. :param rpipe: an integer file descriptor which supports read ops :param wpipe: an integer file descriptor which supports write ops :param auto: flag to indicate to close the stream automatically or not """ assert rpipe is not None self._rpipe = rpipe self._wpipe = wpipe self._rs = PipeIOStream(self._rpipe) if self._rpipe is not None else None self._ws = PipeIOStream(self._wpipe) if self._wpipe is not None else None self.auto_close = auto_close self.state = StreamState.init self.exception = None @tornado.gen.coroutine def read(self): if self.exception: raise self.exception if self.state == StreamState.completed or self._rpipe is None: raise tornado.gen.Return("") elif self.state == StreamState.init: self.state = StreamState.streaming chunk = "" try: chunk = yield self._rs.read_bytes(common.MAX_PAYLOAD_SIZE, partial=True) except StreamClosedError: # reach the end of the pipe stream self.state = StreamState.completed finally: if self.exception: raise self.exception raise tornado.gen.Return(chunk) @tornado.gen.coroutine def write(self, chunk): assert self._wpipe is not None if self.exception: raise self.exception try: yield self._ws.write(chunk) self.state = StreamState.streaming except StreamClosedError: self.state = StreamState.completed raise UnexpectedError("Stream has been closed.") finally: if self.exception: raise self.exception def set_exception(self, exception): self.exception = exception self.close() def close(self): self.state = StreamState.completed if self._ws and self.auto_close: self._ws.close() if self._rs and self.auto_close: self._rs.close()
def __init__(self, command, timeout=-1, stdout_chunk_callback=None, stderr_chunk_callback=None, exit_process_callback=None, stdin_bytes=None, io_loop=None, kill_on_timeout=False): """ Initializes the subprocess with callbacks and timeout. :param command: command like ['java', '-jar', 'test.jar'] :param timeout: timeout for subprocess to complete, if negative or zero then no timeout :param stdout_chunk_callback: callback(bytes_data_chuck_from_stdout) :param stderr_chunk_callback: callback(bytes_data_chuck_from_stderr) :param exit_process_callback: callback(exit_code, was_expired_by_timeout) :param stdin_bytes: bytes data to send to stdin :param io_loop: tornado io loop on None for current :param kill_on_timeout: kill(-9) or terminate(-15)? """ self.aa_exit_process_callback = exit_process_callback self.aa_kill_on_timeout = kill_on_timeout stdin = Subprocess.STREAM if stdin_bytes else None stdout = Subprocess.STREAM if stdout_chunk_callback else None stderr = Subprocess.STREAM if stderr_chunk_callback else None Subprocess.__init__(self, command, stdin=stdin, stdout=stdout, stderr=stderr, io_loop=io_loop, shell=True) self.aa_process_expired = False self.aa_terminate_timeout = self.io_loop.call_later( timeout, self.aa_timeout_callback) if timeout > 0 else None self.set_exit_callback(self.aa_exit_callback) if stdin: self.stdin.write(stdin_bytes) self.stdin.close() if stdout: output_stream = PipeIOStream(self.stdout.fileno()) def on_stdout_chunk(data): stdout_chunk_callback(data) if not output_stream.closed(): output_stream.read_bytes(102400, on_stdout_chunk, None, True) output_stream.read_bytes(102400, on_stdout_chunk, None, True) if stderr: stderr_stream = PipeIOStream(self.stderr.fileno()) def on_stderr_chunk(data): stdout_chunk_callback(data) if not stderr_stream.closed(): stderr_stream.read_bytes(102400, on_stderr_chunk, None, True) stderr_stream.read_bytes(102400, on_stderr_chunk, None, True)