async def run_task(context): """Run the task, sending stdout+stderr to files. https://github.com/python/asyncio/blob/master/examples/subprocess_shell.py Args: context (scriptworker.context.Context): the scriptworker context. Returns: int: exit code """ kwargs = { # pragma: no branch 'stdout': PIPE, 'stderr': PIPE, 'stdin': None, 'close_fds': True, 'preexec_fn': lambda: os.setsid(), } context.proc = await asyncio.create_subprocess_exec( *context.config['task_script'], **kwargs) timeout = context.config['task_max_timeout'] with get_log_filehandle(context) as log_filehandle: stderr_future = asyncio.ensure_future( pipe_to_log(context.proc.stderr, filehandles=[log_filehandle])) stdout_future = asyncio.ensure_future( pipe_to_log(context.proc.stdout, filehandles=[log_filehandle])) try: _, pending = await asyncio.wait([stderr_future, stdout_future], timeout=timeout) if pending: await kill_proc( context.proc, "Exceeded task_max_timeout of {} seconds".format(timeout), context.config['task_max_timeout_status']) finally: # in the case of a timeout, this will be -15. # this code is in the finally: block so we still get the final # log lines. exitcode = await context.proc.wait() # make sure we haven't lost any of the logs await asyncio.wait([stdout_future, stderr_future]) # add an exit code line at the end of the log status_line = "exit code: {}".format(exitcode) if exitcode < 0: status_line = "Automation Error: python exited with signal {}".format( exitcode) log.info(status_line) print(status_line, file=log_filehandle) # reset context.proc context.proc = None return exitcode
async def test_pipe_to_log(context): cmd = r""">&2 echo "foo" && echo "bar" && exit 0""" proc = await asyncio.create_subprocess_exec( "bash", "-c", cmd, stdout=PIPE, stderr=PIPE, stdin=None ) tasks = [] with swlog.get_log_filehandle(context) as log_fh: tasks.append(swlog.pipe_to_log(proc.stderr, filehandles=[log_fh])) tasks.append(swlog.pipe_to_log(proc.stdout, filehandles=[log_fh])) await asyncio.wait(tasks) await proc.wait() log_file = swlog.get_log_filename(context) assert read(log_file) in ("foo\nbar\n", "bar\nfoo\n")
async def test_pipe_to_log(context): cmd = r""">&2 echo "foo" && echo "bar" && exit 0""" proc = await asyncio.create_subprocess_exec("bash", "-c", cmd, stdout=PIPE, stderr=PIPE, stdin=None) tasks = [] with swlog.get_log_filehandle(context) as log_fh: tasks.append(swlog.pipe_to_log(proc.stderr, filehandles=[log_fh])) tasks.append(swlog.pipe_to_log(proc.stdout, filehandles=[log_fh])) await asyncio.wait(tasks) await proc.wait() log_file = swlog.get_log_filename(context) assert read(log_file) in ("foo\nbar\n", "bar\nfoo\n")
def test_pipe_to_log(context, event_loop): cmd = r""">&2 echo "foo" && echo "bar" && exit 0""" proc = event_loop.run_until_complete( asyncio.create_subprocess_exec("bash", "-c", cmd, stdout=PIPE, stderr=PIPE, stdin=None)) tasks = [] with swlog.get_log_fhs(context) as (log_fh, error_fh): tasks.append( swlog.pipe_to_log(proc.stderr, filehandles=[log_fh, error_fh])) tasks.append(swlog.pipe_to_log(proc.stdout, filehandles=[log_fh])) event_loop.run_until_complete(asyncio.wait(tasks)) event_loop.run_until_complete(proc.wait()) log_file, error_file = swlog.get_log_filenames(context) assert read(log_file) in ("foo\nbar\n", "bar\nfoo\n") assert read(error_file) == "foo\n"
async def run_task(context): """Run the task, sending stdout+stderr to files. https://github.com/python/asyncio/blob/master/examples/subprocess_shell.py Args: context (scriptworker.context.Context): the scriptworker context. Returns: int: exit code """ loop = asyncio.get_event_loop() kwargs = { # pragma: no branch 'stdout': PIPE, 'stderr': PIPE, 'stdin': None, 'close_fds': True, 'preexec_fn': lambda: os.setsid(), } context.proc = await asyncio.create_subprocess_exec( *context.config['task_script'], **kwargs) loop.call_later(context.config['task_max_timeout'], max_timeout, context, context.proc, context.config['task_max_timeout']) tasks = [] with get_log_filehandle(context) as log_filehandle: tasks.append( pipe_to_log(context.proc.stderr, filehandles=[log_filehandle])) tasks.append( pipe_to_log(context.proc.stdout, filehandles=[log_filehandle])) await asyncio.wait(tasks) exitcode = await context.proc.wait() status_line = "exit code: {}".format(exitcode) if exitcode == -11: status_line = "Automation Error: python exited with signal {}".format( exitcode) log.info(status_line) print(status_line, file=log_filehandle) context.proc = None return exitcode
async def run_task(context, to_cancellable_process): """Run the task, sending stdout+stderr to files. https://github.com/python/asyncio/blob/master/examples/subprocess_shell.py Args: context (scriptworker.context.Context): the scriptworker context. to_cancellable_process (types.Callable): tracks the process so that it can be stopped if the worker is shut down Returns: int: 1 on failure, 0 on success """ env = deepcopy(os.environ) env['TASK_ID'] = context.task_id or 'None' kwargs = { # pragma: no branch 'stdout': PIPE, 'stderr': PIPE, 'stdin': None, 'close_fds': True, 'preexec_fn': lambda: os.setsid(), 'env': env, } subprocess = await asyncio.create_subprocess_exec( *context.config['task_script'], **kwargs) context.proc = await to_cancellable_process(TaskProcess(subprocess)) timeout = context.config['task_max_timeout'] with get_log_filehandle(context) as log_filehandle: stderr_future = asyncio.ensure_future( pipe_to_log(context.proc.process.stderr, filehandles=[log_filehandle])) stdout_future = asyncio.ensure_future( pipe_to_log(context.proc.process.stdout, filehandles=[log_filehandle])) try: _, pending = await asyncio.wait([stderr_future, stdout_future], timeout=timeout) if pending: message = "Exceeded task_max_timeout of {} seconds".format( timeout) log.warning(message) await context.proc.stop() raise ScriptWorkerTaskException( message, exit_code=context.config['task_max_timeout_status']) finally: # in the case of a timeout, this will be -15. # this code is in the finally: block so we still get the final # log lines. exitcode = await context.proc.process.wait() # make sure we haven't lost any of the logs await asyncio.wait([stdout_future, stderr_future]) # add an exit code line at the end of the log status_line = "exit code: {}".format(exitcode) if exitcode < 0: status_line = "Automation Error: python exited with signal {}".format( exitcode) log.info(status_line) print(status_line, file=log_filehandle) stopped_due_to_worker_shutdown = context.proc.stopped_due_to_worker_shutdown context.proc = None if stopped_due_to_worker_shutdown: raise WorkerShutdownDuringTask return 1 if exitcode != 0 else 0
async def run_task(context, to_cancellable_process): """Run the task, sending stdout+stderr to files. https://github.com/python/asyncio/blob/master/examples/subprocess_shell.py Args: context (scriptworker.context.Context): the scriptworker context. to_cancellable_process (types.Callable): tracks the process so that it can be stopped if the worker is shut down Returns: int: exit code """ kwargs = { # pragma: no branch 'stdout': PIPE, 'stderr': PIPE, 'stdin': None, 'close_fds': True, 'preexec_fn': lambda: os.setsid(), } subprocess = await asyncio.create_subprocess_exec(*context.config['task_script'], **kwargs) context.proc = await to_cancellable_process(TaskProcess(subprocess)) timeout = context.config['task_max_timeout'] with get_log_filehandle(context) as log_filehandle: stderr_future = asyncio.ensure_future( pipe_to_log(context.proc.process.stderr, filehandles=[log_filehandle]) ) stdout_future = asyncio.ensure_future( pipe_to_log(context.proc.process.stdout, filehandles=[log_filehandle]) ) try: _, pending = await asyncio.wait( [stderr_future, stdout_future], timeout=timeout ) if pending: message = "Exceeded task_max_timeout of {} seconds".format(timeout) log.warning(message) await context.proc.stop() raise ScriptWorkerTaskException(message, exit_code=context.config['task_max_timeout_status']) finally: # in the case of a timeout, this will be -15. # this code is in the finally: block so we still get the final # log lines. exitcode = await context.proc.process.wait() # make sure we haven't lost any of the logs await asyncio.wait([stdout_future, stderr_future]) # add an exit code line at the end of the log status_line = "exit code: {}".format(exitcode) if exitcode < 0: status_line = "Automation Error: python exited with signal {}".format(exitcode) log.info(status_line) print(status_line, file=log_filehandle) stopped_due_to_worker_shutdown = context.proc.stopped_due_to_worker_shutdown context.proc = None if stopped_due_to_worker_shutdown: raise WorkerShutdownDuringTask return exitcode