async def run_task(context):
"""Run the task, sending stdout+stderr to files.
https://github.com/python/asyncio/blob/master/examples/subprocess_shell.py
Args:
context (scriptworker.context.Context): the scriptworker context.
Returns:
int: exit code
"""
kwargs = { # pragma: no branch
'stdout': PIPE,
'stderr': PIPE,
'stdin': None,
'close_fds': True,
'preexec_fn': lambda: os.setsid(),
}
context.proc = await asyncio.create_subprocess_exec(
*context.config['task_script'], **kwargs)
timeout = context.config['task_max_timeout']
with get_log_filehandle(context) as log_filehandle:
stderr_future = asyncio.ensure_future(
pipe_to_log(context.proc.stderr, filehandles=[log_filehandle]))
stdout_future = asyncio.ensure_future(
pipe_to_log(context.proc.stdout, filehandles=[log_filehandle]))
try:
_, pending = await asyncio.wait([stderr_future, stdout_future],
timeout=timeout)
if pending:
await kill_proc(
context.proc,
"Exceeded task_max_timeout of {} seconds".format(timeout),
context.config['task_max_timeout_status'])
finally:
# in the case of a timeout, this will be -15.
# this code is in the finally: block so we still get the final
# log lines.
exitcode = await context.proc.wait()
# make sure we haven't lost any of the logs
await asyncio.wait([stdout_future, stderr_future])
# add an exit code line at the end of the log
status_line = "exit code: {}".format(exitcode)
if exitcode < 0:
status_line = "Automation Error: python exited with signal {}".format(
exitcode)
log.info(status_line)
print(status_line, file=log_filehandle)
# reset context.proc
context.proc = None
return exitcode
async def test_pipe_to_log(context):
cmd = r""">&2 echo "foo" && echo "bar" && exit 0"""
proc = await asyncio.create_subprocess_exec(
"bash", "-c", cmd,
stdout=PIPE, stderr=PIPE, stdin=None
)
tasks = []
with swlog.get_log_filehandle(context) as log_fh:
tasks.append(swlog.pipe_to_log(proc.stderr, filehandles=[log_fh]))
tasks.append(swlog.pipe_to_log(proc.stdout, filehandles=[log_fh]))
await asyncio.wait(tasks)
await proc.wait()
log_file = swlog.get_log_filename(context)
assert read(log_file) in ("foo\nbar\n", "bar\nfoo\n")
async def test_pipe_to_log(context):
cmd = r""">&2 echo "foo" && echo "bar" && exit 0"""
proc = await asyncio.create_subprocess_exec("bash",
"-c",
cmd,
stdout=PIPE,
stderr=PIPE,
stdin=None)
tasks = []
with swlog.get_log_filehandle(context) as log_fh:
tasks.append(swlog.pipe_to_log(proc.stderr, filehandles=[log_fh]))
tasks.append(swlog.pipe_to_log(proc.stdout, filehandles=[log_fh]))
await asyncio.wait(tasks)
await proc.wait()
log_file = swlog.get_log_filename(context)
assert read(log_file) in ("foo\nbar\n", "bar\nfoo\n")
async def run_task(context):
"""Run the task, sending stdout+stderr to files.
https://github.com/python/asyncio/blob/master/examples/subprocess_shell.py
Args:
context (scriptworker.context.Context): the scriptworker context.
Returns:
int: exit code
"""
loop = asyncio.get_event_loop()
kwargs = { # pragma: no branch
'stdout': PIPE,
'stderr': PIPE,
'stdin': None,
'close_fds': True,
'preexec_fn': lambda: os.setsid(),
}
context.proc = await asyncio.create_subprocess_exec(
*context.config['task_script'], **kwargs)
loop.call_later(context.config['task_max_timeout'], max_timeout, context,
context.proc, context.config['task_max_timeout'])
tasks = []
with get_log_filehandle(context) as log_filehandle:
tasks.append(
pipe_to_log(context.proc.stderr, filehandles=[log_filehandle]))
tasks.append(
pipe_to_log(context.proc.stdout, filehandles=[log_filehandle]))
await asyncio.wait(tasks)
exitcode = await context.proc.wait()
status_line = "exit code: {}".format(exitcode)
if exitcode == -11:
status_line = "Automation Error: python exited with signal {}".format(
exitcode)
log.info(status_line)
print(status_line, file=log_filehandle)
context.proc = None
return exitcode
async def run_task(context, to_cancellable_process):
"""Run the task, sending stdout+stderr to files.
https://github.com/python/asyncio/blob/master/examples/subprocess_shell.py
Args:
context (scriptworker.context.Context): the scriptworker context.
to_cancellable_process (types.Callable): tracks the process so that it can be stopped if the worker is shut down
Returns:
int: 1 on failure, 0 on success
"""
env = deepcopy(os.environ)
env['TASK_ID'] = context.task_id or 'None'
kwargs = { # pragma: no branch
'stdout': PIPE,
'stderr': PIPE,
'stdin': None,
'close_fds': True,
'preexec_fn': lambda: os.setsid(),
'env': env,
}
subprocess = await asyncio.create_subprocess_exec(
*context.config['task_script'], **kwargs)
context.proc = await to_cancellable_process(TaskProcess(subprocess))
timeout = context.config['task_max_timeout']
with get_log_filehandle(context) as log_filehandle:
stderr_future = asyncio.ensure_future(
pipe_to_log(context.proc.process.stderr,
filehandles=[log_filehandle]))
stdout_future = asyncio.ensure_future(
pipe_to_log(context.proc.process.stdout,
filehandles=[log_filehandle]))
try:
_, pending = await asyncio.wait([stderr_future, stdout_future],
timeout=timeout)
if pending:
message = "Exceeded task_max_timeout of {} seconds".format(
timeout)
log.warning(message)
await context.proc.stop()
raise ScriptWorkerTaskException(
message,
exit_code=context.config['task_max_timeout_status'])
finally:
# in the case of a timeout, this will be -15.
# this code is in the finally: block so we still get the final
# log lines.
exitcode = await context.proc.process.wait()
# make sure we haven't lost any of the logs
await asyncio.wait([stdout_future, stderr_future])
# add an exit code line at the end of the log
status_line = "exit code: {}".format(exitcode)
if exitcode < 0:
status_line = "Automation Error: python exited with signal {}".format(
exitcode)
log.info(status_line)
print(status_line, file=log_filehandle)
stopped_due_to_worker_shutdown = context.proc.stopped_due_to_worker_shutdown
context.proc = None
if stopped_due_to_worker_shutdown:
raise WorkerShutdownDuringTask
return 1 if exitcode != 0 else 0
def test_get_log_filehandle(context, text):
log_file = swlog.get_log_filename(context)
with swlog.get_log_filehandle(context) as log_fh:
log_fh.write(text)
log_fh.write(text)
assert read(log_file) == text + text
async def run_task(context, to_cancellable_process):
"""Run the task, sending stdout+stderr to files.
https://github.com/python/asyncio/blob/master/examples/subprocess_shell.py
Args:
context (scriptworker.context.Context): the scriptworker context.
to_cancellable_process (types.Callable): tracks the process so that it can be stopped if the worker is shut down
Returns:
int: exit code
"""
kwargs = { # pragma: no branch
'stdout': PIPE,
'stderr': PIPE,
'stdin': None,
'close_fds': True,
'preexec_fn': lambda: os.setsid(),
}
subprocess = await asyncio.create_subprocess_exec(*context.config['task_script'], **kwargs)
context.proc = await to_cancellable_process(TaskProcess(subprocess))
timeout = context.config['task_max_timeout']
with get_log_filehandle(context) as log_filehandle:
stderr_future = asyncio.ensure_future(
pipe_to_log(context.proc.process.stderr, filehandles=[log_filehandle])
)
stdout_future = asyncio.ensure_future(
pipe_to_log(context.proc.process.stdout, filehandles=[log_filehandle])
)
try:
_, pending = await asyncio.wait(
[stderr_future, stdout_future], timeout=timeout
)
if pending:
message = "Exceeded task_max_timeout of {} seconds".format(timeout)
log.warning(message)
await context.proc.stop()
raise ScriptWorkerTaskException(message, exit_code=context.config['task_max_timeout_status'])
finally:
# in the case of a timeout, this will be -15.
# this code is in the finally: block so we still get the final
# log lines.
exitcode = await context.proc.process.wait()
# make sure we haven't lost any of the logs
await asyncio.wait([stdout_future, stderr_future])
# add an exit code line at the end of the log
status_line = "exit code: {}".format(exitcode)
if exitcode < 0:
status_line = "Automation Error: python exited with signal {}".format(exitcode)
log.info(status_line)
print(status_line, file=log_filehandle)
stopped_due_to_worker_shutdown = context.proc.stopped_due_to_worker_shutdown
context.proc = None
if stopped_due_to_worker_shutdown:
raise WorkerShutdownDuringTask
return exitcode
def test_get_log_filehandle(context, text):
log_file = swlog.get_log_filename(context)
with swlog.get_log_filehandle(context) as log_fh:
log_fh.write(text)
log_fh.write(text)
assert read(log_file) == text + text
