def test_output_tailing(self):
shell_command = ShellCommand(delay=DELAY)
output_reader, output_writer = Pipe(duplex=False)
command = 'echo "Hello"; sleep 1; echo "Bye"'
process = Process(target=shell_command,
args=(output_writer, command),
kwargs=dict(shell=True))
process.start()
self.assertEqual(list(read_messages(output_reader, timeout=DELAY)), [
'[Command: {}] -- Starting...'.format(command), '[STDOUT] -- Hello'
])
process.join()
self.assertEqual(list(read_messages(output_reader, timeout=DELAY)),
['[STDOUT] -- Bye'])
def serialize_step_data(context, timeout=0.1):
"""Serialize the step data in the context dictionary.
:param context: The context mapping of the following form:
{
'step_data': {
<step id>: {
# All fields are optional in this sub-dictionary
'rw_connection': The connection used to for two-way communication with a step.
'ro_connection': The connection used for one-way communication out of a step.
'io': The I/O messages collected from a step.
'output': The output messages collected from a step.
'return_value: The return value of running a step.
'exception': The exception raised by the step.
},
...
}
# Other custom fields.
...
}
:param timeout: The timeout in seconds to wait for messages from the step (I/O and output).
:return: A dictionary of the form:
{
<ID of Step 1>: {
'return_value': <The value returned by running the step> or None,
'exception': <Any exception raised by the step> or None,
'io': <List of messages sent to and from the step> or [],
'output': <List of output messages sent from the step> or [],
},
...
}
"""
serialized_step_data = {}
for step_id, step_data in context['step_data'].items():
io = list(read_messages(step_data['rw_connection'], timeout=timeout)) \
if 'rw_connection' in step_data else []
output = list(read_messages(step_data['ro_connection'], timeout=timeout)) \
if 'ro_connection' in step_data else []
step_data.setdefault('io', []).extend(io)
step_data.setdefault('output', []).extend(output)
serialized_step_data[step_id] = {
'return_value': step_data.get('return_value'),
'exception': step_data.get('exception'),
'io': step_data['io'],
'output': step_data['output']
}
return serialized_step_data
def test_successful_run(self):
shell_command = ShellCommand(delay=DELAY)
output_reader, output_writer = Pipe(duplex=False)
shell_command(output_writer, 'echo "Hello"')
self.assertEqual(
list(read_messages(output_reader, timeout=DELAY)),
['[Command: echo "Hello"] -- Starting...', '[STDOUT] -- Hello'])
def test_output_filter(self):
shell_command = ShellCommand(stdout_filter=lambda x: x
if 'Hello' not in x else None,
delay=DELAY)
output_reader, output_writer = Pipe(duplex=False)
shell_command(output_writer, 'echo "Hello\nBye"')
self.assertEqual(
list(read_messages(output_reader, timeout=DELAY)),
['[Command: echo "Hello\nBye"] -- Starting...', '[STDOUT] -- Bye'])
def test_shell_command_step(self):
shell_command = ShellCommand(delay=DELAY)
output_reader, output_writer = Pipe(duplex=False)
step = Step(shell_command)
step.start(output_writer, 'echo "Hello"')
step.join()
self.assertFalse(step.is_alive())
self.assertEqual(str(step), str(shell_command))
self.assertEqual(
list(read_messages(output_reader, timeout=DELAY)),
['[Command: echo "Hello"] -- Starting...', '[STDOUT] -- Hello'])
def test_stdin(self):
shell_command = ShellCommand(delay=DELAY)
output_reader, output_writer = Pipe(duplex=False)
input_reader, input_writer = Pipe(duplex=False)
input_writer.send('foo\n')
shell_command(output_writer,
'read a; echo $a',
input_reader=input_reader,
shell=True)
self.assertEqual(
list(read_messages(output_reader, timeout=DELAY)),
['[Command: read a; echo $a] -- Starting...', '[STDOUT] -- foo'])
def test_error_detection(self):
shell_command = ShellCommand(error_filter=lambda x: x
if 'Error' in x else None,
delay=DELAY)
output_reader, output_writer = Pipe(duplex=False)
with self.assertRaises(ShellCommandFailedError):
shell_command(output_writer, 'echo "Hello\nSome Error"')
self.assertEqual(list(read_messages(output_reader, timeout=DELAY)), [
'[Command: echo "Hello\nSome Error"] -- Starting...',
'[STDOUT] -- Hello'
])
def test_error_filter(self):
shell_command = ShellCommand(stderr_filter=lambda x: None
if 'No such file' in x else x,
delay=DELAY)
output_reader, output_writer = Pipe(duplex=False)
filename = 'foo_bar_baz'
command = 'ls {}'.format(filename)
with self.assertRaises(ShellCommandFailedError):
shell_command(output_writer, command)
self.assertEqual(list(read_messages(output_reader, timeout=DELAY)), [
'[Command: {}] -- Starting...'.format(command),
])
def test_exit_code_check(self):
shell_command = ShellCommand(delay=DELAY)
output_reader, output_writer = Pipe(duplex=False)
filename = 'foo_bar_baz'
command = 'ls {}'.format(filename)
with self.assertRaises(ShellCommandFailedError):
shell_command(output_writer, command)
stdout = list(read_messages(output_reader, timeout=DELAY))
self.assertEqual(stdout[0],
'[Command: {}] -- Starting...'.format(command))
self.assertIn('[STDERR] -- ', stdout[1])
self.assertIn('No such file or directory', stdout[1])
def test_exit_code_filter(self):
shell_command = ShellCommand(exit_code_filter=lambda x: None
if x != 0 else x,
delay=DELAY)
output_reader, output_writer = Pipe(duplex=False)
filename = 'foo_bar_baz'
command = 'ls {}'.format(filename)
shell_command(output_writer, command)
stdout = list(read_messages(output_reader, timeout=DELAY))
self.assertEqual(stdout[0],
'[Command: {}] -- Starting...'.format(command))
self.assertIn('[STDERR] -- ', stdout[1])
self.assertIn('No such file or directory', stdout[1])
def __call__(self, output_writer, command, input_reader=None, shell=False):
"""Runs the system command, send STDOUT and STDERR messages to the output_writer and relay
messages from the input_reader to STDIN.
This callable does the following:
1. Runs the system command as a sub-process.
2. Reads the messages from STDOUT and STDERR.
3. Check for errors. (See __init__). Raises ShellCommandFailedError if any errors are detected.
4. Filter out STDOUT and STDERR messages (See __init__).
5. Send the messages using the output_writer's send() method.
6. Read any messages received at the input_reader and relay them to STDIN.
7. Once the command completes, checks the exit code. Raises ShellCommandFailedError based on the exit code.
:param output_writer: A multiprocessing.Connection like object that supports a send(<utf-8 string>) method.
All output message (STDOUT and STDERR) will be written to this connection. They will be
formatted and prefixed with "STDOUT" and "STDERR" respectively.
:param command: The system command to run as a string. E.g., 'ls -l'
:param input_reader: A multiprocessing.Connection like object that supports the poll(<timeout>) and recv()
methods.
Any messages received here will be written to the STDIN of the command being run.
:param shell: The shell argument (which defaults to False) specifies whether to use the shell as the
program to execute.
:return: None.
Side-effect:
"""
output_writer.send('[Command: {}] -- Starting...'.format(command))
# If shell is True, it is recommended to pass args as a string rather than as a sequence. (Python docs)
command = shlex.split(command) if shell is False else command
command_process, stdin, stdout, stderr = run_shell_command(command,
shell=shell)
processes = [command_process]
stdout_reader, stdout_writer = Pipe(duplex=False)
stderr_reader, stderr_writer = Pipe(duplex=False)
processes.append(create_stream_listener(stdout, stdout_writer))
processes.append(create_stream_listener(stderr, stderr_writer))
if input_reader:
processes.append(create_stream_writer(input_reader, stdin))
while True:
# The process may be running at the start of the loop and may finish when the loop body is handling the
# messages (e.g., the process may write messages to STDOUT when the messages from STDERR are being handled)
# To avoid such messages being lost, we collect the process state before handling messages and decide to
# continue the loop if the process was running when we started reading messages. Doing this later may cause
# messages to be lost.
exit_code = command_process.poll()
try:
send_messages(
output_writer,
('[STDOUT] -- {}'.format(message)
for message in self._filter_messages(
read_messages(stdout_reader, timeout=self.delay),
self.stdout_filter)))
send_messages(
output_writer,
('[STDERR] -- {}'.format(message)
for message in self._filter_messages(
read_messages(stderr_reader), self.stderr_filter)))
except ShellCommandFailedError:
terminate(processes)
raise
if exit_code is not None:
break
terminate(processes)
if self.exit_code_filter(exit_code):
raise ShellCommandFailedError(
'[Command: {command}] -- Failed with exit code: {exit_code}'.
format(command=command, exit_code=exit_code))