def test_no_timestamp_doesnt_delete_things(self):
now = date_utils.get_datetime_utc_now()
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=15)
exec_model['end_timestamp'] = now - timedelta(days=14)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(
exec_model['id'], count=3)
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 3)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 3)
expected_msg = 'Specify a valid timestamp'
self.assertRaisesRegexp(ValueError,
expected_msg,
purge_executions,
logger=LOG,
timestamp=None)
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 3)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 3)
def test_real_time_output_streaming_bufsize(self):
# Test various values for bufsize and verify it works / doesn't hang the process
cfg.CONF.set_override(name='stream_output', group='actionrunner', override=True)
bufsize_values = [-100, -2, -1, 0, 1, 2, 1024, 2048, 4096, 10000]
for index, bufsize in enumerate(bufsize_values, 1):
cfg.CONF.set_override(name='stream_output_buffer_size', override=bufsize,
group='actionrunner')
output_dbs = ActionExecutionOutput.get_all()
# Unexpected third party warnings will also inflate this number
self.assertGreaterEqual(len(output_dbs), (index - 1) * 4)
runner = self._get_mock_runner_obj()
runner.entry_point = PRINT_TO_STDOUT_STDERR_ACTION
runner.pre_run()
(_, output, _) = runner.run({'stdout_count': 2, 'stderr_count': 2})
# assertMultiLineEqual displays a diff if the two don't match
self.assertMultiLineEqual(output['stdout'], 'stdout line 0\nstdout line 1\n')
# Third party packages can unexpectedly emit warnings and add more
# output to the streamed stderr, so we check that the expected
# lines occurred, but we allow additional lines to exist
self.assertIn('stderr line 0\n', output['stderr'])
self.assertIn('stderr line 1\n', output['stderr'])
self.assertEqual(output['exit_code'], 0)
output_dbs = ActionExecutionOutput.get_all()
# Unexpected third party warnings will also inflate this number
self.assertGreaterEqual(len(output_dbs), (index) * 4)
def _insert_mock_stdout_and_stderr_objects_for_execution(
self, execution_id, count=5
):
execution_id = str(execution_id)
stdout_dbs, stderr_dbs = [], []
for i in range(0, count):
stdout_db = ActionExecutionOutputDB(
execution_id=execution_id,
action_ref="dummy.pack",
runner_ref="dummy",
output_type="stdout",
data="stdout %s" % (i),
)
ActionExecutionOutput.add_or_update(stdout_db)
stderr_db = ActionExecutionOutputDB(
execution_id=execution_id,
action_ref="dummy.pack",
runner_ref="dummy",
output_type="stderr",
data="stderr%s" % (i),
)
ActionExecutionOutput.add_or_update(stderr_db)
return stdout_dbs, stderr_dbs
def test_shell_command_action_basic(self):
models = self.fixtures_loader.load_models(
fixtures_pack='generic', fixtures_dict={'actions': ['local.yaml']})
action_db = models['actions']['local.yaml']
runner = self._get_runner(action_db, cmd='echo 10')
runner.pre_run()
status, result, _ = runner.run({})
runner.post_run(status, result)
self.assertEquals(status, action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertEquals(result['stdout'], 10)
# End result should be the same when streaming is enabled
cfg.CONF.set_override(name='stream_output',
group='actionrunner',
override=True)
# Verify initial state
output_dbs = ActionExecutionOutput.get_all()
self.assertEqual(len(output_dbs), 0)
runner = self._get_runner(action_db, cmd='echo 10')
runner.pre_run()
status, result, _ = runner.run({})
runner.post_run(status, result)
self.assertEquals(status, action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertEquals(result['stdout'], 10)
output_dbs = ActionExecutionOutput.get_all()
self.assertEqual(len(output_dbs), 1)
self.assertEqual(output_dbs[0].output_type, 'stdout')
self.assertEqual(output_dbs[0].data, '10\n')
def test_shell_command_action_basic(self):
models = self.fixtures_loader.load_models(
fixtures_pack="generic", fixtures_dict={"actions": ["local.yaml"]})
action_db = models["actions"]["local.yaml"]
runner = self._get_runner(action_db, cmd="echo 10")
runner.pre_run()
status, result, _ = runner.run({})
runner.post_run(status, result)
self.assertEqual(status, action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertEqual(result["stdout"], 10)
# End result should be the same when streaming is enabled
cfg.CONF.set_override(name="stream_output",
group="actionrunner",
override=True)
# Verify initial state
output_dbs = ActionExecutionOutput.get_all()
self.assertEqual(len(output_dbs), 0)
runner = self._get_runner(action_db, cmd="echo 10")
runner.pre_run()
status, result, _ = runner.run({})
runner.post_run(status, result)
self.assertEqual(status, action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertEqual(result["stdout"], 10)
output_dbs = ActionExecutionOutput.get_all()
self.assertEqual(len(output_dbs), 1)
self.assertEqual(output_dbs[0].output_type, "stdout")
self.assertEqual(output_dbs[0].data, "10\n")
def test_no_timestamp_doesnt_delete_things(self):
now = date_utils.get_datetime_utc_now()
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=15)
exec_model['end_timestamp'] = now - timedelta(days=14)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=3)
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 3)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 3)
expected_msg = 'Specify a valid timestamp'
self.assertRaisesRegexp(ValueError, expected_msg, purge_executions,
logger=LOG, timestamp=None)
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 3)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 3)
def test_shell_command_action_basic(self):
models = self.fixtures_loader.load_models(
fixtures_pack='generic', fixtures_dict={'actions': ['local.yaml']})
action_db = models['actions']['local.yaml']
runner = self._get_runner(action_db, cmd='echo 10')
runner.pre_run()
status, result, _ = runner.run({})
runner.post_run(status, result)
self.assertEquals(status, action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertEquals(result['stdout'], 10)
# End result should be the same when streaming is enabled
cfg.CONF.set_override(name='stream_output', group='actionrunner', override=True)
# Verify initial state
output_dbs = ActionExecutionOutput.get_all()
self.assertEqual(len(output_dbs), 0)
runner = self._get_runner(action_db, cmd='echo 10')
runner.pre_run()
status, result, _ = runner.run({})
runner.post_run(status, result)
self.assertEquals(status, action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertEquals(result['stdout'], 10)
output_dbs = ActionExecutionOutput.get_all()
self.assertEqual(len(output_dbs), 1)
self.assertEqual(output_dbs[0].output_type, 'stdout')
self.assertEqual(output_dbs[0].data, '10\n')
def test_real_time_output_streaming_bufsize(self):
# Test various values for bufsize and verify it works / doesn't hang the process
cfg.CONF.set_override(name='stream_output',
group='actionrunner',
override=True)
bufsize_values = [-100, -2, -1, 0, 1, 2, 1024, 2048, 4096, 10000]
for index, bufsize in enumerate(bufsize_values, 1):
cfg.CONF.set_override(name='stream_output_buffer_size',
override=bufsize,
group='actionrunner')
output_dbs = ActionExecutionOutput.get_all()
self.assertEqual(len(output_dbs), (index - 1) * 4)
runner = self._get_mock_runner_obj()
runner.entry_point = PRINT_TO_STDOUT_STDERR_ACTION
runner.pre_run()
(_, output, _) = runner.run({'stdout_count': 2, 'stderr_count': 2})
self.assertEqual(output['stdout'],
'stdout line 0\nstdout line 1\n')
self.assertEqual(output['stderr'],
'stderr line 0\nstderr line 1\n')
self.assertEqual(output['exit_code'], 0)
output_dbs = ActionExecutionOutput.get_all()
self.assertEqual(len(output_dbs), (index) * 4)
def test_action_stdout_and_stderr_is_stored_in_the_db(
self, mock_spawn, mock_popen):
# Feature is enabled
cfg.CONF.set_override(name="stream_output",
group="actionrunner",
override=True)
values = {"delimiter": ACTION_OUTPUT_RESULT_DELIMITER}
# Note: We need to mock spawn function so we can test everything in single event loop
# iteration
mock_spawn.side_effect = blocking_eventlet_spawn
# No output to stdout and no result (implicit None)
mock_stdout = [
"pre result line 1\n",
"pre result line 2\n",
"%(delimiter)sTrue%(delimiter)s" % values,
"post result line 1",
]
mock_stderr = ["stderr line 1\n", "stderr line 2\n", "stderr line 3\n"]
mock_process = mock.Mock()
mock_process.returncode = 0
mock_popen.return_value = mock_process
mock_process.stdout.closed = False
mock_process.stderr.closed = False
mock_process.stdout.readline = make_mock_stream_readline(
mock_process.stdout, mock_stdout, stop_counter=4)
mock_process.stderr.readline = make_mock_stream_readline(
mock_process.stderr, mock_stderr, stop_counter=3)
runner = self._get_mock_runner_obj()
runner.entry_point = PASCAL_ROW_ACTION_PATH
runner.pre_run()
(_, output, _) = runner.run({"row_index": 4})
self.assertMultiLineEqual(
output["stdout"],
"pre result line 1\npre result line 2\npost result line 1")
self.assertMultiLineEqual(
output["stderr"], "stderr line 1\nstderr line 2\nstderr line 3\n")
self.assertEqual(output["result"], "True")
self.assertEqual(output["exit_code"], 0)
# Verify stdout and stderr lines have been correctly stored in the db
# Note - result delimiter should not be stored in the db
output_dbs = ActionExecutionOutput.query(output_type="stdout")
self.assertEqual(len(output_dbs), 3)
self.assertEqual(output_dbs[0].runner_ref, "python-script")
self.assertEqual(output_dbs[0].data, mock_stdout[0])
self.assertEqual(output_dbs[1].data, mock_stdout[1])
self.assertEqual(output_dbs[2].data, mock_stdout[3])
output_dbs = ActionExecutionOutput.query(output_type="stderr")
self.assertEqual(len(output_dbs), 3)
self.assertEqual(output_dbs[0].runner_ref, "python-script")
self.assertEqual(output_dbs[0].data, mock_stderr[0])
self.assertEqual(output_dbs[1].data, mock_stderr[1])
self.assertEqual(output_dbs[2].data, mock_stderr[2])
def test_get_output_finished_execution(self):
# Test the execution output API endpoint for execution which has finished
for status in action_constants.LIVEACTION_COMPLETED_STATES:
# Insert mock execution and output objects
status = action_constants.LIVEACTION_STATUS_SUCCEEDED
timestamp = date_utils.get_datetime_utc_now()
action_execution_db = ActionExecutionDB(
start_timestamp=timestamp,
end_timestamp=timestamp,
status=status,
action={"ref": "core.local"},
runner={"name": "local-shell-cmd"},
liveaction={"ref": "foo"},
)
action_execution_db = ActionExecution.add_or_update(
action_execution_db)
for i in range(1, 6):
stdout_db = ActionExecutionOutputDB(
execution_id=str(action_execution_db.id),
action_ref="core.local",
runner_ref="dummy",
timestamp=timestamp,
output_type="stdout",
data="stdout %s\n" % (i),
)
ActionExecutionOutput.add_or_update(stdout_db)
for i in range(10, 15):
stderr_db = ActionExecutionOutputDB(
execution_id=str(action_execution_db.id),
action_ref="core.local",
runner_ref="dummy",
timestamp=timestamp,
output_type="stderr",
data="stderr %s\n" % (i),
)
ActionExecutionOutput.add_or_update(stderr_db)
resp = self.app.get(
"/v1/executions/%s/output" % (str(action_execution_db.id)),
expect_errors=False,
)
self.assertEqual(resp.status_int, 200)
events = self._parse_response(resp.text)
self.assertEqual(len(events), 11)
self.assertEqual(events[0][1]["data"], "stdout 1\n")
self.assertEqual(events[9][1]["data"], "stderr 14\n")
self.assertEqual(events[10][0], "EOF")
# Verify "last" short-hand id works
resp = self.app.get("/v1/executions/last/output",
expect_errors=False)
self.assertEqual(resp.status_int, 200)
events = self._parse_response(resp.text)
self.assertEqual(len(events), 11)
self.assertEqual(events[10][0], "EOF")
def test_action_stdout_and_stderr_is_stored_in_the_db(
self, mock_spawn, mock_popen):
# Feature is enabled
cfg.CONF.set_override(name='stream_output',
group='actionrunner',
override=True)
values = {'delimiter': ACTION_OUTPUT_RESULT_DELIMITER}
# Note: We need to mock spawn function so we can test everything in single event loop
# iteration
mock_spawn.side_effect = blocking_eventlet_spawn
# No output to stdout and no result (implicit None)
mock_stdout = [
'pre result line 1\n', 'pre result line 2\n',
'%(delimiter)sTrue%(delimiter)s' % values, 'post result line 1'
]
mock_stderr = ['stderr line 1\n', 'stderr line 2\n', 'stderr line 3\n']
mock_process = mock.Mock()
mock_process.returncode = 0
mock_popen.return_value = mock_process
mock_process.stdout.closed = False
mock_process.stderr.closed = False
mock_process.stdout.readline = make_mock_stream_readline(
mock_process.stdout, mock_stdout, stop_counter=4)
mock_process.stderr.readline = make_mock_stream_readline(
mock_process.stderr, mock_stderr, stop_counter=3)
runner = self._get_mock_runner_obj()
runner.entry_point = PASCAL_ROW_ACTION_PATH
runner.container_service = service.RunnerContainerService()
runner.pre_run()
(_, output, _) = runner.run({'row_index': 4})
self.assertEqual(
output['stdout'],
'pre result line 1\npre result line 2\npost result line 1')
self.assertEqual(output['stderr'],
'stderr line 1\nstderr line 2\nstderr line 3\n')
self.assertEqual(output['result'], 'True')
self.assertEqual(output['exit_code'], 0)
# Verify stdout and stderr lines have been correctly stored in the db
# Note - result delimiter should not be stored in the db
output_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(output_dbs), 3)
self.assertEqual(output_dbs[0].runner_ref, 'python-script')
self.assertEqual(output_dbs[0].data, mock_stdout[0])
self.assertEqual(output_dbs[1].data, mock_stdout[1])
self.assertEqual(output_dbs[2].data, mock_stdout[3])
output_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(output_dbs), 3)
self.assertEqual(output_dbs[0].runner_ref, 'python-script')
self.assertEqual(output_dbs[0].data, mock_stderr[0])
self.assertEqual(output_dbs[1].data, mock_stderr[1])
self.assertEqual(output_dbs[2].data, mock_stderr[2])
def test_action_stdout_and_stderr_is_stored_in_the_db(self, mock_spawn, mock_popen):
# Feature is enabled
cfg.CONF.set_override(name='stream_output', group='actionrunner', override=True)
values = {'delimiter': ACTION_OUTPUT_RESULT_DELIMITER}
# Note: We need to mock spawn function so we can test everything in single event loop
# iteration
mock_spawn.side_effect = blocking_eventlet_spawn
# No output to stdout and no result (implicit None)
mock_stdout = [
'pre result line 1\n',
'pre result line 2\n',
'%(delimiter)sTrue%(delimiter)s' % values,
'post result line 1'
]
mock_stderr = [
'stderr line 1\n',
'stderr line 2\n',
'stderr line 3\n'
]
mock_process = mock.Mock()
mock_process.returncode = 0
mock_popen.return_value = mock_process
mock_process.stdout.closed = False
mock_process.stderr.closed = False
mock_process.stdout.readline = make_mock_stream_readline(mock_process.stdout, mock_stdout,
stop_counter=4)
mock_process.stderr.readline = make_mock_stream_readline(mock_process.stderr, mock_stderr,
stop_counter=3)
runner = self._get_mock_runner_obj()
runner.entry_point = PASCAL_ROW_ACTION_PATH
runner.pre_run()
(_, output, _) = runner.run({'row_index': 4})
self.assertEqual(output['stdout'],
'pre result line 1\npre result line 2\npost result line 1')
self.assertEqual(output['stderr'], 'stderr line 1\nstderr line 2\nstderr line 3\n')
self.assertEqual(output['result'], 'True')
self.assertEqual(output['exit_code'], 0)
# Verify stdout and stderr lines have been correctly stored in the db
# Note - result delimiter should not be stored in the db
output_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(output_dbs), 3)
self.assertEqual(output_dbs[0].runner_ref, 'python-script')
self.assertEqual(output_dbs[0].data, mock_stdout[0])
self.assertEqual(output_dbs[1].data, mock_stdout[1])
self.assertEqual(output_dbs[2].data, mock_stdout[3])
output_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(output_dbs), 3)
self.assertEqual(output_dbs[0].runner_ref, 'python-script')
self.assertEqual(output_dbs[0].data, mock_stderr[0])
self.assertEqual(output_dbs[1].data, mock_stderr[1])
self.assertEqual(output_dbs[2].data, mock_stderr[2])
def publish_action_finished(action_execution_db):
# Insert mock output object
output_params['data'] = 'stdout pre finish 1\n'
output_db = ActionExecutionOutputDB(**output_params)
ActionExecutionOutput.add_or_update(output_db)
# Transition execution to completed state so the connection closes
action_execution_db.status = action_constants.LIVEACTION_STATUS_SUCCEEDED
action_execution_db = ActionExecution.add_or_update(action_execution_db)
def test_action_stdout_and_stderr_is_stored_in_the_db(
self, mock_spawn, mock_popen):
# Feature is enabled
cfg.CONF.set_override(name='stream_output',
group='actionrunner',
override=True)
# Note: We need to mock spawn function so we can test everything in single event loop
# iteration
mock_spawn.side_effect = blocking_eventlet_spawn
# No output to stdout and no result (implicit None)
mock_stdout = [
'stdout line 1\n',
'stdout line 2\n',
]
mock_stderr = ['stderr line 1\n', 'stderr line 2\n', 'stderr line 3\n']
mock_process = mock.Mock()
mock_process.returncode = 0
mock_popen.return_value = mock_process
mock_process.stdout.closed = False
mock_process.stderr.closed = False
mock_process.stdout.readline = make_mock_stream_readline(
mock_process.stdout, mock_stdout, stop_counter=2)
mock_process.stderr.readline = make_mock_stream_readline(
mock_process.stderr, mock_stderr, stop_counter=3)
models = self.fixtures_loader.load_models(
fixtures_pack='generic', fixtures_dict={'actions': ['local.yaml']})
action_db = models['actions']['local.yaml']
runner = self._get_runner(action_db, cmd='echo $ST2_ACTION_API_URL')
runner.pre_run()
status, result, _ = runner.run({})
runner.post_run(status, result)
self.assertEquals(status, action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertEqual(result['stdout'], 'stdout line 1\nstdout line 2')
self.assertEqual(result['stderr'],
'stderr line 1\nstderr line 2\nstderr line 3')
self.assertEqual(result['return_code'], 0)
# Verify stdout and stderr lines have been correctly stored in the db
output_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(output_dbs), 2)
self.assertEqual(output_dbs[0].data, mock_stdout[0])
self.assertEqual(output_dbs[1].data, mock_stdout[1])
output_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(output_dbs), 3)
self.assertEqual(output_dbs[0].data, mock_stderr[0])
self.assertEqual(output_dbs[1].data, mock_stderr[1])
self.assertEqual(output_dbs[2].data, mock_stderr[2])
def test_action_stdout_and_stderr_is_stored_in_the_db(self, mock_spawn, mock_popen):
# Feature is enabled
cfg.CONF.set_override(name='stream_output', group='actionrunner', override=True)
# Note: We need to mock spawn function so we can test everything in single event loop
# iteration
mock_spawn.side_effect = blocking_eventlet_spawn
# No output to stdout and no result (implicit None)
mock_stdout = [
'stdout line 1\n',
'stdout line 2\n',
]
mock_stderr = [
'stderr line 1\n',
'stderr line 2\n',
'stderr line 3\n'
]
mock_process = mock.Mock()
mock_process.returncode = 0
mock_popen.return_value = mock_process
mock_process.stdout.closed = False
mock_process.stderr.closed = False
mock_process.stdout.readline = make_mock_stream_readline(mock_process.stdout, mock_stdout,
stop_counter=2)
mock_process.stderr.readline = make_mock_stream_readline(mock_process.stderr, mock_stderr,
stop_counter=3)
models = self.fixtures_loader.load_models(
fixtures_pack='generic', fixtures_dict={'actions': ['local.yaml']})
action_db = models['actions']['local.yaml']
runner = self._get_runner(action_db, cmd='echo $ST2_ACTION_API_URL')
runner.pre_run()
status, result, _ = runner.run({})
runner.post_run(status, result)
self.assertEquals(status, action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertEqual(result['stdout'], 'stdout line 1\nstdout line 2')
self.assertEqual(result['stderr'], 'stderr line 1\nstderr line 2\nstderr line 3')
self.assertEqual(result['return_code'], 0)
# Verify stdout and stderr lines have been correctly stored in the db
output_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(output_dbs), 2)
self.assertEqual(output_dbs[0].data, mock_stdout[0])
self.assertEqual(output_dbs[1].data, mock_stdout[1])
output_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(output_dbs), 3)
self.assertEqual(output_dbs[0].data, mock_stderr[0])
self.assertEqual(output_dbs[1].data, mock_stderr[1])
self.assertEqual(output_dbs[2].data, mock_stderr[2])
def test_get_output_finished_execution(self):
# Test the execution output API endpoint for execution which has finished
for status in action_constants.LIVEACTION_COMPLETED_STATES:
# Insert mock execution and output objects
status = action_constants.LIVEACTION_STATUS_SUCCEEDED
timestamp = date_utils.get_datetime_utc_now()
action_execution_db = ActionExecutionDB(
start_timestamp=timestamp,
end_timestamp=timestamp,
status=status,
action={'ref': 'core.local'},
runner={'name': 'local-shell-cmd'},
liveaction={'ref': 'foo'})
action_execution_db = ActionExecution.add_or_update(
action_execution_db)
for i in range(1, 6):
stdout_db = ActionExecutionOutputDB(execution_id=str(
action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stdout',
data='stdout %s\n' % (i))
ActionExecutionOutput.add_or_update(stdout_db)
for i in range(10, 15):
stderr_db = ActionExecutionOutputDB(execution_id=str(
action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stderr',
data='stderr %s\n' % (i))
ActionExecutionOutput.add_or_update(stderr_db)
resp = self.app.get('/v1/executions/%s/output' %
(str(action_execution_db.id)),
expect_errors=False)
self.assertEqual(resp.status_int, 200)
events = self._parse_response(resp.text)
self.assertEqual(len(events), 11)
self.assertEqual(events[0][1]['data'], 'stdout 1\n')
self.assertEqual(events[9][1]['data'], 'stderr 14\n')
self.assertEqual(events[10][0], 'EOF')
# Verify "last" short-hand id works
resp = self.app.get('/v1/executions/last/output',
expect_errors=False)
self.assertEqual(resp.status_int, 200)
events = self._parse_response(resp.text)
self.assertEqual(len(events), 11)
self.assertEqual(events[10][0], 'EOF')
def publish_action_finished(action_execution_db):
# Insert mock output object
output_params['data'] = 'stdout pre finish 1\n'
output_db = ActionExecutionOutputDB(**output_params)
ActionExecutionOutput.add_or_update(output_db)
eventlet.sleep(1.0)
# Transition execution to completed state so the connection closes
action_execution_db.status = action_constants.LIVEACTION_STATUS_SUCCEEDED
action_execution_db = ActionExecution.add_or_update(action_execution_db)
def test_get_output_finished_execution(self):
# Test the execution output API endpoint for execution which has finished
for status in action_constants.LIVEACTION_COMPLETED_STATES:
# Insert mock execution and output objects
status = action_constants.LIVEACTION_STATUS_SUCCEEDED
timestamp = date_utils.get_datetime_utc_now()
action_execution_db = ActionExecutionDB(start_timestamp=timestamp,
end_timestamp=timestamp,
status=status,
action={'ref': 'core.local'},
runner={'name': 'local-shell-cmd'},
liveaction={'ref': 'foo'})
action_execution_db = ActionExecution.add_or_update(action_execution_db)
for i in range(1, 6):
stdout_db = ActionExecutionOutputDB(execution_id=str(action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stdout',
data='stdout %s\n' % (i))
ActionExecutionOutput.add_or_update(stdout_db)
for i in range(10, 15):
stderr_db = ActionExecutionOutputDB(execution_id=str(action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stderr',
data='stderr %s\n' % (i))
ActionExecutionOutput.add_or_update(stderr_db)
resp = self.app.get('/v1/executions/%s/output' % (str(action_execution_db.id)),
expect_errors=False)
self.assertEqual(resp.status_int, 200)
events = self._parse_response(resp.text)
self.assertEqual(len(events), 11)
self.assertEqual(events[0][1]['data'], 'stdout 1\n')
self.assertEqual(events[9][1]['data'], 'stderr 14\n')
self.assertEqual(events[10][0], 'EOF')
# Verify "last" short-hand id works
resp = self.app.get('/v1/executions/last/output', expect_errors=False)
self.assertEqual(resp.status_int, 200)
events = self._parse_response(resp.text)
self.assertEqual(len(events), 11)
self.assertEqual(events[10][0], 'EOF')
def purge_execution_output_objects(logger, timestamp, action_ref=None):
"""
Purge action executions output objects.
:param timestamp: Objects older than this timestamp will be deleted.
:type timestamp: ``datetime.datetime
:param action_ref: Only delete objects for the provided actions.
:type action_ref: ``str``
"""
if not timestamp:
raise ValueError('Specify a valid timestamp to purge.')
logger.info('Purging action execution output objects older than timestamp: %s' %
timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ'))
filters = {}
filters['timestamp__lt'] = timestamp
if action_ref:
filters['action_ref'] = action_ref
try:
deleted_count = ActionExecutionOutput.delete_by_query(**filters)
except InvalidQueryError as e:
msg = ('Bad query (%s) used to delete execution output instances: %s'
'Please contact support.' % (filters, six.text_type(e)))
raise InvalidQueryError(msg)
except:
logger.exception('Deletion of execution output models failed for query with filters: %s.',
filters)
else:
logger.info('Deleted %s execution output objects' % (deleted_count))
def existing_output_iter():
# Consume and return all of the existing lines
# pylint: disable=no-member
output_dbs = ActionExecutionOutput.query(execution_id=execution_id, **query_filters)
output = ''.join([output_db.data for output_db in output_dbs])
yield six.binary_type(output.encode('utf-8'))
def purge_execution_output_objects(logger, timestamp, action_ref=None):
"""
Purge action executions output objects.
:param timestamp: Objects older than this timestamp will be deleted.
:type timestamp: ``datetime.datetime
:param action_ref: Only delete objects for the provided actions.
:type action_ref: ``str``
"""
if not timestamp:
raise ValueError('Specify a valid timestamp to purge.')
logger.info('Purging action execution output objects older than timestamp: %s' %
timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ'))
filters = {}
filters['timestamp__lt'] = timestamp
if action_ref:
filters['action_ref'] = action_ref
try:
deleted_count = ActionExecutionOutput.delete_by_query(**filters)
except InvalidQueryError as e:
msg = ('Bad query (%s) used to delete execution output instances: %s'
'Please contact support.' % (filters, six.text_type(e)))
raise InvalidQueryError(msg)
except:
logger.exception('Deletion of execution output models failed for query with filters: %s.',
filters)
else:
logger.info('Deleted %s execution output objects' % (deleted_count))
def existing_output_iter():
# Consume and return all of the existing lines
# pylint: disable=no-member
output_dbs = ActionExecutionOutput.query(execution_id=execution_id, **query_filters)
output = ''.join([output_db.data for output_db in output_dbs])
yield six.binary_type(output.encode('utf-8'))
def existing_output_iter():
# Consume and return all of the existing lines
# pylint: disable=no-member
output_dbs = ActionExecutionOutput.query(execution_id=execution_id, **query_filters)
# Note: We return all at once instead of yield line by line to avoid multiple socket
# writes and to achieve better performance
output = ''.join([output_db.data for output_db in output_dbs])
yield six.binary_type(output.encode('utf-8'))
def existing_output_iter():
# Consume and return all of the existing lines
output_dbs = ActionExecutionOutput.query(execution_id=execution_id, **query_filters)
# Note: We return all at once instead of yield line by line to avoid multiple socket
# writes and to achieve better performance
output = [format_output_object(output_db) for output_db in output_dbs]
output = ''.join(output)
yield six.binary_type(output.encode('utf-8'))
def test_purge_executions_with_timestamp(self):
now = date_utils.get_datetime_utc_now()
# Write one execution after cut-off threshold
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=15)
exec_model['end_timestamp'] = now - timedelta(days=14)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(
exec_model['id'], count=3)
# Write one execution before cut-off threshold
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=22)
exec_model['end_timestamp'] = now - timedelta(days=21)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(
exec_model['id'], count=3)
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 2)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 6)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 6)
purge_executions(logger=LOG, timestamp=now - timedelta(days=20))
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 3)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 3)
def existing_output_iter():
# Consume and return all of the existing lines
output_dbs = ActionExecutionOutput.query(
execution_id=execution_id, **query_filters
)
# Note: We return all at once instead of yield line by line to avoid multiple socket
# writes and to achieve better performance
output = [format_output_object(output_db) for output_db in output_dbs]
output = "".join(output)
yield six.binary_type(output.encode("utf-8"))
def _insert_mock_stdout_and_stderr_objects_for_execution(self, execution_id, count=5):
execution_id = str(execution_id)
stdout_dbs, stderr_dbs = [], []
for i in range(0, count):
stdout_db = ActionExecutionOutputDB(execution_id=execution_id,
action_ref='dummy.pack',
runner_ref='dummy',
output_type='stdout',
data='stdout %s' % (i))
ActionExecutionOutput.add_or_update(stdout_db)
stderr_db = ActionExecutionOutputDB(execution_id=execution_id,
action_ref='dummy.pack',
runner_ref='dummy',
output_type='stderr',
data='stderr%s' % (i))
ActionExecutionOutput.add_or_update(stderr_db)
return stdout_dbs, stderr_dbs
def test_purge_executions_with_timestamp(self):
now = date_utils.get_datetime_utc_now()
# Write one execution after cut-off threshold
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=15)
exec_model['end_timestamp'] = now - timedelta(days=14)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=3)
# Write one execution before cut-off threshold
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=22)
exec_model['end_timestamp'] = now - timedelta(days=21)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=3)
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 2)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 6)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 6)
purge_executions(logger=LOG, timestamp=now - timedelta(days=20))
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 3)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 3)
def store_execution_output_data_ex(
execution_id, action_ref, runner_ref, data, output_type="output", timestamp=None
):
timestamp = timestamp or date_utils.get_datetime_utc_now()
output_db = ActionExecutionOutputDB(
execution_id=execution_id,
action_ref=action_ref,
runner_ref=runner_ref,
timestamp=timestamp,
output_type=output_type,
data=data,
)
output_db = ActionExecutionOutput.add_or_update(
output_db, publish=True, dispatch_trigger=False
)
return output_db
def store_execution_output_data(execution_db, action_db, data, output_type='output',
timestamp=None):
"""
Store output from an execution as a new document in the collection.
"""
execution_id = str(execution_db.id)
action_ref = action_db.ref
runner_ref = getattr(action_db, 'runner_type', {}).get('name', 'unknown')
timestamp = timestamp or date_utils.get_datetime_utc_now()
output_db = ActionExecutionOutputDB(execution_id=execution_id,
action_ref=action_ref,
runner_ref=runner_ref,
timestamp=timestamp,
output_type=output_type,
data=data)
output_db = ActionExecutionOutput.add_or_update(output_db, publish=True,
dispatch_trigger=False)
return output_db
def store_execution_output_data(execution_db, action_db, data, output_type='output',
timestamp=None):
"""
Store output from an execution as a new document in the collection.
"""
execution_id = str(execution_db.id)
action_ref = action_db.ref
runner_ref = getattr(action_db, 'runner_type', {}).get('name', 'unknown')
timestamp = timestamp or date_utils.get_datetime_utc_now()
output_db = ActionExecutionOutputDB(execution_id=execution_id,
action_ref=action_ref,
runner_ref=runner_ref,
timestamp=timestamp,
output_type=output_type,
data=data)
output_db = ActionExecutionOutput.add_or_update(output_db, publish=True,
dispatch_trigger=False)
return output_db
def test_purge_executions_with_action_ref(self):
now = date_utils.get_datetime_utc_now()
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=15)
exec_model['end_timestamp'] = now - timedelta(days=14)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(
exec_model['id'], count=3)
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 3)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 3)
# Invalid action reference, nothing should be deleted
purge_executions(logger=LOG,
action_ref='core.localzzz',
timestamp=now - timedelta(days=10))
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 3)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 3)
purge_executions(logger=LOG,
action_ref='core.local',
timestamp=now - timedelta(days=10))
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 0)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 0)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 0)
def test_purge_executions_with_action_ref(self):
now = date_utils.get_datetime_utc_now()
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=15)
exec_model['end_timestamp'] = now - timedelta(days=14)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=3)
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 3)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 3)
# Invalid action reference, nothing should be deleted
purge_executions(logger=LOG, action_ref='core.localzzz', timestamp=now - timedelta(days=10))
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 3)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 3)
purge_executions(logger=LOG, action_ref='core.local', timestamp=now - timedelta(days=10))
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 0)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 0)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 0)
def test_script_with_paramters_parameter_serialization(self):
models = self.fixtures_loader.load_models(
fixtures_pack='generic',
fixtures_dict={'actions': ['local_script_with_params.yaml']})
action_db = models['actions']['local_script_with_params.yaml']
entry_point = os.path.join(
get_fixtures_base_path(),
'generic/actions/local_script_with_params.sh')
action_parameters = {
'param_string': 'test string',
'param_integer': 1,
'param_float': 2.55,
'param_boolean': True,
'param_list': ['a', 'b', 'c'],
'param_object': {
'foo': 'bar'
}
}
runner = self._get_runner(action_db=action_db, entry_point=entry_point)
runner.pre_run()
status, result, _ = runner.run(action_parameters=action_parameters)
runner.post_run(status, result)
self.assertEqual(status, action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertTrue('PARAM_STRING=test string' in result['stdout'])
self.assertTrue('PARAM_INTEGER=1' in result['stdout'])
self.assertTrue('PARAM_FLOAT=2.55' in result['stdout'])
self.assertTrue('PARAM_BOOLEAN=1' in result['stdout'])
self.assertTrue('PARAM_LIST=a,b,c' in result['stdout'])
self.assertTrue('PARAM_OBJECT={"foo": "bar"}' in result['stdout'])
action_parameters = {
'param_string': 'test string',
'param_integer': 1,
'param_float': 2.55,
'param_boolean': False,
'param_list': ['a', 'b', 'c'],
'param_object': {
'foo': 'bar'
}
}
runner = self._get_runner(action_db=action_db, entry_point=entry_point)
runner.pre_run()
status, result, _ = runner.run(action_parameters=action_parameters)
runner.post_run(status, result)
self.assertEqual(status, action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertTrue('PARAM_BOOLEAN=0' in result['stdout'])
action_parameters = {
'param_string': '',
'param_integer': None,
'param_float': None,
}
runner = self._get_runner(action_db=action_db, entry_point=entry_point)
runner.pre_run()
status, result, _ = runner.run(action_parameters=action_parameters)
runner.post_run(status, result)
self.assertEqual(status, action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertTrue('PARAM_STRING=\n' in result['stdout'])
self.assertTrue('PARAM_INTEGER=\n' in result['stdout'])
self.assertTrue('PARAM_FLOAT=\n' in result['stdout'])
# End result should be the same when streaming is enabled
cfg.CONF.set_override(name='stream_output',
group='actionrunner',
override=True)
# Verify initial state
output_dbs = ActionExecutionOutput.get_all()
self.assertEqual(len(output_dbs), 0)
action_parameters = {
'param_string': 'test string',
'param_integer': 1,
'param_float': 2.55,
'param_boolean': True,
'param_list': ['a', 'b', 'c'],
'param_object': {
'foo': 'bar'
}
}
runner = self._get_runner(action_db=action_db, entry_point=entry_point)
runner.pre_run()
status, result, _ = runner.run(action_parameters=action_parameters)
runner.post_run(status, result)
self.assertEqual(status, action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertTrue('PARAM_STRING=test string' in result['stdout'])
self.assertTrue('PARAM_INTEGER=1' in result['stdout'])
self.assertTrue('PARAM_FLOAT=2.55' in result['stdout'])
self.assertTrue('PARAM_BOOLEAN=1' in result['stdout'])
self.assertTrue('PARAM_LIST=a,b,c' in result['stdout'])
self.assertTrue('PARAM_OBJECT={"foo": "bar"}' in result['stdout'])
output_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(output_dbs), 6)
self.assertEqual(output_dbs[0].data, 'PARAM_STRING=test string\n')
self.assertEqual(output_dbs[5].data, 'PARAM_OBJECT={"foo": "bar"}\n')
output_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(output_dbs), 0)
def test_action_stdout_and_stderr_is_stored_in_the_db(
self, mock_spawn, mock_popen):
# Feature is enabled
cfg.CONF.set_override(name='stream_output',
group='actionrunner',
override=True)
# Note: We need to mock spawn function so we can test everything in single event loop
# iteration
mock_spawn.side_effect = blocking_eventlet_spawn
# No output to stdout and no result (implicit None)
mock_stdout = [
'stdout line 1\n', 'stdout line 2\n', 'stdout line 3\n',
'stdout line 4\n'
]
mock_stderr = ['stderr line 1\n', 'stderr line 2\n', 'stderr line 3\n']
mock_process = mock.Mock()
mock_process.returncode = 0
mock_popen.return_value = mock_process
mock_process.stdout.closed = False
mock_process.stderr.closed = False
mock_process.stdout.readline = make_mock_stream_readline(
mock_process.stdout, mock_stdout, stop_counter=4)
mock_process.stderr.readline = make_mock_stream_readline(
mock_process.stderr, mock_stderr, stop_counter=3)
models = self.fixtures_loader.load_models(
fixtures_pack='generic',
fixtures_dict={'actions': ['local_script_with_params.yaml']})
action_db = models['actions']['local_script_with_params.yaml']
entry_point = os.path.join(
get_fixtures_base_path(),
'generic/actions/local_script_with_params.sh')
action_parameters = {
'param_string': 'test string',
'param_integer': 1,
'param_float': 2.55,
'param_boolean': True,
'param_list': ['a', 'b', 'c'],
'param_object': {
'foo': 'bar'
}
}
runner = self._get_runner(action_db=action_db, entry_point=entry_point)
runner.pre_run()
status, result, _ = runner.run(action_parameters=action_parameters)
runner.post_run(status, result)
self.assertEqual(
result['stdout'],
'stdout line 1\nstdout line 2\nstdout line 3\nstdout line 4')
self.assertEqual(result['stderr'],
'stderr line 1\nstderr line 2\nstderr line 3')
self.assertEqual(result['return_code'], 0)
# Verify stdout and stderr lines have been correctly stored in the db
output_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(output_dbs), 4)
self.assertEqual(output_dbs[0].data, mock_stdout[0])
self.assertEqual(output_dbs[1].data, mock_stdout[1])
self.assertEqual(output_dbs[2].data, mock_stdout[2])
self.assertEqual(output_dbs[3].data, mock_stdout[3])
output_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(output_dbs), 3)
self.assertEqual(output_dbs[0].data, mock_stderr[0])
self.assertEqual(output_dbs[1].data, mock_stderr[1])
self.assertEqual(output_dbs[2].data, mock_stderr[2])
def test_script_with_parameters_parameter_serialization(self):
models = self.fixtures_loader.load_models(
fixtures_pack='generic', fixtures_dict={'actions': ['local_script_with_params.yaml']})
action_db = models['actions']['local_script_with_params.yaml']
entry_point = os.path.join(get_fixtures_base_path(),
'generic/actions/local_script_with_params.sh')
action_parameters = {
'param_string': 'test string',
'param_integer': 1,
'param_float': 2.55,
'param_boolean': True,
'param_list': ['a', 'b', 'c'],
'param_object': {'foo': 'bar'}
}
runner = self._get_runner(action_db=action_db, entry_point=entry_point)
runner.pre_run()
status, result, _ = runner.run(action_parameters=action_parameters)
runner.post_run(status, result)
self.assertEqual(status, action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertTrue('PARAM_STRING=test string' in result['stdout'])
self.assertTrue('PARAM_INTEGER=1' in result['stdout'])
self.assertTrue('PARAM_FLOAT=2.55' in result['stdout'])
self.assertTrue('PARAM_BOOLEAN=1' in result['stdout'])
self.assertTrue('PARAM_LIST=a,b,c' in result['stdout'])
self.assertTrue('PARAM_OBJECT={"foo": "bar"}' in result['stdout'])
action_parameters = {
'param_string': 'test string',
'param_integer': 1,
'param_float': 2.55,
'param_boolean': False,
'param_list': ['a', 'b', 'c'],
'param_object': {'foo': 'bar'}
}
runner = self._get_runner(action_db=action_db, entry_point=entry_point)
runner.pre_run()
status, result, _ = runner.run(action_parameters=action_parameters)
runner.post_run(status, result)
self.assertEqual(status, action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertTrue('PARAM_BOOLEAN=0' in result['stdout'])
action_parameters = {
'param_string': '',
'param_integer': None,
'param_float': None,
}
runner = self._get_runner(action_db=action_db, entry_point=entry_point)
runner.pre_run()
status, result, _ = runner.run(action_parameters=action_parameters)
runner.post_run(status, result)
self.assertEqual(status, action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertTrue('PARAM_STRING=\n' in result['stdout'])
self.assertTrue('PARAM_INTEGER=\n' in result['stdout'])
self.assertTrue('PARAM_FLOAT=\n' in result['stdout'])
# End result should be the same when streaming is enabled
cfg.CONF.set_override(name='stream_output', group='actionrunner', override=True)
# Verify initial state
output_dbs = ActionExecutionOutput.get_all()
self.assertEqual(len(output_dbs), 0)
action_parameters = {
'param_string': 'test string',
'param_integer': 1,
'param_float': 2.55,
'param_boolean': True,
'param_list': ['a', 'b', 'c'],
'param_object': {'foo': 'bar'}
}
runner = self._get_runner(action_db=action_db, entry_point=entry_point)
runner.pre_run()
status, result, _ = runner.run(action_parameters=action_parameters)
runner.post_run(status, result)
self.assertEqual(status, action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertTrue('PARAM_STRING=test string' in result['stdout'])
self.assertTrue('PARAM_INTEGER=1' in result['stdout'])
self.assertTrue('PARAM_FLOAT=2.55' in result['stdout'])
self.assertTrue('PARAM_BOOLEAN=1' in result['stdout'])
self.assertTrue('PARAM_LIST=a,b,c' in result['stdout'])
self.assertTrue('PARAM_OBJECT={"foo": "bar"}' in result['stdout'])
output_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(output_dbs), 6)
self.assertEqual(output_dbs[0].data, 'PARAM_STRING=test string\n')
self.assertEqual(output_dbs[5].data, 'PARAM_OBJECT={"foo": "bar"}\n')
output_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(output_dbs), 0)
def test_action_stdout_and_stderr_is_stored_in_the_db_short_running_action(
self, mock_spawn, mock_popen):
# Verify that we correctly retrieve all the output and wait for stdout and stderr reading
# threads for short running actions.
models = self.fixtures_loader.load_models(
fixtures_pack='generic', fixtures_dict={'actions': ['local.yaml']})
action_db = models['actions']['local.yaml']
# Feature is enabled
cfg.CONF.set_override(name='stream_output',
group='actionrunner',
override=True)
# Note: We need to mock spawn function so we can test everything in single event loop
# iteration
mock_spawn.side_effect = blocking_eventlet_spawn
# No output to stdout and no result (implicit None)
mock_stdout = ['stdout line 1\n', 'stdout line 2\n']
mock_stderr = ['stderr line 1\n', 'stderr line 2\n']
# We add a sleep to simulate action process exiting before we finish reading data from
mock_process = mock.Mock()
mock_process.returncode = 0
mock_popen.return_value = mock_process
mock_process.stdout.closed = False
mock_process.stderr.closed = False
mock_process.stdout.readline = make_mock_stream_readline(
mock_process.stdout, mock_stdout, stop_counter=2, sleep_delay=1)
mock_process.stderr.readline = make_mock_stream_readline(
mock_process.stderr, mock_stderr, stop_counter=2)
for index in range(1, 4):
mock_process.stdout.closed = False
mock_process.stderr.closed = False
mock_process.stdout.counter = 0
mock_process.stderr.counter = 0
runner = self._get_runner(action_db, cmd='echo "foobar"')
runner.pre_run()
status, result, _ = runner.run({})
self.assertEquals(status,
action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertEqual(result['stdout'], 'stdout line 1\nstdout line 2')
self.assertEqual(result['stderr'], 'stderr line 1\nstderr line 2')
self.assertEqual(result['return_code'], 0)
# Verify stdout and stderr lines have been correctly stored in the db
output_dbs = ActionExecutionOutput.query(output_type='stdout')
if index == 1:
db_index_1 = 0
db_index_2 = 1
elif index == 2:
db_index_1 = 2
db_index_2 = 3
elif index == 3:
db_index_1 = 4
db_index_2 = 5
elif index == 4:
db_index_1 = 6
db_index_2 = 7
self.assertEqual(len(output_dbs), (index * 2))
self.assertEqual(output_dbs[db_index_1].data, mock_stdout[0])
self.assertEqual(output_dbs[db_index_2].data, mock_stdout[1])
output_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(output_dbs), (index * 2))
self.assertEqual(output_dbs[db_index_1].data, mock_stderr[0])
self.assertEqual(output_dbs[db_index_2].data, mock_stderr[1])
def purge_executions(logger, timestamp, action_ref=None, purge_incomplete=False):
"""
Purge action executions and corresponding live action, execution output objects.
:param timestamp: Exections older than this timestamp will be deleted.
:type timestamp: ``datetime.datetime
:param action_ref: Only delete executions for the provided actions.
:type action_ref: ``str``
:param purge_incomplete: True to also delete executions which are not in a done state.
:type purge_incomplete: ``bool``
"""
if not timestamp:
raise ValueError('Specify a valid timestamp to purge.')
logger.info('Purging executions older than timestamp: %s' %
timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ'))
filters = {}
if purge_incomplete:
filters['start_timestamp__lt'] = timestamp
else:
filters['end_timestamp__lt'] = timestamp
filters['start_timestamp__lt'] = timestamp
filters['status'] = {'$in': DONE_STATES}
exec_filters = copy.copy(filters)
if action_ref:
exec_filters['action__ref'] = action_ref
liveaction_filters = copy.deepcopy(filters)
if action_ref:
liveaction_filters['action'] = action_ref
to_delete_execution_dbs = []
# 1. Delete ActionExecutionDB objects
try:
# Note: We call list() on the query set object because it's lazyily evaluated otherwise
to_delete_execution_dbs = list(ActionExecution.query(only_fields=['id'],
no_dereference=True,
**exec_filters))
deleted_count = ActionExecution.delete_by_query(**exec_filters)
except InvalidQueryError as e:
msg = ('Bad query (%s) used to delete execution instances: %s'
'Please contact support.' % (exec_filters, six.text_type(e)))
raise InvalidQueryError(msg)
except:
logger.exception('Deletion of execution models failed for query with filters: %s.',
exec_filters)
else:
logger.info('Deleted %s action execution objects' % (deleted_count))
# 2. Delete LiveActionDB objects
try:
deleted_count = LiveAction.delete_by_query(**liveaction_filters)
except InvalidQueryError as e:
msg = ('Bad query (%s) used to delete liveaction instances: %s'
'Please contact support.' % (liveaction_filters, six.text_type(e)))
raise InvalidQueryError(msg)
except:
logger.exception('Deletion of liveaction models failed for query with filters: %s.',
liveaction_filters)
else:
logger.info('Deleted %s liveaction objects' % (deleted_count))
# 3. Delete ActionExecutionOutputDB objects
to_delete_exection_ids = [str(execution_db.id) for execution_db in to_delete_execution_dbs]
output_dbs_filters = {}
output_dbs_filters['execution_id'] = {'$in': to_delete_exection_ids}
try:
deleted_count = ActionExecutionOutput.delete_by_query(**output_dbs_filters)
except InvalidQueryError as e:
msg = ('Bad query (%s) used to delete execution output instances: %s'
'Please contact support.' % (output_dbs_filters, six.text_type(e)))
raise InvalidQueryError(msg)
except:
logger.exception('Deletion of execution output models failed for query with filters: %s.',
output_dbs_filters)
else:
logger.info('Deleted %s execution output objects' % (deleted_count))
zombie_execution_instances = len(ActionExecution.query(only_fields=['id'],
no_dereference=True,
**exec_filters))
zombie_liveaction_instances = len(LiveAction.query(only_fields=['id'],
no_dereference=True,
**liveaction_filters))
if (zombie_execution_instances > 0) or (zombie_liveaction_instances > 0):
logger.error('Zombie execution instances left: %d.', zombie_execution_instances)
logger.error('Zombie liveaction instances left: %s.', zombie_liveaction_instances)
# Print stats
logger.info('All execution models older than timestamp %s were deleted.', timestamp)
def test_action_stdout_and_stderr_is_stored_in_the_db(self, mock_spawn, mock_popen):
# Feature is enabled
cfg.CONF.set_override(name='stream_output', group='actionrunner', override=True)
# Note: We need to mock spawn function so we can test everything in single event loop
# iteration
mock_spawn.side_effect = blocking_eventlet_spawn
# No output to stdout and no result (implicit None)
mock_stdout = [
'stdout line 1\n',
'stdout line 2\n',
'stdout line 3\n',
'stdout line 4\n'
]
mock_stderr = [
'stderr line 1\n',
'stderr line 2\n',
'stderr line 3\n'
]
mock_process = mock.Mock()
mock_process.returncode = 0
mock_popen.return_value = mock_process
mock_process.stdout.closed = False
mock_process.stderr.closed = False
mock_process.stdout.readline = make_mock_stream_readline(mock_process.stdout, mock_stdout,
stop_counter=4)
mock_process.stderr.readline = make_mock_stream_readline(mock_process.stderr, mock_stderr,
stop_counter=3)
models = self.fixtures_loader.load_models(
fixtures_pack='generic', fixtures_dict={'actions': ['local_script_with_params.yaml']})
action_db = models['actions']['local_script_with_params.yaml']
entry_point = os.path.join(get_fixtures_base_path(),
'generic/actions/local_script_with_params.sh')
action_parameters = {
'param_string': 'test string',
'param_integer': 1,
'param_float': 2.55,
'param_boolean': True,
'param_list': ['a', 'b', 'c'],
'param_object': {'foo': 'bar'}
}
runner = self._get_runner(action_db=action_db, entry_point=entry_point)
runner.pre_run()
status, result, _ = runner.run(action_parameters=action_parameters)
runner.post_run(status, result)
self.assertEqual(result['stdout'],
'stdout line 1\nstdout line 2\nstdout line 3\nstdout line 4')
self.assertEqual(result['stderr'], 'stderr line 1\nstderr line 2\nstderr line 3')
self.assertEqual(result['return_code'], 0)
# Verify stdout and stderr lines have been correctly stored in the db
output_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(output_dbs), 4)
self.assertEqual(output_dbs[0].data, mock_stdout[0])
self.assertEqual(output_dbs[1].data, mock_stdout[1])
self.assertEqual(output_dbs[2].data, mock_stdout[2])
self.assertEqual(output_dbs[3].data, mock_stdout[3])
output_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(output_dbs), 3)
self.assertEqual(output_dbs[0].data, mock_stderr[0])
self.assertEqual(output_dbs[1].data, mock_stderr[1])
self.assertEqual(output_dbs[2].data, mock_stderr[2])
def test_garbage_collection(self):
now = date_utils.get_datetime_utc_now()
status = action_constants.LIVEACTION_STATUS_SUCCEEDED
# Insert come mock ActionExecutionDB objects with start_timestamp < TTL defined in the
# config
old_executions_count = 15
ttl_days = 30 # > 20
timestamp = (now - datetime.timedelta(days=ttl_days))
for index in range(0, old_executions_count):
action_execution_db = ActionExecutionDB(start_timestamp=timestamp,
end_timestamp=timestamp,
status=status,
action={'ref': 'core.local'},
runner={'name': 'local-shell-cmd'},
liveaction={'ref': 'foo'})
ActionExecution.add_or_update(action_execution_db)
stdout_db = ActionExecutionOutputDB(execution_id=str(action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stdout',
data='stdout')
ActionExecutionOutput.add_or_update(stdout_db)
stderr_db = ActionExecutionOutputDB(execution_id=str(action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stderr',
data='stderr')
ActionExecutionOutput.add_or_update(stderr_db)
# Insert come mock ActionExecutionDB objects with start_timestamp > TTL defined in the
# config
new_executions_count = 5
ttl_days = 2 # < 20
timestamp = (now - datetime.timedelta(days=ttl_days))
for index in range(0, new_executions_count):
action_execution_db = ActionExecutionDB(start_timestamp=timestamp,
end_timestamp=timestamp,
status=status,
action={'ref': 'core.local'},
runner={'name': 'local-shell-cmd'},
liveaction={'ref': 'foo'})
ActionExecution.add_or_update(action_execution_db)
stdout_db = ActionExecutionOutputDB(execution_id=str(action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stdout',
data='stdout')
ActionExecutionOutput.add_or_update(stdout_db)
stderr_db = ActionExecutionOutputDB(execution_id=str(action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stderr',
data='stderr')
ActionExecutionOutput.add_or_update(stderr_db)
# Insert some mock output objects where start_timestamp > action_executions_output_ttl
new_output_count = 5
ttl_days = 15 # > 10 and < 20
timestamp = (now - datetime.timedelta(days=ttl_days))
for index in range(0, new_output_count):
action_execution_db = ActionExecutionDB(start_timestamp=timestamp,
end_timestamp=timestamp,
status=status,
action={'ref': 'core.local'},
runner={'name': 'local-shell-cmd'},
liveaction={'ref': 'foo'})
ActionExecution.add_or_update(action_execution_db)
stdout_db = ActionExecutionOutputDB(execution_id=str(action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stdout',
data='stdout')
ActionExecutionOutput.add_or_update(stdout_db)
stderr_db = ActionExecutionOutputDB(execution_id=str(action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stderr',
data='stderr')
ActionExecutionOutput.add_or_update(stderr_db)
execs = ActionExecution.get_all()
self.assertEqual(len(execs),
(old_executions_count + new_executions_count + new_output_count))
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs),
(old_executions_count + new_executions_count + new_output_count))
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs),
(old_executions_count + new_executions_count + new_output_count))
# Start garbage collector
process = self._start_garbage_collector()
# Give it some time to perform garbage collection and kill it
eventlet.sleep(15)
process.send_signal(signal.SIGKILL)
self.remove_process(process=process)
# Old executions and corresponding objects should have been garbage collected
execs = ActionExecution.get_all()
self.assertEqual(len(execs), (new_executions_count + new_output_count))
# Collection for output objects older than 10 days is also enabled, so those objects
# should be deleted as well
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), (new_executions_count))
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), (new_executions_count))
def test_get_output_running_execution(self):
# Retrieve lister instance to avoid race with listener connection not being established
# early enough for tests to pass.
# NOTE: This only affects tests where listeners are not pre-initialized.
listener = get_listener(name='execution_output')
eventlet.sleep(1.0)
# Test the execution output API endpoint for execution which is running (blocking)
status = action_constants.LIVEACTION_STATUS_RUNNING
timestamp = date_utils.get_datetime_utc_now()
action_execution_db = ActionExecutionDB(
start_timestamp=timestamp,
end_timestamp=timestamp,
status=status,
action={'ref': 'core.local'},
runner={'name': 'local-shell-cmd'},
liveaction={'ref': 'foo'})
action_execution_db = ActionExecution.add_or_update(
action_execution_db)
output_params = dict(execution_id=str(action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stdout',
data='stdout before start\n')
# Insert mock output object
output_db = ActionExecutionOutputDB(**output_params)
ActionExecutionOutput.add_or_update(output_db, publish=False)
def insert_mock_data():
output_params['data'] = 'stdout mid 1\n'
output_db = ActionExecutionOutputDB(**output_params)
ActionExecutionOutput.add_or_update(output_db)
# Since the API endpoint is blocking (connection is kept open until action finishes), we
# spawn an eventlet which eventually finishes the action.
def publish_action_finished(action_execution_db):
# Insert mock output object
output_params['data'] = 'stdout pre finish 1\n'
output_db = ActionExecutionOutputDB(**output_params)
ActionExecutionOutput.add_or_update(output_db)
eventlet.sleep(1.0)
# Transition execution to completed state so the connection closes
action_execution_db.status = action_constants.LIVEACTION_STATUS_SUCCEEDED
action_execution_db = ActionExecution.add_or_update(
action_execution_db)
eventlet.spawn_after(0.2, insert_mock_data)
eventlet.spawn_after(1.5, publish_action_finished, action_execution_db)
# Retrieve data while execution is running - endpoint return new data once it's available
# and block until the execution finishes
resp = self.app.get('/v1/executions/%s/output' %
(str(action_execution_db.id)),
expect_errors=False)
self.assertEqual(resp.status_int, 200)
events = self._parse_response(resp.text)
self.assertEqual(len(events), 4)
self.assertEqual(events[0][1]['data'], 'stdout before start\n')
self.assertEqual(events[1][1]['data'], 'stdout mid 1\n')
self.assertEqual(events[2][1]['data'], 'stdout pre finish 1\n')
self.assertEqual(events[3][0], 'EOF')
# Once the execution is in completed state, existing output should be returned immediately
resp = self.app.get('/v1/executions/%s/output' %
(str(action_execution_db.id)),
expect_errors=False)
self.assertEqual(resp.status_int, 200)
events = self._parse_response(resp.text)
self.assertEqual(len(events), 4)
self.assertEqual(events[0][1]['data'], 'stdout before start\n')
self.assertEqual(events[1][1]['data'], 'stdout mid 1\n')
self.assertEqual(events[2][1]['data'], 'stdout pre finish 1\n')
self.assertEqual(events[3][0], 'EOF')
listener.shutdown()
def test_action_stdout_and_stderr_is_not_stored_in_db_by_default(
self, mock_spawn, mock_popen):
# Feature should be disabled by default
values = {'delimiter': ACTION_OUTPUT_RESULT_DELIMITER}
# Note: We need to mock spawn function so we can test everything in single event loop
# iteration
mock_spawn.side_effect = blocking_eventlet_spawn
# No output to stdout and no result (implicit None)
mock_stdout = [
'pre result line 1\n',
'%(delimiter)sTrue%(delimiter)s' % values, 'post result line 1'
]
mock_stderr = ['stderr line 1\n', 'stderr line 2\n', 'stderr line 3\n']
mock_process = mock.Mock()
mock_process.returncode = 0
mock_popen.return_value = mock_process
mock_process.stdout.closed = False
mock_process.stderr.closed = False
mock_process.stdout.readline = make_mock_stream_readline(
mock_process.stdout, mock_stdout, stop_counter=3)
mock_process.stderr.readline = make_mock_stream_readline(
mock_process.stderr, mock_stderr, stop_counter=3)
runner = self._get_mock_runner_obj()
runner.entry_point = PASCAL_ROW_ACTION_PATH
runner.pre_run()
(_, output, _) = runner.run({'row_index': 4})
self.assertEqual(output['stdout'],
'pre result line 1\npost result line 1')
self.assertEqual(output['stderr'],
'stderr line 1\nstderr line 2\nstderr line 3\n')
self.assertEqual(output['result'], 'True')
self.assertEqual(output['exit_code'], 0)
output_dbs = ActionExecutionOutput.get_all()
self.assertEqual(len(output_dbs), 0)
# False is a default behavior so end result should be the same
cfg.CONF.set_override(name='stream_output',
group='actionrunner',
override=False)
mock_process = mock.Mock()
mock_process.returncode = 0
mock_popen.return_value = mock_process
mock_process.stdout.closed = False
mock_process.stderr.closed = False
mock_process.stdout.readline = make_mock_stream_readline(
mock_process.stdout, mock_stdout, stop_counter=3)
mock_process.stderr.readline = make_mock_stream_readline(
mock_process.stderr, mock_stderr, stop_counter=3)
runner.pre_run()
(_, output, _) = runner.run({'row_index': 4})
self.assertEqual(output['stdout'],
'pre result line 1\npost result line 1')
self.assertEqual(output['stderr'],
'stderr line 1\nstderr line 2\nstderr line 3\n')
self.assertEqual(output['result'], 'True')
self.assertEqual(output['exit_code'], 0)
output_dbs = ActionExecutionOutput.get_all()
self.assertEqual(len(output_dbs), 0)
def test_purge_incomplete(self):
now = date_utils.get_datetime_utc_now()
start_ts = now - timedelta(days=15)
# Write executions before cut-off threshold
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_SCHEDULED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=1)
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_RUNNING
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=1)
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_DELAYED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=1)
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_CANCELING
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=1)
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_REQUESTED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=1)
self.assertEqual(len(ActionExecution.get_all()), 5)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 5)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 5)
# Incompleted executions shouldnt be purged
purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=False)
self.assertEqual(len(ActionExecution.get_all()), 5)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 5)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 5)
purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=True)
self.assertEqual(len(ActionExecution.get_all()), 0)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 0)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 0)
def purge_executions(logger, timestamp, action_ref=None, purge_incomplete=False):
"""
Purge action executions and corresponding live action, execution output objects.
:param timestamp: Exections older than this timestamp will be deleted.
:type timestamp: ``datetime.datetime
:param action_ref: Only delete executions for the provided actions.
:type action_ref: ``str``
:param purge_incomplete: True to also delete executions which are not in a done state.
:type purge_incomplete: ``bool``
"""
if not timestamp:
raise ValueError('Specify a valid timestamp to purge.')
logger.info('Purging executions older than timestamp: %s' %
timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ'))
filters = {}
if purge_incomplete:
filters['start_timestamp__lt'] = timestamp
else:
filters['end_timestamp__lt'] = timestamp
filters['start_timestamp__lt'] = timestamp
filters['status'] = {'$in': DONE_STATES}
exec_filters = copy.copy(filters)
if action_ref:
exec_filters['action__ref'] = action_ref
liveaction_filters = copy.deepcopy(filters)
if action_ref:
liveaction_filters['action'] = action_ref
to_delete_execution_dbs = []
# 1. Delete ActionExecutionDB objects
try:
# Note: We call list() on the query set object because it's lazyily evaluated otherwise
to_delete_execution_dbs = list(ActionExecution.query(only_fields=['id'],
no_dereference=True,
**exec_filters))
deleted_count = ActionExecution.delete_by_query(**exec_filters)
except InvalidQueryError as e:
msg = ('Bad query (%s) used to delete execution instances: %s'
'Please contact support.' % (exec_filters, six.text_type(e)))
raise InvalidQueryError(msg)
except:
logger.exception('Deletion of execution models failed for query with filters: %s.',
exec_filters)
else:
logger.info('Deleted %s action execution objects' % (deleted_count))
# 2. Delete LiveActionDB objects
try:
deleted_count = LiveAction.delete_by_query(**liveaction_filters)
except InvalidQueryError as e:
msg = ('Bad query (%s) used to delete liveaction instances: %s'
'Please contact support.' % (liveaction_filters, six.text_type(e)))
raise InvalidQueryError(msg)
except:
logger.exception('Deletion of liveaction models failed for query with filters: %s.',
liveaction_filters)
else:
logger.info('Deleted %s liveaction objects' % (deleted_count))
# 3. Delete ActionExecutionOutputDB objects
to_delete_exection_ids = [str(execution_db.id) for execution_db in to_delete_execution_dbs]
output_dbs_filters = {}
output_dbs_filters['execution_id'] = {'$in': to_delete_exection_ids}
try:
deleted_count = ActionExecutionOutput.delete_by_query(**output_dbs_filters)
except InvalidQueryError as e:
msg = ('Bad query (%s) used to delete execution output instances: %s'
'Please contact support.' % (output_dbs_filters, six.text_type(e)))
raise InvalidQueryError(msg)
except:
logger.exception('Deletion of execution output models failed for query with filters: %s.',
output_dbs_filters)
else:
logger.info('Deleted %s execution output objects' % (deleted_count))
zombie_execution_instances = len(ActionExecution.query(only_fields=['id'],
no_dereference=True,
**exec_filters))
zombie_liveaction_instances = len(LiveAction.query(only_fields=['id'],
no_dereference=True,
**liveaction_filters))
if (zombie_execution_instances > 0) or (zombie_liveaction_instances > 0):
logger.error('Zombie execution instances left: %d.', zombie_execution_instances)
logger.error('Zombie liveaction instances left: %s.', zombie_liveaction_instances)
# Print stats
logger.info('All execution models older than timestamp %s were deleted.', timestamp)
def insert_mock_data():
output_params["data"] = "stdout mid 1\n"
output_db = ActionExecutionOutputDB(**output_params)
ActionExecutionOutput.add_or_update(output_db)
def test_purge_incomplete(self):
now = date_utils.get_datetime_utc_now()
start_ts = now - timedelta(days=15)
# Write executions before cut-off threshold
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_SCHEDULED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(
exec_model['id'], count=1)
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_RUNNING
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(
exec_model['id'], count=1)
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_DELAYED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(
exec_model['id'], count=1)
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_CANCELING
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(
exec_model['id'], count=1)
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_REQUESTED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(
exec_model['id'], count=1)
self.assertEqual(len(ActionExecution.get_all()), 5)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 5)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 5)
# Incompleted executions shouldnt be purged
purge_executions(logger=LOG,
timestamp=now - timedelta(days=10),
purge_incomplete=False)
self.assertEqual(len(ActionExecution.get_all()), 5)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 5)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 5)
purge_executions(logger=LOG,
timestamp=now - timedelta(days=10),
purge_incomplete=True)
self.assertEqual(len(ActionExecution.get_all()), 0)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 0)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 0)
def test_action_stdout_and_stderr_is_stored_in_the_db_short_running_action(self, mock_spawn,
mock_popen):
# Verify that we correctly retrieve all the output and wait for stdout and stderr reading
# threads for short running actions.
models = self.fixtures_loader.load_models(
fixtures_pack='generic', fixtures_dict={'actions': ['local.yaml']})
action_db = models['actions']['local.yaml']
# Feature is enabled
cfg.CONF.set_override(name='stream_output', group='actionrunner', override=True)
# Note: We need to mock spawn function so we can test everything in single event loop
# iteration
mock_spawn.side_effect = blocking_eventlet_spawn
# No output to stdout and no result (implicit None)
mock_stdout = [
'stdout line 1\n',
'stdout line 2\n'
]
mock_stderr = [
'stderr line 1\n',
'stderr line 2\n'
]
# We add a sleep to simulate action process exiting before we finish reading data from
mock_process = mock.Mock()
mock_process.returncode = 0
mock_popen.return_value = mock_process
mock_process.stdout.closed = False
mock_process.stderr.closed = False
mock_process.stdout.readline = make_mock_stream_readline(mock_process.stdout, mock_stdout,
stop_counter=2,
sleep_delay=1)
mock_process.stderr.readline = make_mock_stream_readline(mock_process.stderr, mock_stderr,
stop_counter=2)
for index in range(1, 4):
mock_process.stdout.closed = False
mock_process.stderr.closed = False
mock_process.stdout.counter = 0
mock_process.stderr.counter = 0
runner = self._get_runner(action_db, cmd='echo "foobar"')
runner.pre_run()
status, result, _ = runner.run({})
self.assertEquals(status, action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertEqual(result['stdout'], 'stdout line 1\nstdout line 2')
self.assertEqual(result['stderr'], 'stderr line 1\nstderr line 2')
self.assertEqual(result['return_code'], 0)
# Verify stdout and stderr lines have been correctly stored in the db
output_dbs = ActionExecutionOutput.query(output_type='stdout')
if index == 1:
db_index_1 = 0
db_index_2 = 1
elif index == 2:
db_index_1 = 2
db_index_2 = 3
elif index == 3:
db_index_1 = 4
db_index_2 = 5
elif index == 4:
db_index_1 = 6
db_index_2 = 7
self.assertEqual(len(output_dbs), (index * 2))
self.assertEqual(output_dbs[db_index_1].data, mock_stdout[0])
self.assertEqual(output_dbs[db_index_2].data, mock_stdout[1])
output_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(output_dbs), (index * 2))
self.assertEqual(output_dbs[db_index_1].data, mock_stderr[0])
self.assertEqual(output_dbs[db_index_2].data, mock_stderr[1])
def test_get_output_running_execution(self):
# Retrieve lister instance to avoid race with listener connection not being established
# early enough for tests to pass.
# NOTE: This only affects tests where listeners are not pre-initialized.
listener = get_listener(name='execution_output')
eventlet.sleep(1.0)
# Test the execution output API endpoint for execution which is running (blocking)
status = action_constants.LIVEACTION_STATUS_RUNNING
timestamp = date_utils.get_datetime_utc_now()
action_execution_db = ActionExecutionDB(start_timestamp=timestamp,
end_timestamp=timestamp,
status=status,
action={'ref': 'core.local'},
runner={'name': 'local-shell-cmd'},
liveaction={'ref': 'foo'})
action_execution_db = ActionExecution.add_or_update(action_execution_db)
output_params = dict(execution_id=str(action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stdout',
data='stdout before start\n')
# Insert mock output object
output_db = ActionExecutionOutputDB(**output_params)
ActionExecutionOutput.add_or_update(output_db, publish=False)
def insert_mock_data():
output_params['data'] = 'stdout mid 1\n'
output_db = ActionExecutionOutputDB(**output_params)
ActionExecutionOutput.add_or_update(output_db)
# Since the API endpoint is blocking (connection is kept open until action finishes), we
# spawn an eventlet which eventually finishes the action.
def publish_action_finished(action_execution_db):
# Insert mock output object
output_params['data'] = 'stdout pre finish 1\n'
output_db = ActionExecutionOutputDB(**output_params)
ActionExecutionOutput.add_or_update(output_db)
eventlet.sleep(1.0)
# Transition execution to completed state so the connection closes
action_execution_db.status = action_constants.LIVEACTION_STATUS_SUCCEEDED
action_execution_db = ActionExecution.add_or_update(action_execution_db)
eventlet.spawn_after(0.2, insert_mock_data)
eventlet.spawn_after(1.5, publish_action_finished, action_execution_db)
# Retrieve data while execution is running - endpoint return new data once it's available
# and block until the execution finishes
resp = self.app.get('/v1/executions/%s/output' % (str(action_execution_db.id)),
expect_errors=False)
self.assertEqual(resp.status_int, 200)
events = self._parse_response(resp.text)
self.assertEqual(len(events), 4)
self.assertEqual(events[0][1]['data'], 'stdout before start\n')
self.assertEqual(events[1][1]['data'], 'stdout mid 1\n')
self.assertEqual(events[2][1]['data'], 'stdout pre finish 1\n')
self.assertEqual(events[3][0], 'EOF')
# Once the execution is in completed state, existing output should be returned immediately
resp = self.app.get('/v1/executions/%s/output' % (str(action_execution_db.id)),
expect_errors=False)
self.assertEqual(resp.status_int, 200)
events = self._parse_response(resp.text)
self.assertEqual(len(events), 4)
self.assertEqual(events[0][1]['data'], 'stdout before start\n')
self.assertEqual(events[1][1]['data'], 'stdout mid 1\n')
self.assertEqual(events[2][1]['data'], 'stdout pre finish 1\n')
self.assertEqual(events[3][0], 'EOF')
listener.shutdown()
def test_action_stdout_and_stderr_is_not_stored_in_db_by_default(self, mock_spawn, mock_popen):
# Feature should be disabled by default
values = {'delimiter': ACTION_OUTPUT_RESULT_DELIMITER}
# Note: We need to mock spawn function so we can test everything in single event loop
# iteration
mock_spawn.side_effect = blocking_eventlet_spawn
# No output to stdout and no result (implicit None)
mock_stdout = [
'pre result line 1\n',
'%(delimiter)sTrue%(delimiter)s' % values,
'post result line 1'
]
mock_stderr = [
'stderr line 1\n',
'stderr line 2\n',
'stderr line 3\n'
]
mock_process = mock.Mock()
mock_process.returncode = 0
mock_popen.return_value = mock_process
mock_process.stdout.closed = False
mock_process.stderr.closed = False
mock_process.stdout.readline = make_mock_stream_readline(mock_process.stdout, mock_stdout,
stop_counter=3)
mock_process.stderr.readline = make_mock_stream_readline(mock_process.stderr, mock_stderr,
stop_counter=3)
runner = self._get_mock_runner_obj()
runner.entry_point = PASCAL_ROW_ACTION_PATH
runner.pre_run()
(_, output, _) = runner.run({'row_index': 4})
self.assertEqual(output['stdout'], 'pre result line 1\npost result line 1')
self.assertEqual(output['stderr'], 'stderr line 1\nstderr line 2\nstderr line 3\n')
self.assertEqual(output['result'], 'True')
self.assertEqual(output['exit_code'], 0)
output_dbs = ActionExecutionOutput.get_all()
self.assertEqual(len(output_dbs), 0)
# False is a default behavior so end result should be the same
cfg.CONF.set_override(name='stream_output', group='actionrunner', override=False)
mock_process = mock.Mock()
mock_process.returncode = 0
mock_popen.return_value = mock_process
mock_process.stdout.closed = False
mock_process.stderr.closed = False
mock_process.stdout.readline = make_mock_stream_readline(mock_process.stdout, mock_stdout,
stop_counter=3)
mock_process.stderr.readline = make_mock_stream_readline(mock_process.stderr, mock_stderr,
stop_counter=3)
runner.pre_run()
(_, output, _) = runner.run({'row_index': 4})
self.assertEqual(output['stdout'], 'pre result line 1\npost result line 1')
self.assertEqual(output['stderr'], 'stderr line 1\nstderr line 2\nstderr line 3\n')
self.assertEqual(output['result'], 'True')
self.assertEqual(output['exit_code'], 0)
output_dbs = ActionExecutionOutput.get_all()
self.assertEqual(len(output_dbs), 0)
def insert_mock_data():
output_params['data'] = 'stdout mid 1\n'
output_db = ActionExecutionOutputDB(**output_params)
ActionExecutionOutput.add_or_update(output_db)
def insert_mock_data():
output_params['data'] = 'stdout mid 1\n'
output_db = ActionExecutionOutputDB(**output_params)
ActionExecutionOutput.add_or_update(output_db)
def test_garbage_collection(self):
now = date_utils.get_datetime_utc_now()
status = action_constants.LIVEACTION_STATUS_SUCCEEDED
# Insert come mock ActionExecutionDB objects with start_timestamp < TTL defined in the
# config
old_executions_count = 15
ttl_days = 30 # > 20
timestamp = (now - datetime.timedelta(days=ttl_days))
for index in range(0, old_executions_count):
action_execution_db = ActionExecutionDB(
start_timestamp=timestamp,
end_timestamp=timestamp,
status=status,
action={'ref': 'core.local'},
runner={'name': 'local-shell-cmd'},
liveaction={'ref': 'foo'})
ActionExecution.add_or_update(action_execution_db)
stdout_db = ActionExecutionOutputDB(execution_id=str(
action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stdout',
data='stdout')
ActionExecutionOutput.add_or_update(stdout_db)
stderr_db = ActionExecutionOutputDB(execution_id=str(
action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stderr',
data='stderr')
ActionExecutionOutput.add_or_update(stderr_db)
# Insert come mock ActionExecutionDB objects with start_timestamp > TTL defined in the
# config
new_executions_count = 5
ttl_days = 2 # < 20
timestamp = (now - datetime.timedelta(days=ttl_days))
for index in range(0, new_executions_count):
action_execution_db = ActionExecutionDB(
start_timestamp=timestamp,
end_timestamp=timestamp,
status=status,
action={'ref': 'core.local'},
runner={'name': 'local-shell-cmd'},
liveaction={'ref': 'foo'})
ActionExecution.add_or_update(action_execution_db)
stdout_db = ActionExecutionOutputDB(execution_id=str(
action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stdout',
data='stdout')
ActionExecutionOutput.add_or_update(stdout_db)
stderr_db = ActionExecutionOutputDB(execution_id=str(
action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stderr',
data='stderr')
ActionExecutionOutput.add_or_update(stderr_db)
# Insert some mock output objects where start_timestamp > action_executions_output_ttl
new_output_count = 5
ttl_days = 15 # > 10 and < 20
timestamp = (now - datetime.timedelta(days=ttl_days))
for index in range(0, new_output_count):
action_execution_db = ActionExecutionDB(
start_timestamp=timestamp,
end_timestamp=timestamp,
status=status,
action={'ref': 'core.local'},
runner={'name': 'local-shell-cmd'},
liveaction={'ref': 'foo'})
ActionExecution.add_or_update(action_execution_db)
stdout_db = ActionExecutionOutputDB(execution_id=str(
action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stdout',
data='stdout')
ActionExecutionOutput.add_or_update(stdout_db)
stderr_db = ActionExecutionOutputDB(execution_id=str(
action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stderr',
data='stderr')
ActionExecutionOutput.add_or_update(stderr_db)
execs = ActionExecution.get_all()
self.assertEqual(
len(execs),
(old_executions_count + new_executions_count + new_output_count))
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(
len(stdout_dbs),
(old_executions_count + new_executions_count + new_output_count))
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(
len(stderr_dbs),
(old_executions_count + new_executions_count + new_output_count))
# Start garbage collector
process = self._start_garbage_collector()
# Give it some time to perform garbage collection and kill it
eventlet.sleep(15)
process.send_signal(signal.SIGKILL)
self.remove_process(process=process)
# Old executions and corresponding objects should have been garbage collected
execs = ActionExecution.get_all()
self.assertEqual(len(execs), (new_executions_count + new_output_count))
# Collection for output objects older than 10 days is also enabled, so those objects
# should be deleted as well
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), (new_executions_count))
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), (new_executions_count))
