def test_liveaction_gets_deleted(self):
now = date_utils.get_datetime_utc_now()
start_ts = now - timedelta(days=15)
end_ts = now - timedelta(days=14)
liveaction_model = copy.deepcopy(self.models['liveactions']['liveaction4.yaml'])
liveaction_model['start_timestamp'] = start_ts
liveaction_model['end_timestamp'] = end_ts
liveaction_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
liveaction = LiveAction.add_or_update(liveaction_model)
# Write one execution before cut-off threshold
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['end_timestamp'] = end_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
exec_model['liveaction']['id'] = str(liveaction.id)
ActionExecution.add_or_update(exec_model)
liveactions = LiveAction.get_all()
executions = ActionExecution.get_all()
self.assertEqual(len(liveactions), 1)
self.assertEqual(len(executions), 1)
purge_executions(logger=LOG, timestamp=now - timedelta(days=10))
liveactions = LiveAction.get_all()
executions = ActionExecution.get_all()
self.assertEqual(len(executions), 0)
self.assertEqual(len(liveactions), 0)
def test_crud_complete(self):
# Create the DB record.
obj = ActionExecutionAPI(**copy.deepcopy(self.fake_history_workflow))
ActionExecution.add_or_update(ActionExecutionAPI.to_model(obj))
model = ActionExecution.get_by_id(obj.id)
self.assertEqual(str(model.id), obj.id)
self.assertDictEqual(model.trigger, self.fake_history_workflow['trigger'])
self.assertDictEqual(model.trigger_type, self.fake_history_workflow['trigger_type'])
self.assertDictEqual(model.trigger_instance, self.fake_history_workflow['trigger_instance'])
self.assertDictEqual(model.rule, self.fake_history_workflow['rule'])
self.assertDictEqual(model.action, self.fake_history_workflow['action'])
self.assertDictEqual(model.runner, self.fake_history_workflow['runner'])
doc = copy.deepcopy(self.fake_history_workflow['liveaction'])
doc['start_timestamp'] = doc['start_timestamp']
doc['end_timestamp'] = doc['end_timestamp']
self.assertDictEqual(model.liveaction, doc)
self.assertIsNone(getattr(model, 'parent', None))
self.assertListEqual(model.children, self.fake_history_workflow['children'])
# Update the DB record.
children = [str(bson.ObjectId()), str(bson.ObjectId())]
model.children = children
ActionExecution.add_or_update(model)
model = ActionExecution.get_by_id(obj.id)
self.assertListEqual(model.children, children)
# Delete the DB record.
ActionExecution.delete(model)
self.assertRaises(ValueError, ActionExecution.get_by_id, obj.id)
def test_no_timestamp_doesnt_delete_things(self):
now = date_utils.get_datetime_utc_now()
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=15)
exec_model['end_timestamp'] = now - timedelta(days=14)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=3)
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 3)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 3)
expected_msg = 'Specify a valid timestamp'
self.assertRaisesRegexp(ValueError, expected_msg, purge_executions,
logger=LOG, timestamp=None)
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 3)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 3)
def test_crud_partial(self):
# Create the DB record.
obj = ActionExecutionAPI(
**copy.deepcopy(self.fake_history_subtasks[0]))
ActionExecution.add_or_update(ActionExecutionAPI.to_model(obj))
model = ActionExecution.get_by_id(obj.id)
self.assertEqual(str(model.id), obj.id)
self.assertDictEqual(model.trigger, {})
self.assertDictEqual(model.trigger_type, {})
self.assertDictEqual(model.trigger_instance, {})
self.assertDictEqual(model.rule, {})
self.assertDictEqual(model.action,
self.fake_history_subtasks[0]['action'])
self.assertDictEqual(model.runner,
self.fake_history_subtasks[0]['runner'])
doc = copy.deepcopy(self.fake_history_subtasks[0]['liveaction'])
doc['start_timestamp'] = doc['start_timestamp']
doc['end_timestamp'] = doc['end_timestamp']
self.assertDictEqual(model.liveaction, doc)
self.assertEqual(model.parent, self.fake_history_subtasks[0]['parent'])
self.assertListEqual(model.children, [])
# Update the DB record.
children = [str(bson.ObjectId()), str(bson.ObjectId())]
model.children = children
ActionExecution.add_or_update(model)
model = ActionExecution.get_by_id(obj.id)
self.assertListEqual(model.children, children)
# Delete the DB record.
ActionExecution.delete(model)
self.assertRaises(StackStormDBObjectNotFoundError,
ActionExecution.get_by_id, obj.id)
def test_no_timestamp_doesnt_delete_things(self):
now = date_utils.get_datetime_utc_now()
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=15)
exec_model['end_timestamp'] = now - timedelta(days=14)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(
exec_model['id'], count=3)
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 3)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 3)
expected_msg = 'Specify a valid timestamp'
self.assertRaisesRegexp(ValueError,
expected_msg,
purge_executions,
logger=LOG,
timestamp=None)
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 3)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 3)
def test_liveaction_gets_deleted(self):
now = date_utils.get_datetime_utc_now()
start_ts = now - timedelta(days=15)
end_ts = now - timedelta(days=14)
liveaction_model = copy.deepcopy(
self.models['liveactions']['liveaction4.yaml'])
liveaction_model['start_timestamp'] = start_ts
liveaction_model['end_timestamp'] = end_ts
liveaction_model[
'status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
liveaction = LiveAction.add_or_update(liveaction_model)
# Write one execution before cut-off threshold
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['end_timestamp'] = end_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
exec_model['liveaction']['id'] = str(liveaction.id)
ActionExecution.add_or_update(exec_model)
liveactions = LiveAction.get_all()
executions = ActionExecution.get_all()
self.assertEqual(len(liveactions), 1)
self.assertEqual(len(executions), 1)
purge_executions(logger=LOG, timestamp=now - timedelta(days=10))
liveactions = LiveAction.get_all()
executions = ActionExecution.get_all()
self.assertEqual(len(executions), 0)
self.assertEqual(len(liveactions), 0)
def test_crud_partial(self):
# Create the DB record.
obj = ActionExecutionAPI(**copy.deepcopy(self.fake_history_subtasks[0]))
ActionExecution.add_or_update(ActionExecutionAPI.to_model(obj))
model = ActionExecution.get_by_id(obj.id)
self.assertEqual(str(model.id), obj.id)
self.assertDictEqual(model.trigger, {})
self.assertDictEqual(model.trigger_type, {})
self.assertDictEqual(model.trigger_instance, {})
self.assertDictEqual(model.rule, {})
self.assertDictEqual(model.action, self.fake_history_subtasks[0]['action'])
self.assertDictEqual(model.runner, self.fake_history_subtasks[0]['runner'])
doc = copy.deepcopy(self.fake_history_subtasks[0]['liveaction'])
doc['start_timestamp'] = doc['start_timestamp']
doc['end_timestamp'] = doc['end_timestamp']
self.assertDictEqual(model.liveaction, doc)
self.assertEqual(model.parent, self.fake_history_subtasks[0]['parent'])
self.assertListEqual(model.children, [])
# Update the DB record.
children = [str(bson.ObjectId()), str(bson.ObjectId())]
model.children = children
ActionExecution.add_or_update(model)
model = ActionExecution.get_by_id(obj.id)
self.assertListEqual(model.children, children)
# Delete the DB record.
ActionExecution.delete(model)
self.assertRaises(StackStormDBObjectNotFoundError, ActionExecution.get_by_id, obj.id)
def test_purge_executions_with_timestamp(self):
now = date_utils.get_datetime_utc_now()
# Write one execution after cut-off threshold
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=15)
exec_model['end_timestamp'] = now - timedelta(days=14)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Write one execution before cut-off threshold
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=22)
exec_model['end_timestamp'] = now - timedelta(days=21)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
execs = ActionExecution.get_all()
purge_executions(logger=LOG, timestamp=now - timedelta(days=20))
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
def create_execution_object(liveaction, publish=True):
action_db = action_utils.get_action_by_ref(liveaction.action)
runner = RunnerType.get_by_name(action_db.runner_type['name'])
attrs = {
'action': vars(ActionAPI.from_model(action_db)),
'parameters': liveaction['parameters'],
'runner': vars(RunnerTypeAPI.from_model(runner))
}
attrs.update(_decompose_liveaction(liveaction))
if 'rule' in liveaction.context:
rule = reference.get_model_from_ref(Rule,
liveaction.context.get('rule', {}))
attrs['rule'] = vars(RuleAPI.from_model(rule))
if 'trigger_instance' in liveaction.context:
trigger_instance_id = liveaction.context.get('trigger_instance', {})
trigger_instance_id = trigger_instance_id.get('id', None)
trigger_instance = TriggerInstance.get_by_id(trigger_instance_id)
trigger = reference.get_model_by_resource_ref(
db_api=Trigger, ref=trigger_instance.trigger)
trigger_type = reference.get_model_by_resource_ref(db_api=TriggerType,
ref=trigger.type)
trigger_instance = reference.get_model_from_ref(
TriggerInstance, liveaction.context.get('trigger_instance', {}))
attrs['trigger_instance'] = vars(
TriggerInstanceAPI.from_model(trigger_instance))
attrs['trigger'] = vars(TriggerAPI.from_model(trigger))
attrs['trigger_type'] = vars(TriggerTypeAPI.from_model(trigger_type))
parent = _get_parent_execution(liveaction)
if parent:
attrs['parent'] = str(parent.id)
attrs['log'] = [_create_execution_log_entry(liveaction['status'])]
execution = ActionExecutionDB(**attrs)
execution = ActionExecution.add_or_update(execution, publish=False)
# Update the web_url field in execution. Unfortunately, we need
# the execution id for constructing the URL which we only get
# after the model is written to disk.
execution.web_url = _get_web_url_for_execution(str(execution.id))
execution = ActionExecution.add_or_update(execution, publish=publish)
if parent:
if str(execution.id) not in parent.children:
parent.children.append(str(execution.id))
ActionExecution.add_or_update(parent)
return execution
def create_execution_object(liveaction, publish=True):
action_db = action_utils.get_action_by_ref(liveaction.action)
runner = RunnerType.get_by_name(action_db.runner_type['name'])
attrs = {
'action': vars(ActionAPI.from_model(action_db)),
'parameters': liveaction['parameters'],
'runner': vars(RunnerTypeAPI.from_model(runner))
}
attrs.update(_decompose_liveaction(liveaction))
if 'rule' in liveaction.context:
rule = reference.get_model_from_ref(Rule, liveaction.context.get('rule', {}))
attrs['rule'] = vars(RuleAPI.from_model(rule))
if 'trigger_instance' in liveaction.context:
trigger_instance_id = liveaction.context.get('trigger_instance', {})
trigger_instance_id = trigger_instance_id.get('id', None)
trigger_instance = TriggerInstance.get_by_id(trigger_instance_id)
trigger = reference.get_model_by_resource_ref(db_api=Trigger,
ref=trigger_instance.trigger)
trigger_type = reference.get_model_by_resource_ref(db_api=TriggerType,
ref=trigger.type)
trigger_instance = reference.get_model_from_ref(
TriggerInstance, liveaction.context.get('trigger_instance', {}))
attrs['trigger_instance'] = vars(TriggerInstanceAPI.from_model(trigger_instance))
attrs['trigger'] = vars(TriggerAPI.from_model(trigger))
attrs['trigger_type'] = vars(TriggerTypeAPI.from_model(trigger_type))
parent = _get_parent_execution(liveaction)
if parent:
attrs['parent'] = str(parent.id)
attrs['log'] = [_create_execution_log_entry(liveaction['status'])]
execution = ActionExecutionDB(**attrs)
execution = ActionExecution.add_or_update(execution, publish=False)
# Update the web_url field in execution. Unfortunately, we need
# the execution id for constructing the URL which we only get
# after the model is written to disk.
execution.web_url = _get_web_url_for_execution(str(execution.id))
execution = ActionExecution.add_or_update(execution, publish=publish)
if parent:
if str(execution.id) not in parent.children:
parent.children.append(str(execution.id))
ActionExecution.add_or_update(parent)
return execution
def test_no_timestamp_doesnt_delete_things(self):
now = date_utils.get_datetime_utc_now()
exec_model = copy.copy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=15)
exec_model['end_timestamp'] = now - timedelta(days=14)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
purge_executions()
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
def test_garbage_collection(self):
now = date_utils.get_datetime_utc_now()
status = action_constants.LIVEACTION_STATUS_SUCCEEDED
# Insert come mock ActionExecutionDB objects with start_timestamp < TTL defined in the
# config
old_executions_count = 15
ttl_days = 30
timestamp = (now - datetime.timedelta(days=ttl_days))
for index in range(0, old_executions_count):
action_execution_db = ActionExecutionDB(
start_timestamp=timestamp,
end_timestamp=timestamp,
status=status,
action={'ref': 'core.local'},
runner={'name': 'run-local'},
liveaction={'ref': 'foo'})
ActionExecution.add_or_update(action_execution_db)
# Insert come mock ActionExecutionDB objects with start_timestamp > TTL defined in the
# config
new_executions_count = 5
ttl_days = 2
timestamp = (now - datetime.timedelta(days=ttl_days))
for index in range(0, new_executions_count):
action_execution_db = ActionExecutionDB(
start_timestamp=timestamp,
end_timestamp=timestamp,
status=status,
action={'ref': 'core.local'},
runner={'name': 'run-local'},
liveaction={'ref': 'foo'})
ActionExecution.add_or_update(action_execution_db)
execs = ActionExecution.get_all()
self.assertEqual(len(execs),
(old_executions_count + new_executions_count))
# Start garbage collector
process = self._start_garbage_collector()
# Give it some time to perform garbage collection and kill it
eventlet.sleep(5)
process.send_signal(signal.SIGKILL)
self.remove_process(process=process)
# Old execution should have been garbage collected
execs = ActionExecution.get_all()
self.assertEqual(len(execs), (new_executions_count))
def test_purge_incomplete(self):
now = date_utils.get_datetime_utc_now()
start_ts = now - timedelta(days=15)
# Write executions before cut-off threshold
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_SCHEDULED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_RUNNING
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_DELAYED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_CANCELING
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_REQUESTED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
self.assertEqual(len(ActionExecution.get_all()), 5)
purge_executions(logger=LOG,
timestamp=now - timedelta(days=10),
purge_incomplete=False)
self.assertEqual(len(ActionExecution.get_all()), 5)
purge_executions(logger=LOG,
timestamp=now - timedelta(days=10),
purge_incomplete=True)
self.assertEqual(len(ActionExecution.get_all()), 0)
def assign_parent(child):
candidates = [v for k, v in cls.refs.iteritems() if v.action["name"] == "chain"]
if candidates:
parent = random.choice(candidates)
child["parent"] = str(parent.id)
parent.children.append(child["id"])
cls.refs[str(parent.id)] = ActionExecution.add_or_update(parent)
def test_update_execution(self):
"""Test ActionExecutionDb update
"""
self.assertTrue(self.executions['execution_1'].end_timestamp is None)
self.executions['execution_1'].end_timestamp = date_utils.get_datetime_utc_now()
updated = ActionExecution.add_or_update(self.executions['execution_1'])
self.assertTrue(updated.end_timestamp == self.executions['execution_1'].end_timestamp)
def assign_parent(child):
candidates = [v for k, v in cls.refs.items() if v.action["name"] == "chain"]
if candidates:
parent = random.choice(candidates)
child["parent"] = str(parent.id)
parent.children.append(child["id"])
cls.refs[str(parent.id)] = ActionExecution.add_or_update(parent)
def assign_parent(child):
candidates = [v for k, v in cls.refs.items() if v.action['name'] == 'chain']
if candidates:
parent = random.choice(candidates)
child['parent'] = str(parent.id)
parent.children.append(child['id'])
cls.refs[str(parent.id)] = ActionExecution.add_or_update(parent)
def test_update_execution(self):
"""Test ActionExecutionDb update
"""
self.assertTrue(self.executions['execution_1'].end_timestamp is None)
self.executions['execution_1'].end_timestamp = date_utils.get_datetime_utc_now()
updated = ActionExecution.add_or_update(self.executions['execution_1'])
self.assertTrue(updated.end_timestamp == self.executions['execution_1'].end_timestamp)
def assign_parent(child):
candidates = [v for k, v in cls.refs.items() if v.action['name'] == 'chain']
if candidates:
parent = random.choice(candidates)
child['parent'] = str(parent.id)
parent.children.append(child['id'])
cls.refs[str(parent.id)] = ActionExecution.add_or_update(parent)
def update_execution(liveaction_db, publish=True):
execution = ActionExecution.get(liveaction__id=str(liveaction_db.id))
decomposed = _decompose_liveaction(liveaction_db)
for k, v in six.iteritems(decomposed):
setattr(execution, k, v)
execution = ActionExecution.add_or_update(execution, publish=publish)
return execution
def test_get_output_finished_execution(self):
# Test the execution output API endpoint for execution which has finished
for status in action_constants.LIVEACTION_COMPLETED_STATES:
# Insert mock execution and output objects
status = action_constants.LIVEACTION_STATUS_SUCCEEDED
timestamp = date_utils.get_datetime_utc_now()
action_execution_db = ActionExecutionDB(
start_timestamp=timestamp,
end_timestamp=timestamp,
status=status,
action={"ref": "core.local"},
runner={"name": "local-shell-cmd"},
liveaction={"ref": "foo"},
)
action_execution_db = ActionExecution.add_or_update(
action_execution_db)
for i in range(1, 6):
stdout_db = ActionExecutionOutputDB(
execution_id=str(action_execution_db.id),
action_ref="core.local",
runner_ref="dummy",
timestamp=timestamp,
output_type="stdout",
data="stdout %s\n" % (i),
)
ActionExecutionOutput.add_or_update(stdout_db)
for i in range(10, 15):
stderr_db = ActionExecutionOutputDB(
execution_id=str(action_execution_db.id),
action_ref="core.local",
runner_ref="dummy",
timestamp=timestamp,
output_type="stderr",
data="stderr %s\n" % (i),
)
ActionExecutionOutput.add_or_update(stderr_db)
resp = self.app.get(
"/v1/executions/%s/output" % (str(action_execution_db.id)),
expect_errors=False,
)
self.assertEqual(resp.status_int, 200)
events = self._parse_response(resp.text)
self.assertEqual(len(events), 11)
self.assertEqual(events[0][1]["data"], "stdout 1\n")
self.assertEqual(events[9][1]["data"], "stderr 14\n")
self.assertEqual(events[10][0], "EOF")
# Verify "last" short-hand id works
resp = self.app.get("/v1/executions/last/output",
expect_errors=False)
self.assertEqual(resp.status_int, 200)
events = self._parse_response(resp.text)
self.assertEqual(len(events), 11)
self.assertEqual(events[10][0], "EOF")
def test_garbage_collection(self):
now = date_utils.get_datetime_utc_now()
status = action_constants.LIVEACTION_STATUS_SUCCEEDED
# Insert come mock ActionExecutionDB objects with start_timestamp < TTL defined in the
# config
old_executions_count = 15
ttl_days = 30
timestamp = (now - datetime.timedelta(days=ttl_days))
for index in range(0, old_executions_count):
action_execution_db = ActionExecutionDB(start_timestamp=timestamp,
end_timestamp=timestamp,
status=status,
action={'ref': 'core.local'},
runner={'name': 'run-local'},
liveaction={'ref': 'foo'})
ActionExecution.add_or_update(action_execution_db)
# Insert come mock ActionExecutionDB objects with start_timestamp > TTL defined in the
# config
new_executions_count = 5
ttl_days = 2
timestamp = (now - datetime.timedelta(days=ttl_days))
for index in range(0, new_executions_count):
action_execution_db = ActionExecutionDB(start_timestamp=timestamp,
end_timestamp=timestamp,
status=status,
action={'ref': 'core.local'},
runner={'name': 'run-local'},
liveaction={'ref': 'foo'})
ActionExecution.add_or_update(action_execution_db)
execs = ActionExecution.get_all()
self.assertEqual(len(execs), (old_executions_count + new_executions_count))
# Start garbage collector
process = self._start_garbage_collector()
# Give it some time to perform garbage collection and kill it
eventlet.sleep(5)
process.send_signal(signal.SIGKILL)
self.remove_process(process=process)
# Old execution should have been garbage collected
execs = ActionExecution.get_all()
self.assertEqual(len(execs), (new_executions_count))
def test_datetime_range(self):
base = date_utils.add_utc_tz(datetime.datetime(2014, 12, 25, 0, 0, 0))
for i in range(60):
timestamp = base + datetime.timedelta(seconds=i)
doc = copy.deepcopy(self.fake_history_subtasks[0])
doc['id'] = str(bson.ObjectId())
doc['start_timestamp'] = isotime.format(timestamp)
obj = ActionExecutionAPI(**doc)
ActionExecution.add_or_update(ActionExecutionAPI.to_model(obj))
dt_range = '2014-12-25T00:00:10Z..2014-12-25T00:00:19Z'
objs = ActionExecution.query(start_timestamp=dt_range)
self.assertEqual(len(objs), 10)
dt_range = '2014-12-25T00:00:19Z..2014-12-25T00:00:10Z'
objs = ActionExecution.query(start_timestamp=dt_range)
self.assertEqual(len(objs), 10)
def test_no_timestamp_doesnt_delete_things(self):
now = date_utils.get_datetime_utc_now()
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=15)
exec_model['end_timestamp'] = now - timedelta(days=14)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
expected_msg = 'Specify a valid timestamp'
self.assertRaisesRegexp(ValueError, expected_msg, purge_executions,
logger=LOG, timestamp=None)
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
def test_datetime_range(self):
base = date_utils.add_utc_tz(datetime.datetime(2014, 12, 25, 0, 0, 0))
for i in range(60):
timestamp = base + datetime.timedelta(seconds=i)
doc = copy.deepcopy(self.fake_history_subtasks[0])
doc['id'] = str(bson.ObjectId())
doc['start_timestamp'] = isotime.format(timestamp)
obj = ActionExecutionAPI(**doc)
ActionExecution.add_or_update(ActionExecutionAPI.to_model(obj))
dt_range = '2014-12-25T00:00:10Z..2014-12-25T00:00:19Z'
objs = ActionExecution.query(start_timestamp=dt_range)
self.assertEqual(len(objs), 10)
dt_range = '2014-12-25T00:00:19Z..2014-12-25T00:00:10Z'
objs = ActionExecution.query(start_timestamp=dt_range)
self.assertEqual(len(objs), 10)
def publish_action_finished(action_execution_db):
# Insert mock output object
output_params['data'] = 'stdout pre finish 1\n'
output_db = ActionExecutionOutputDB(**output_params)
ActionExecutionOutput.add_or_update(output_db)
# Transition execution to completed state so the connection closes
action_execution_db.status = action_constants.LIVEACTION_STATUS_SUCCEEDED
action_execution_db = ActionExecution.add_or_update(action_execution_db)
def test_purge_executions_with_action_ref(self):
now = date_utils.get_datetime_utc_now()
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=15)
exec_model['end_timestamp'] = now - timedelta(days=14)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
purge_executions(logger=LOG, action_ref='core.localzzz', timestamp=now - timedelta(days=10))
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
purge_executions(logger=LOG, action_ref='core.local', timestamp=now - timedelta(days=10))
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 0)
def test_purge_executions_with_action_ref(self):
now = date_utils.get_datetime_utc_now()
exec_model = copy.copy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=15)
exec_model['end_timestamp'] = now - timedelta(days=14)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
purge_executions(action_ref='core.localzzz', timestamp=now - timedelta(days=10))
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
purge_executions(action_ref='core.local', timestamp=now - timedelta(days=10))
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 0)
def setUpClass(cls):
super(TestActionExecutionFilters, cls).setUpClass()
cls.dt_base = date_utils.add_utc_tz(datetime.datetime(2014, 12, 25, 0, 0, 0))
cls.num_records = 100
cls.refs = {}
cls.start_timestamps = []
cls.fake_types = [
{
"trigger": copy.deepcopy(fixture.ARTIFACTS["trigger"]),
"trigger_type": copy.deepcopy(fixture.ARTIFACTS["trigger_type"]),
"trigger_instance": copy.deepcopy(
fixture.ARTIFACTS["trigger_instance"]
),
"rule": copy.deepcopy(fixture.ARTIFACTS["rule"]),
"action": copy.deepcopy(fixture.ARTIFACTS["actions"]["chain"]),
"runner": copy.deepcopy(fixture.ARTIFACTS["runners"]["action-chain"]),
"liveaction": copy.deepcopy(
fixture.ARTIFACTS["liveactions"]["workflow"]
),
"context": copy.deepcopy(fixture.ARTIFACTS["context"]),
"children": [],
},
{
"action": copy.deepcopy(fixture.ARTIFACTS["actions"]["local"]),
"runner": copy.deepcopy(fixture.ARTIFACTS["runners"]["run-local"]),
"liveaction": copy.deepcopy(fixture.ARTIFACTS["liveactions"]["task1"]),
},
]
def assign_parent(child):
candidates = [v for k, v in cls.refs.items() if v.action["name"] == "chain"]
if candidates:
parent = random.choice(candidates)
child["parent"] = str(parent.id)
parent.children.append(child["id"])
cls.refs[str(parent.id)] = ActionExecution.add_or_update(parent)
for i in range(cls.num_records):
obj_id = str(bson.ObjectId())
timestamp = cls.dt_base + datetime.timedelta(seconds=i)
fake_type = random.choice(cls.fake_types)
data = copy.deepcopy(fake_type)
data["id"] = obj_id
data["start_timestamp"] = isotime.format(timestamp, offset=False)
data["end_timestamp"] = isotime.format(timestamp, offset=False)
data["status"] = data["liveaction"]["status"]
data["result"] = data["liveaction"]["result"]
if fake_type["action"]["name"] == "local" and random.choice([True, False]):
assign_parent(data)
wb_obj = ActionExecutionAPI(**data)
db_obj = ActionExecutionAPI.to_model(wb_obj)
cls.refs[obj_id] = ActionExecution.add_or_update(db_obj)
cls.start_timestamps.append(timestamp)
cls.start_timestamps = sorted(cls.start_timestamps)
def create_execution_object(liveaction, publish=True):
action_db = action_utils.get_action_by_ref(liveaction.action)
runner = RunnerType.get_by_name(action_db.runner_type['name'])
attrs = {
'action': vars(ActionAPI.from_model(action_db)),
'runner': vars(RunnerTypeAPI.from_model(runner))
}
attrs.update(_decompose_liveaction(liveaction))
if 'rule' in liveaction.context:
rule = reference.get_model_from_ref(Rule,
liveaction.context.get('rule', {}))
attrs['rule'] = vars(RuleAPI.from_model(rule))
if 'trigger_instance' in liveaction.context:
trigger_instance_id = liveaction.context.get('trigger_instance', {})
trigger_instance_id = trigger_instance_id.get('id', None)
trigger_instance = TriggerInstance.get_by_id(trigger_instance_id)
trigger = reference.get_model_by_resource_ref(
db_api=Trigger, ref=trigger_instance.trigger)
trigger_type = reference.get_model_by_resource_ref(db_api=TriggerType,
ref=trigger.type)
trigger_instance = reference.get_model_from_ref(
TriggerInstance, liveaction.context.get('trigger_instance', {}))
attrs['trigger_instance'] = vars(
TriggerInstanceAPI.from_model(trigger_instance))
attrs['trigger'] = vars(TriggerAPI.from_model(trigger))
attrs['trigger_type'] = vars(TriggerTypeAPI.from_model(trigger_type))
parent = ActionExecution.get(
liveaction__id=liveaction.context.get('parent', ''))
if parent:
attrs['parent'] = str(parent.id)
execution = ActionExecutionDB(**attrs)
execution = ActionExecution.add_or_update(execution, publish=publish)
if parent:
if str(execution.id) not in parent.children:
parent.children.append(str(execution.id))
ActionExecution.add_or_update(parent)
return execution
def test_purge_executions_with_timestamp(self):
now = date_utils.get_datetime_utc_now()
# Write one execution after cut-off threshold
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=15)
exec_model['end_timestamp'] = now - timedelta(days=14)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(
exec_model['id'], count=3)
# Write one execution before cut-off threshold
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=22)
exec_model['end_timestamp'] = now - timedelta(days=21)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(
exec_model['id'], count=3)
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 2)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 6)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 6)
purge_executions(logger=LOG, timestamp=now - timedelta(days=20))
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 3)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 3)
def test_purge_executions_with_action_ref(self):
now = date_utils.get_datetime_utc_now()
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=15)
exec_model['end_timestamp'] = now - timedelta(days=14)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(
exec_model['id'], count=3)
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 3)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 3)
# Invalid action reference, nothing should be deleted
purge_executions(logger=LOG,
action_ref='core.localzzz',
timestamp=now - timedelta(days=10))
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 3)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 3)
purge_executions(logger=LOG,
action_ref='core.local',
timestamp=now - timedelta(days=10))
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 0)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 0)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 0)
def test_sort_by_start_timestamp(self):
base = date_utils.add_utc_tz(datetime.datetime(2014, 12, 25, 0, 0, 0))
for i in range(60):
timestamp = base + datetime.timedelta(seconds=i)
doc = copy.deepcopy(self.fake_history_subtasks[0])
doc["id"] = str(bson.ObjectId())
doc["start_timestamp"] = isotime.format(timestamp)
obj = ActionExecutionAPI(**doc)
ActionExecution.add_or_update(ActionExecutionAPI.to_model(obj))
dt_range = "2014-12-25T00:00:10Z..2014-12-25T00:00:19Z"
objs = ActionExecution.query(start_timestamp=dt_range,
order_by=["start_timestamp"])
self.assertLess(objs[0]["start_timestamp"], objs[9]["start_timestamp"])
dt_range = "2014-12-25T00:00:19Z..2014-12-25T00:00:10Z"
objs = ActionExecution.query(start_timestamp=dt_range,
order_by=["-start_timestamp"])
self.assertLess(objs[9]["start_timestamp"], objs[0]["start_timestamp"])
def test_get_output_finished_execution(self):
# Test the execution output API endpoint for execution which has finished
for status in action_constants.LIVEACTION_COMPLETED_STATES:
# Insert mock execution and output objects
status = action_constants.LIVEACTION_STATUS_SUCCEEDED
timestamp = date_utils.get_datetime_utc_now()
action_execution_db = ActionExecutionDB(
start_timestamp=timestamp,
end_timestamp=timestamp,
status=status,
action={'ref': 'core.local'},
runner={'name': 'local-shell-cmd'},
liveaction={'ref': 'foo'})
action_execution_db = ActionExecution.add_or_update(
action_execution_db)
for i in range(1, 6):
stdout_db = ActionExecutionOutputDB(execution_id=str(
action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stdout',
data='stdout %s\n' % (i))
ActionExecutionOutput.add_or_update(stdout_db)
for i in range(10, 15):
stderr_db = ActionExecutionOutputDB(execution_id=str(
action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stderr',
data='stderr %s\n' % (i))
ActionExecutionOutput.add_or_update(stderr_db)
resp = self.app.get('/v1/executions/%s/output' %
(str(action_execution_db.id)),
expect_errors=False)
self.assertEqual(resp.status_int, 200)
events = self._parse_response(resp.text)
self.assertEqual(len(events), 11)
self.assertEqual(events[0][1]['data'], 'stdout 1\n')
self.assertEqual(events[9][1]['data'], 'stderr 14\n')
self.assertEqual(events[10][0], 'EOF')
# Verify "last" short-hand id works
resp = self.app.get('/v1/executions/last/output',
expect_errors=False)
self.assertEqual(resp.status_int, 200)
events = self._parse_response(resp.text)
self.assertEqual(len(events), 11)
self.assertEqual(events[10][0], 'EOF')
def create_execution_object(liveaction, publish=True):
action_db = action_utils.get_action_by_ref(liveaction.action)
runner = RunnerType.get_by_name(action_db.runner_type['name'])
attrs = {
'action': vars(ActionAPI.from_model(action_db)),
'parameters': liveaction['parameters'],
'runner': vars(RunnerTypeAPI.from_model(runner))
}
attrs.update(_decompose_liveaction(liveaction))
if 'rule' in liveaction.context:
rule = reference.get_model_from_ref(Rule, liveaction.context.get('rule', {}))
attrs['rule'] = vars(RuleAPI.from_model(rule))
if 'trigger_instance' in liveaction.context:
trigger_instance_id = liveaction.context.get('trigger_instance', {})
trigger_instance_id = trigger_instance_id.get('id', None)
trigger_instance = TriggerInstance.get_by_id(trigger_instance_id)
trigger = reference.get_model_by_resource_ref(db_api=Trigger,
ref=trigger_instance.trigger)
trigger_type = reference.get_model_by_resource_ref(db_api=TriggerType,
ref=trigger.type)
trigger_instance = reference.get_model_from_ref(
TriggerInstance, liveaction.context.get('trigger_instance', {}))
attrs['trigger_instance'] = vars(TriggerInstanceAPI.from_model(trigger_instance))
attrs['trigger'] = vars(TriggerAPI.from_model(trigger))
attrs['trigger_type'] = vars(TriggerTypeAPI.from_model(trigger_type))
parent = _get_parent_execution(liveaction)
if parent:
attrs['parent'] = str(parent.id)
execution = ActionExecutionDB(**attrs)
execution = ActionExecution.add_or_update(execution, publish=publish)
if parent:
if str(execution.id) not in parent.children:
parent.children.append(str(execution.id))
ActionExecution.add_or_update(parent)
return execution
def publish_action_finished(action_execution_db):
# Insert mock output object
output_params['data'] = 'stdout pre finish 1\n'
output_db = ActionExecutionOutputDB(**output_params)
ActionExecutionOutput.add_or_update(output_db)
eventlet.sleep(1.0)
# Transition execution to completed state so the connection closes
action_execution_db.status = action_constants.LIVEACTION_STATUS_SUCCEEDED
action_execution_db = ActionExecution.add_or_update(action_execution_db)
def create_execution_object(liveaction, action_db=None, runnertype_db=None, publish=True):
if not action_db:
action_db = action_utils.get_action_by_ref(liveaction.action)
if not runnertype_db:
runnertype_db = RunnerType.get_by_name(action_db.runner_type['name'])
attrs = {
'action': vars(ActionAPI.from_model(action_db)),
'parameters': liveaction['parameters'],
'runner': vars(RunnerTypeAPI.from_model(runnertype_db))
}
attrs.update(_decompose_liveaction(liveaction))
if 'rule' in liveaction.context:
rule = reference.get_model_from_ref(Rule, liveaction.context.get('rule', {}))
attrs['rule'] = vars(RuleAPI.from_model(rule))
if 'trigger_instance' in liveaction.context:
trigger_instance_id = liveaction.context.get('trigger_instance', {})
trigger_instance_id = trigger_instance_id.get('id', None)
trigger_instance = TriggerInstance.get_by_id(trigger_instance_id)
trigger = reference.get_model_by_resource_ref(db_api=Trigger,
ref=trigger_instance.trigger)
trigger_type = reference.get_model_by_resource_ref(db_api=TriggerType,
ref=trigger.type)
trigger_instance = reference.get_model_from_ref(
TriggerInstance, liveaction.context.get('trigger_instance', {}))
attrs['trigger_instance'] = vars(TriggerInstanceAPI.from_model(trigger_instance))
attrs['trigger'] = vars(TriggerAPI.from_model(trigger))
attrs['trigger_type'] = vars(TriggerTypeAPI.from_model(trigger_type))
parent = _get_parent_execution(liveaction)
if parent:
attrs['parent'] = str(parent.id)
attrs['log'] = [_create_execution_log_entry(liveaction['status'])]
# TODO: This object initialization takes 20-30or so ms
execution = ActionExecutionDB(**attrs)
# TODO: Do 100% research this is fully safe and unique in distributed setups
execution.id = ObjectId()
execution.web_url = _get_web_url_for_execution(str(execution.id))
# NOTE: User input data is already validate as part of the API request,
# other data is set by us. Skipping validation here makes operation 10%-30% faster
execution = ActionExecution.add_or_update(execution, publish=publish, validate=False)
if parent and str(execution.id) not in parent.children:
values = {}
values['push__children'] = str(execution.id)
ActionExecution.update(parent, **values)
return execution
def setUpClass(cls):
super(TestActionExecutionFilters, cls).setUpClass()
cls.dt_base = date_utils.add_utc_tz(datetime.datetime(2014, 12, 25, 0, 0, 0))
cls.num_records = 100
cls.refs = {}
cls.start_timestamps = []
cls.fake_types = [
{
'trigger': copy.deepcopy(fixture.ARTIFACTS['trigger']),
'trigger_type': copy.deepcopy(fixture.ARTIFACTS['trigger_type']),
'trigger_instance': copy.deepcopy(fixture.ARTIFACTS['trigger_instance']),
'rule': copy.deepcopy(fixture.ARTIFACTS['rule']),
'action': copy.deepcopy(fixture.ARTIFACTS['actions']['chain']),
'runner': copy.deepcopy(fixture.ARTIFACTS['runners']['action-chain']),
'liveaction': copy.deepcopy(fixture.ARTIFACTS['liveactions']['workflow']),
'context': copy.deepcopy(fixture.ARTIFACTS['context']),
'children': []
},
{
'action': copy.deepcopy(fixture.ARTIFACTS['actions']['local']),
'runner': copy.deepcopy(fixture.ARTIFACTS['runners']['run-local']),
'liveaction': copy.deepcopy(fixture.ARTIFACTS['liveactions']['task1'])
}
]
def assign_parent(child):
candidates = [v for k, v in cls.refs.items() if v.action['name'] == 'chain']
if candidates:
parent = random.choice(candidates)
child['parent'] = str(parent.id)
parent.children.append(child['id'])
cls.refs[str(parent.id)] = ActionExecution.add_or_update(parent)
for i in range(cls.num_records):
obj_id = str(bson.ObjectId())
timestamp = cls.dt_base + datetime.timedelta(seconds=i)
fake_type = random.choice(cls.fake_types)
data = copy.deepcopy(fake_type)
data['id'] = obj_id
data['start_timestamp'] = isotime.format(timestamp, offset=False)
data['end_timestamp'] = isotime.format(timestamp, offset=False)
data['status'] = data['liveaction']['status']
data['result'] = data['liveaction']['result']
if fake_type['action']['name'] == 'local' and random.choice([True, False]):
assign_parent(data)
wb_obj = ActionExecutionAPI(**data)
db_obj = ActionExecutionAPI.to_model(wb_obj)
cls.refs[obj_id] = ActionExecution.add_or_update(db_obj)
cls.start_timestamps.append(timestamp)
cls.start_timestamps = sorted(cls.start_timestamps)
def setUpClass(cls):
super(TestActionExecutionFilters, cls).setUpClass()
cls.dt_base = date_utils.add_utc_tz(datetime.datetime(2014, 12, 25, 0, 0, 0))
cls.num_records = 100
cls.refs = {}
cls.start_timestamps = []
cls.fake_types = [
{
'trigger': copy.deepcopy(fixture.ARTIFACTS['trigger']),
'trigger_type': copy.deepcopy(fixture.ARTIFACTS['trigger_type']),
'trigger_instance': copy.deepcopy(fixture.ARTIFACTS['trigger_instance']),
'rule': copy.deepcopy(fixture.ARTIFACTS['rule']),
'action': copy.deepcopy(fixture.ARTIFACTS['actions']['chain']),
'runner': copy.deepcopy(fixture.ARTIFACTS['runners']['action-chain']),
'liveaction': copy.deepcopy(fixture.ARTIFACTS['liveactions']['workflow']),
'context': copy.deepcopy(fixture.ARTIFACTS['context']),
'children': []
},
{
'action': copy.deepcopy(fixture.ARTIFACTS['actions']['local']),
'runner': copy.deepcopy(fixture.ARTIFACTS['runners']['run-local']),
'liveaction': copy.deepcopy(fixture.ARTIFACTS['liveactions']['task1'])
}
]
def assign_parent(child):
candidates = [v for k, v in cls.refs.items() if v.action['name'] == 'chain']
if candidates:
parent = random.choice(candidates)
child['parent'] = str(parent.id)
parent.children.append(child['id'])
cls.refs[str(parent.id)] = ActionExecution.add_or_update(parent)
for i in range(cls.num_records):
obj_id = str(bson.ObjectId())
timestamp = cls.dt_base + datetime.timedelta(seconds=i)
fake_type = random.choice(cls.fake_types)
data = copy.deepcopy(fake_type)
data['id'] = obj_id
data['start_timestamp'] = isotime.format(timestamp, offset=False)
data['end_timestamp'] = isotime.format(timestamp, offset=False)
data['status'] = data['liveaction']['status']
data['result'] = data['liveaction']['result']
if fake_type['action']['name'] == 'local' and random.choice([True, False]):
assign_parent(data)
wb_obj = ActionExecutionAPI(**data)
db_obj = ActionExecutionAPI.to_model(wb_obj)
cls.refs[obj_id] = ActionExecution.add_or_update(db_obj)
cls.start_timestamps.append(timestamp)
cls.start_timestamps = sorted(cls.start_timestamps)
def setUpClass(cls):
super(TestActionExecutionFilters, cls).setUpClass()
cls.dt_base = date_utils.add_utc_tz(datetime.datetime(2014, 12, 25, 0, 0, 0))
cls.num_records = 100
cls.refs = {}
cls.start_timestamps = []
cls.fake_types = [
{
"trigger": copy.deepcopy(fixture.ARTIFACTS["trigger"]),
"trigger_type": copy.deepcopy(fixture.ARTIFACTS["trigger_type"]),
"trigger_instance": copy.deepcopy(fixture.ARTIFACTS["trigger_instance"]),
"rule": copy.deepcopy(fixture.ARTIFACTS["rule"]),
"action": copy.deepcopy(fixture.ARTIFACTS["actions"]["chain"]),
"runner": copy.deepcopy(fixture.ARTIFACTS["runners"]["action-chain"]),
"liveaction": copy.deepcopy(fixture.ARTIFACTS["liveactions"]["workflow"]),
"context": copy.deepcopy(fixture.ARTIFACTS["context"]),
"children": [],
},
{
"action": copy.deepcopy(fixture.ARTIFACTS["actions"]["local"]),
"runner": copy.deepcopy(fixture.ARTIFACTS["runners"]["run-local"]),
"liveaction": copy.deepcopy(fixture.ARTIFACTS["liveactions"]["task1"]),
},
]
def assign_parent(child):
candidates = [v for k, v in cls.refs.iteritems() if v.action["name"] == "chain"]
if candidates:
parent = random.choice(candidates)
child["parent"] = str(parent.id)
parent.children.append(child["id"])
cls.refs[str(parent.id)] = ActionExecution.add_or_update(parent)
for i in range(cls.num_records):
obj_id = str(bson.ObjectId())
timestamp = cls.dt_base + datetime.timedelta(seconds=i)
fake_type = random.choice(cls.fake_types)
data = copy.deepcopy(fake_type)
data["id"] = obj_id
data["start_timestamp"] = isotime.format(timestamp, offset=False)
data["end_timestamp"] = isotime.format(timestamp, offset=False)
data["status"] = data["liveaction"]["status"]
data["result"] = data["liveaction"]["result"]
if fake_type["action"]["name"] == "local" and random.choice([True, False]):
assign_parent(data)
wb_obj = ActionExecutionAPI(**data)
db_obj = ActionExecutionAPI.to_model(wb_obj)
cls.refs[obj_id] = ActionExecution.add_or_update(db_obj)
cls.start_timestamps.append(timestamp)
cls.start_timestamps = sorted(cls.start_timestamps)
def test_sort_by_start_timestamp(self):
base = isotime.add_utc_tz(datetime.datetime(2014, 12, 25, 0, 0, 0))
for i in range(60):
timestamp = base + datetime.timedelta(seconds=i)
doc = copy.deepcopy(self.fake_history_subtasks[0])
doc['id'] = str(bson.ObjectId())
doc['start_timestamp'] = isotime.format(timestamp)
obj = ActionExecutionAPI(**doc)
ActionExecution.add_or_update(ActionExecutionAPI.to_model(obj))
dt_range = '2014-12-25T00:00:10Z..2014-12-25T00:00:19Z'
objs = ActionExecution.query(start_timestamp=dt_range,
order_by=['start_timestamp'])
self.assertLess(objs[0]['start_timestamp'],
objs[9]['start_timestamp'])
dt_range = '2014-12-25T00:00:19Z..2014-12-25T00:00:10Z'
objs = ActionExecution.query(start_timestamp=dt_range,
order_by=['-start_timestamp'])
self.assertLess(objs[9]['start_timestamp'],
objs[0]['start_timestamp'])
def test_no_timestamp_doesnt_delete_things(self):
now = date_utils.get_datetime_utc_now()
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=15)
exec_model['end_timestamp'] = now - timedelta(days=14)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
expected_msg = 'Specify a valid timestamp'
self.assertRaisesRegexp(ValueError,
expected_msg,
purge_executions,
logger=LOG,
timestamp=None)
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
def test_purge_incomplete(self):
now = date_utils.get_datetime_utc_now()
start_ts = now - timedelta(days=15)
# Write executions before cut-off threshold
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_SCHEDULED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_RUNNING
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_DELAYED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_CANCELING
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_REQUESTED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
self.assertEqual(len(ActionExecution.get_all()), 5)
purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=False)
self.assertEqual(len(ActionExecution.get_all()), 5)
purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=True)
self.assertEqual(len(ActionExecution.get_all()), 0)
def test_purge_executions_with_timestamp(self):
now = date_utils.get_datetime_utc_now()
# Write one execution after cut-off threshold
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=15)
exec_model['end_timestamp'] = now - timedelta(days=14)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=3)
# Write one execution before cut-off threshold
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=22)
exec_model['end_timestamp'] = now - timedelta(days=21)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=3)
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 2)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 6)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 6)
purge_executions(logger=LOG, timestamp=now - timedelta(days=20))
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 3)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 3)
def test_purge_executions_with_timestamp(self):
now = date_utils.get_datetime_utc_now()
# Write one execution after cut-off threshold
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=15)
exec_model['end_timestamp'] = now - timedelta(days=14)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Write one execution before cut-off threshold
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=22)
exec_model['end_timestamp'] = now - timedelta(days=21)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
execs = ActionExecution.get_all()
purge_executions(logger=LOG, timestamp=now - timedelta(days=20))
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
def test_purge_executions_with_action_ref(self):
now = date_utils.get_datetime_utc_now()
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=15)
exec_model['end_timestamp'] = now - timedelta(days=14)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=3)
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 3)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 3)
# Invalid action reference, nothing should be deleted
purge_executions(logger=LOG, action_ref='core.localzzz', timestamp=now - timedelta(days=10))
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 3)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 3)
purge_executions(logger=LOG, action_ref='core.local', timestamp=now - timedelta(days=10))
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 0)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 0)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 0)
def test_get_output_finished_execution(self):
# Test the execution output API endpoint for execution which has finished
for status in action_constants.LIVEACTION_COMPLETED_STATES:
# Insert mock execution and output objects
status = action_constants.LIVEACTION_STATUS_SUCCEEDED
timestamp = date_utils.get_datetime_utc_now()
action_execution_db = ActionExecutionDB(start_timestamp=timestamp,
end_timestamp=timestamp,
status=status,
action={'ref': 'core.local'},
runner={'name': 'local-shell-cmd'},
liveaction={'ref': 'foo'})
action_execution_db = ActionExecution.add_or_update(action_execution_db)
for i in range(1, 6):
stdout_db = ActionExecutionOutputDB(execution_id=str(action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stdout',
data='stdout %s\n' % (i))
ActionExecutionOutput.add_or_update(stdout_db)
for i in range(10, 15):
stderr_db = ActionExecutionOutputDB(execution_id=str(action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stderr',
data='stderr %s\n' % (i))
ActionExecutionOutput.add_or_update(stderr_db)
resp = self.app.get('/v1/executions/%s/output' % (str(action_execution_db.id)),
expect_errors=False)
self.assertEqual(resp.status_int, 200)
events = self._parse_response(resp.text)
self.assertEqual(len(events), 11)
self.assertEqual(events[0][1]['data'], 'stdout 1\n')
self.assertEqual(events[9][1]['data'], 'stderr 14\n')
self.assertEqual(events[10][0], 'EOF')
# Verify "last" short-hand id works
resp = self.app.get('/v1/executions/last/output', expect_errors=False)
self.assertEqual(resp.status_int, 200)
events = self._parse_response(resp.text)
self.assertEqual(len(events), 11)
self.assertEqual(events[10][0], 'EOF')
def test_crud_complete(self):
# Create the DB record.
obj = ActionExecutionAPI(**copy.deepcopy(self.fake_history_workflow))
ActionExecution.add_or_update(ActionExecutionAPI.to_model(obj))
model = ActionExecution.get_by_id(obj.id)
self.assertEqual(str(model.id), obj.id)
self.assertDictEqual(model.trigger,
self.fake_history_workflow["trigger"])
self.assertDictEqual(model.trigger_type,
self.fake_history_workflow["trigger_type"])
self.assertDictEqual(model.trigger_instance,
self.fake_history_workflow["trigger_instance"])
self.assertDictEqual(model.rule, self.fake_history_workflow["rule"])
self.assertDictEqual(model.action,
self.fake_history_workflow["action"])
self.assertDictEqual(model.runner,
self.fake_history_workflow["runner"])
doc = copy.deepcopy(self.fake_history_workflow["liveaction"])
doc["start_timestamp"] = doc["start_timestamp"]
doc["end_timestamp"] = doc["end_timestamp"]
self.assertDictEqual(model.liveaction, doc)
self.assertIsNone(getattr(model, "parent", None))
self.assertListEqual(model.children,
self.fake_history_workflow["children"])
# Update the DB record.
children = [str(bson.ObjectId()), str(bson.ObjectId())]
model.children = children
ActionExecution.add_or_update(model)
model = ActionExecution.get_by_id(obj.id)
self.assertListEqual(model.children, children)
# Delete the DB record.
ActionExecution.delete(model)
self.assertRaises(StackStormDBObjectNotFoundError,
ActionExecution.get_by_id, obj.id)
def setUp(self):
super(ExecutionPermissionsResolverTestCase, self).setUp()
# Create some mock users
user_1_db = UserDB(name='custom_role_unrelated_pack_action_grant')
user_1_db = User.add_or_update(user_1_db)
self.users['custom_role_unrelated_pack_action_grant'] = user_1_db
user_2_db = UserDB(
name='custom_role_pack_action_grant_unrelated_permission')
user_2_db = User.add_or_update(user_2_db)
self.users[
'custom_role_pack_action_grant_unrelated_permission'] = user_2_db
user_3_db = UserDB(name='custom_role_pack_action_view_grant')
user_3_db = User.add_or_update(user_3_db)
self.users['custom_role_pack_action_view_grant'] = user_3_db
user_4_db = UserDB(name='custom_role_action_view_grant')
user_4_db = User.add_or_update(user_4_db)
self.users['custom_role_action_view_grant'] = user_4_db
user_5_db = UserDB(name='custom_role_pack_action_execute_grant')
user_5_db = User.add_or_update(user_5_db)
self.users['custom_role_pack_action_execute_grant'] = user_5_db
user_6_db = UserDB(name='custom_role_action_execute_grant')
user_6_db = User.add_or_update(user_6_db)
self.users['custom_role_action_execute_grant'] = user_6_db
user_7_db = UserDB(name='custom_role_pack_action_all_grant')
user_7_db = User.add_or_update(user_7_db)
self.users['custom_role_pack_action_all_grant'] = user_7_db
user_8_db = UserDB(name='custom_role_action_all_grant')
user_8_db = User.add_or_update(user_8_db)
self.users['custom_role_action_all_grant'] = user_8_db
user_9_db = UserDB(name='custom_role_execution_list_grant')
user_9_db = User.add_or_update(user_5_db)
self.users['custom_role_execution_list_grant'] = user_9_db
# Create some mock resources on which permissions can be granted
action_1_db = ActionDB(pack='test_pack_2',
name='action1',
entry_point='',
runner_type={'name': 'run-local'})
action_1_db = Action.add_or_update(action_1_db)
self.resources['action_1'] = action_1_db
runner = {'name': 'run-python'}
liveaction = {'action': 'test_pack_2.action1'}
status = action_constants.LIVEACTION_STATUS_REQUESTED
action = {'uid': action_1_db.get_uid(), 'pack': 'test_pack_2'}
exec_1_db = ActionExecutionDB(action=action,
runner=runner,
liveaction=liveaction,
status=status)
exec_1_db = ActionExecution.add_or_update(exec_1_db)
self.resources['exec_1'] = exec_1_db
# Create some mock roles with associated permission grants
# Custom role - one grant to an unrelated pack
grant_db = PermissionGrantDB(
resource_uid=self.resources['pack_1'].get_uid(),
resource_type=ResourceType.PACK,
permission_types=[PermissionType.ACTION_VIEW])
grant_db = PermissionGrant.add_or_update(grant_db)
permission_grants = [str(grant_db.id)]
role_db = RoleDB(name='custom_role_unrelated_pack_action_grant',
permission_grants=permission_grants)
role_db = Role.add_or_update(role_db)
self.roles['custom_role_unrelated_pack_action_grant'] = role_db
# Custom role - one grant of unrelated permission type to parent action pack
grant_db = PermissionGrantDB(
resource_uid=self.resources['pack_2'].get_uid(),
resource_type=ResourceType.PACK,
permission_types=[PermissionType.RULE_VIEW])
grant_db = PermissionGrant.add_or_update(grant_db)
permission_grants = [str(grant_db.id)]
role_db = RoleDB(
name='custom_role_pack_action_grant_unrelated_permission',
permission_grants=permission_grants)
role_db = Role.add_or_update(role_db)
self.roles[
'custom_role_pack_action_grant_unrelated_permission'] = role_db
# Custom role - one grant of "action_view" to the parent pack of the action the execution
# belongs to
grant_db = PermissionGrantDB(
resource_uid=self.resources['pack_2'].get_uid(),
resource_type=ResourceType.PACK,
permission_types=[PermissionType.ACTION_VIEW])
grant_db = PermissionGrant.add_or_update(grant_db)
permission_grants = [str(grant_db.id)]
role_db = RoleDB(name='custom_role_pack_action_view_grant',
permission_grants=permission_grants)
role_db = Role.add_or_update(role_db)
self.roles['custom_role_pack_action_view_grant'] = role_db
# Custom role - one grant of "action_view" to the action the execution belongs to
grant_db = PermissionGrantDB(
resource_uid=self.resources['action_1'].get_uid(),
resource_type=ResourceType.ACTION,
permission_types=[PermissionType.ACTION_VIEW])
grant_db = PermissionGrant.add_or_update(grant_db)
permission_grants = [str(grant_db.id)]
role_db = RoleDB(name='custom_role_action_view_grant',
permission_grants=permission_grants)
role_db = Role.add_or_update(role_db)
self.roles['custom_role_action_view_grant'] = role_db
# Custom role - one grant of "action_execute" to the parent pack of the action the
# execution belongs to
grant_db = PermissionGrantDB(
resource_uid=self.resources['pack_2'].get_uid(),
resource_type=ResourceType.PACK,
permission_types=[PermissionType.ACTION_EXECUTE])
grant_db = PermissionGrant.add_or_update(grant_db)
permission_grants = [str(grant_db.id)]
role_db = RoleDB(name='custom_role_pack_action_execute_grant',
permission_grants=permission_grants)
role_db = Role.add_or_update(role_db)
self.roles['custom_role_pack_action_execute_grant'] = role_db
# Custom role - one grant of "action_execute" to the the action the execution belongs to
grant_db = PermissionGrantDB(
resource_uid=self.resources['action_1'].get_uid(),
resource_type=ResourceType.ACTION,
permission_types=[PermissionType.ACTION_EXECUTE])
grant_db = PermissionGrant.add_or_update(grant_db)
permission_grants = [str(grant_db.id)]
role_db = RoleDB(name='custom_role_action_execute_grant',
permission_grants=permission_grants)
role_db = Role.add_or_update(role_db)
self.roles['custom_role_action_execute_grant'] = role_db
# Custom role - "action_all" grant on a parent action pack the execution belongs to
grant_db = PermissionGrantDB(
resource_uid=self.resources['pack_2'].get_uid(),
resource_type=ResourceType.PACK,
permission_types=[PermissionType.ACTION_ALL])
grant_db = PermissionGrant.add_or_update(grant_db)
permission_grants = [str(grant_db.id)]
role_4_db = RoleDB(name='custom_role_pack_action_all_grant',
permission_grants=permission_grants)
role_4_db = Role.add_or_update(role_4_db)
self.roles['custom_role_pack_action_all_grant'] = role_4_db
# Custom role - "action_all" grant on action the execution belongs to
grant_db = PermissionGrantDB(
resource_uid=self.resources['action_1'].get_uid(),
resource_type=ResourceType.ACTION,
permission_types=[PermissionType.ACTION_ALL])
grant_db = PermissionGrant.add_or_update(grant_db)
permission_grants = [str(grant_db.id)]
role_4_db = RoleDB(name='custom_role_action_all_grant',
permission_grants=permission_grants)
role_4_db = Role.add_or_update(role_4_db)
self.roles['custom_role_action_all_grant'] = role_4_db
# Custom role - "execution_list" grant
grant_db = PermissionGrantDB(
resource_uid=None,
resource_type=None,
permission_types=[PermissionType.EXECUTION_LIST])
grant_db = PermissionGrant.add_or_update(grant_db)
permission_grants = [str(grant_db.id)]
role_5_db = RoleDB(name='custom_role_execution_list_grant',
permission_grants=permission_grants)
role_5_db = Role.add_or_update(role_5_db)
self.roles['custom_role_execution_list_grant'] = role_5_db
# Create some mock role assignments
user_db = self.users['custom_role_unrelated_pack_action_grant']
role_assignment_db = UserRoleAssignmentDB(
user=user_db.name,
role=self.roles['custom_role_unrelated_pack_action_grant'].name,
source='assignments/%s.yaml' % user_db.name)
UserRoleAssignment.add_or_update(role_assignment_db)
user_db = self.users[
'custom_role_pack_action_grant_unrelated_permission']
role_assignment_db = UserRoleAssignmentDB(
user=user_db.name,
role=self.
roles['custom_role_pack_action_grant_unrelated_permission'].name,
source='assignments/%s.yaml' % user_db.name)
UserRoleAssignment.add_or_update(role_assignment_db)
user_db = self.users['custom_role_pack_action_view_grant']
role_assignment_db = UserRoleAssignmentDB(
user=user_db.name,
role=self.roles['custom_role_pack_action_view_grant'].name,
source='assignments/%s.yaml' % user_db.name)
UserRoleAssignment.add_or_update(role_assignment_db)
user_db = self.users['custom_role_action_view_grant']
role_assignment_db = UserRoleAssignmentDB(
user=user_db.name,
role=self.roles['custom_role_action_view_grant'].name,
source='assignments/%s.yaml' % user_db.name)
UserRoleAssignment.add_or_update(role_assignment_db)
user_db = self.users['custom_role_pack_action_execute_grant']
role_assignment_db = UserRoleAssignmentDB(
user=user_db.name,
role=self.roles['custom_role_pack_action_execute_grant'].name,
source='assignments/%s.yaml' % user_db.name)
UserRoleAssignment.add_or_update(role_assignment_db)
user_db = self.users['custom_role_action_execute_grant']
role_assignment_db = UserRoleAssignmentDB(
user=user_db.name,
role=self.roles['custom_role_action_execute_grant'].name,
source='assignments/%s.yaml' % user_db.name)
UserRoleAssignment.add_or_update(role_assignment_db)
user_db = self.users['custom_role_pack_action_all_grant']
role_assignment_db = UserRoleAssignmentDB(
user=user_db.name,
role=self.roles['custom_role_pack_action_all_grant'].name,
source='assignments/%s.yaml' % user_db.name)
UserRoleAssignment.add_or_update(role_assignment_db)
user_db = self.users['custom_role_action_all_grant']
role_assignment_db = UserRoleAssignmentDB(
user=user_db.name,
role=self.roles['custom_role_action_all_grant'].name,
source='assignments/%s.yaml' % user_db.name)
UserRoleAssignment.add_or_update(role_assignment_db)
user_db = self.users['custom_role_execution_list_grant']
role_assignment_db = UserRoleAssignmentDB(
user=user_db.name,
role=self.roles['custom_role_execution_list_grant'].name,
source='assignments/%s.yaml' % user_db.name)
UserRoleAssignment.add_or_update(role_assignment_db)
def test_get_output_running_execution(self):
# Retrieve lister instance to avoid race with listener connection not being established
# early enough for tests to pass.
# NOTE: This only affects tests where listeners are not pre-initialized.
listener = get_listener(name='execution_output')
eventlet.sleep(1.0)
# Test the execution output API endpoint for execution which is running (blocking)
status = action_constants.LIVEACTION_STATUS_RUNNING
timestamp = date_utils.get_datetime_utc_now()
action_execution_db = ActionExecutionDB(
start_timestamp=timestamp,
end_timestamp=timestamp,
status=status,
action={'ref': 'core.local'},
runner={'name': 'local-shell-cmd'},
liveaction={'ref': 'foo'})
action_execution_db = ActionExecution.add_or_update(
action_execution_db)
output_params = dict(execution_id=str(action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stdout',
data='stdout before start\n')
# Insert mock output object
output_db = ActionExecutionOutputDB(**output_params)
ActionExecutionOutput.add_or_update(output_db, publish=False)
def insert_mock_data():
output_params['data'] = 'stdout mid 1\n'
output_db = ActionExecutionOutputDB(**output_params)
ActionExecutionOutput.add_or_update(output_db)
# Since the API endpoint is blocking (connection is kept open until action finishes), we
# spawn an eventlet which eventually finishes the action.
def publish_action_finished(action_execution_db):
# Insert mock output object
output_params['data'] = 'stdout pre finish 1\n'
output_db = ActionExecutionOutputDB(**output_params)
ActionExecutionOutput.add_or_update(output_db)
eventlet.sleep(1.0)
# Transition execution to completed state so the connection closes
action_execution_db.status = action_constants.LIVEACTION_STATUS_SUCCEEDED
action_execution_db = ActionExecution.add_or_update(
action_execution_db)
eventlet.spawn_after(0.2, insert_mock_data)
eventlet.spawn_after(1.5, publish_action_finished, action_execution_db)
# Retrieve data while execution is running - endpoint return new data once it's available
# and block until the execution finishes
resp = self.app.get('/v1/executions/%s/output' %
(str(action_execution_db.id)),
expect_errors=False)
self.assertEqual(resp.status_int, 200)
events = self._parse_response(resp.text)
self.assertEqual(len(events), 4)
self.assertEqual(events[0][1]['data'], 'stdout before start\n')
self.assertEqual(events[1][1]['data'], 'stdout mid 1\n')
self.assertEqual(events[2][1]['data'], 'stdout pre finish 1\n')
self.assertEqual(events[3][0], 'EOF')
# Once the execution is in completed state, existing output should be returned immediately
resp = self.app.get('/v1/executions/%s/output' %
(str(action_execution_db.id)),
expect_errors=False)
self.assertEqual(resp.status_int, 200)
events = self._parse_response(resp.text)
self.assertEqual(len(events), 4)
self.assertEqual(events[0][1]['data'], 'stdout before start\n')
self.assertEqual(events[1][1]['data'], 'stdout mid 1\n')
self.assertEqual(events[2][1]['data'], 'stdout pre finish 1\n')
self.assertEqual(events[3][0], 'EOF')
listener.shutdown()
def _save_execution(execution):
return ActionExecution.add_or_update(execution)
def test_garbage_collection(self):
now = date_utils.get_datetime_utc_now()
status = action_constants.LIVEACTION_STATUS_SUCCEEDED
# Insert come mock ActionExecutionDB objects with start_timestamp < TTL defined in the
# config
old_executions_count = 15
ttl_days = 30 # > 20
timestamp = (now - datetime.timedelta(days=ttl_days))
for index in range(0, old_executions_count):
action_execution_db = ActionExecutionDB(start_timestamp=timestamp,
end_timestamp=timestamp,
status=status,
action={'ref': 'core.local'},
runner={'name': 'local-shell-cmd'},
liveaction={'ref': 'foo'})
ActionExecution.add_or_update(action_execution_db)
stdout_db = ActionExecutionOutputDB(execution_id=str(action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stdout',
data='stdout')
ActionExecutionOutput.add_or_update(stdout_db)
stderr_db = ActionExecutionOutputDB(execution_id=str(action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stderr',
data='stderr')
ActionExecutionOutput.add_or_update(stderr_db)
# Insert come mock ActionExecutionDB objects with start_timestamp > TTL defined in the
# config
new_executions_count = 5
ttl_days = 2 # < 20
timestamp = (now - datetime.timedelta(days=ttl_days))
for index in range(0, new_executions_count):
action_execution_db = ActionExecutionDB(start_timestamp=timestamp,
end_timestamp=timestamp,
status=status,
action={'ref': 'core.local'},
runner={'name': 'local-shell-cmd'},
liveaction={'ref': 'foo'})
ActionExecution.add_or_update(action_execution_db)
stdout_db = ActionExecutionOutputDB(execution_id=str(action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stdout',
data='stdout')
ActionExecutionOutput.add_or_update(stdout_db)
stderr_db = ActionExecutionOutputDB(execution_id=str(action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stderr',
data='stderr')
ActionExecutionOutput.add_or_update(stderr_db)
# Insert some mock output objects where start_timestamp > action_executions_output_ttl
new_output_count = 5
ttl_days = 15 # > 10 and < 20
timestamp = (now - datetime.timedelta(days=ttl_days))
for index in range(0, new_output_count):
action_execution_db = ActionExecutionDB(start_timestamp=timestamp,
end_timestamp=timestamp,
status=status,
action={'ref': 'core.local'},
runner={'name': 'local-shell-cmd'},
liveaction={'ref': 'foo'})
ActionExecution.add_or_update(action_execution_db)
stdout_db = ActionExecutionOutputDB(execution_id=str(action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stdout',
data='stdout')
ActionExecutionOutput.add_or_update(stdout_db)
stderr_db = ActionExecutionOutputDB(execution_id=str(action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stderr',
data='stderr')
ActionExecutionOutput.add_or_update(stderr_db)
execs = ActionExecution.get_all()
self.assertEqual(len(execs),
(old_executions_count + new_executions_count + new_output_count))
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs),
(old_executions_count + new_executions_count + new_output_count))
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs),
(old_executions_count + new_executions_count + new_output_count))
# Start garbage collector
process = self._start_garbage_collector()
# Give it some time to perform garbage collection and kill it
eventlet.sleep(15)
process.send_signal(signal.SIGKILL)
self.remove_process(process=process)
# Old executions and corresponding objects should have been garbage collected
execs = ActionExecution.get_all()
self.assertEqual(len(execs), (new_executions_count + new_output_count))
# Collection for output objects older than 10 days is also enabled, so those objects
# should be deleted as well
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), (new_executions_count))
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), (new_executions_count))
def test_purge_incomplete(self):
now = date_utils.get_datetime_utc_now()
start_ts = now - timedelta(days=15)
# Write executions before cut-off threshold
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_SCHEDULED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=1)
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_RUNNING
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=1)
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_DELAYED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=1)
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_CANCELING
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=1)
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_REQUESTED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=1)
self.assertEqual(len(ActionExecution.get_all()), 5)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 5)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 5)
# Incompleted executions shouldnt be purged
purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=False)
self.assertEqual(len(ActionExecution.get_all()), 5)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 5)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 5)
purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=True)
self.assertEqual(len(ActionExecution.get_all()), 0)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 0)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 0)
def test_get_output_running_execution(self):
# Retrieve lister instance to avoid race with listener connection not being established
# early enough for tests to pass.
# NOTE: This only affects tests where listeners are not pre-initialized.
listener = get_listener(name='execution_output')
eventlet.sleep(1.0)
# Test the execution output API endpoint for execution which is running (blocking)
status = action_constants.LIVEACTION_STATUS_RUNNING
timestamp = date_utils.get_datetime_utc_now()
action_execution_db = ActionExecutionDB(start_timestamp=timestamp,
end_timestamp=timestamp,
status=status,
action={'ref': 'core.local'},
runner={'name': 'local-shell-cmd'},
liveaction={'ref': 'foo'})
action_execution_db = ActionExecution.add_or_update(action_execution_db)
output_params = dict(execution_id=str(action_execution_db.id),
action_ref='core.local',
runner_ref='dummy',
timestamp=timestamp,
output_type='stdout',
data='stdout before start\n')
# Insert mock output object
output_db = ActionExecutionOutputDB(**output_params)
ActionExecutionOutput.add_or_update(output_db, publish=False)
def insert_mock_data():
output_params['data'] = 'stdout mid 1\n'
output_db = ActionExecutionOutputDB(**output_params)
ActionExecutionOutput.add_or_update(output_db)
# Since the API endpoint is blocking (connection is kept open until action finishes), we
# spawn an eventlet which eventually finishes the action.
def publish_action_finished(action_execution_db):
# Insert mock output object
output_params['data'] = 'stdout pre finish 1\n'
output_db = ActionExecutionOutputDB(**output_params)
ActionExecutionOutput.add_or_update(output_db)
eventlet.sleep(1.0)
# Transition execution to completed state so the connection closes
action_execution_db.status = action_constants.LIVEACTION_STATUS_SUCCEEDED
action_execution_db = ActionExecution.add_or_update(action_execution_db)
eventlet.spawn_after(0.2, insert_mock_data)
eventlet.spawn_after(1.5, publish_action_finished, action_execution_db)
# Retrieve data while execution is running - endpoint return new data once it's available
# and block until the execution finishes
resp = self.app.get('/v1/executions/%s/output' % (str(action_execution_db.id)),
expect_errors=False)
self.assertEqual(resp.status_int, 200)
events = self._parse_response(resp.text)
self.assertEqual(len(events), 4)
self.assertEqual(events[0][1]['data'], 'stdout before start\n')
self.assertEqual(events[1][1]['data'], 'stdout mid 1\n')
self.assertEqual(events[2][1]['data'], 'stdout pre finish 1\n')
self.assertEqual(events[3][0], 'EOF')
# Once the execution is in completed state, existing output should be returned immediately
resp = self.app.get('/v1/executions/%s/output' % (str(action_execution_db.id)),
expect_errors=False)
self.assertEqual(resp.status_int, 200)
events = self._parse_response(resp.text)
self.assertEqual(len(events), 4)
self.assertEqual(events[0][1]['data'], 'stdout before start\n')
self.assertEqual(events[1][1]['data'], 'stdout mid 1\n')
self.assertEqual(events[2][1]['data'], 'stdout pre finish 1\n')
self.assertEqual(events[3][0], 'EOF')
listener.shutdown()
def setUp(self):
super(ExecutionPermissionsResolverTestCase, self).setUp()
# Create some mock users
user_1_db = UserDB(name='custom_role_unrelated_pack_action_grant')
user_1_db = User.add_or_update(user_1_db)
self.users['custom_role_unrelated_pack_action_grant'] = user_1_db
user_2_db = UserDB(name='custom_role_pack_action_grant_unrelated_permission')
user_2_db = User.add_or_update(user_2_db)
self.users['custom_role_pack_action_grant_unrelated_permission'] = user_2_db
user_3_db = UserDB(name='custom_role_pack_action_view_grant')
user_3_db = User.add_or_update(user_3_db)
self.users['custom_role_pack_action_view_grant'] = user_3_db
user_4_db = UserDB(name='custom_role_action_view_grant')
user_4_db = User.add_or_update(user_4_db)
self.users['custom_role_action_view_grant'] = user_4_db
user_5_db = UserDB(name='custom_role_pack_action_execute_grant')
user_5_db = User.add_or_update(user_5_db)
self.users['custom_role_pack_action_execute_grant'] = user_5_db
user_6_db = UserDB(name='custom_role_action_execute_grant')
user_6_db = User.add_or_update(user_6_db)
self.users['custom_role_action_execute_grant'] = user_6_db
user_7_db = UserDB(name='custom_role_pack_action_all_grant')
user_7_db = User.add_or_update(user_7_db)
self.users['custom_role_pack_action_all_grant'] = user_7_db
user_8_db = UserDB(name='custom_role_action_all_grant')
user_8_db = User.add_or_update(user_8_db)
self.users['custom_role_action_all_grant'] = user_8_db
# Create some mock resources on which permissions can be granted
action_1_db = ActionDB(pack='test_pack_2', name='action1', entry_point='',
runner_type={'name': 'run-local'})
action_1_db = Action.add_or_update(action_1_db)
self.resources['action_1'] = action_1_db
runner = {'name': 'run-python'}
liveaction = {'action': 'test_pack_2.action1'}
status = action_constants.LIVEACTION_STATUS_REQUESTED
action = {'uid': action_1_db.get_uid(), 'pack': 'test_pack_2'}
exec_1_db = ActionExecutionDB(action=action, runner=runner, liveaction=liveaction,
status=status)
exec_1_db = ActionExecution.add_or_update(exec_1_db)
self.resources['exec_1'] = exec_1_db
# Create some mock roles with associated permission grants
# Custom role - one grant to an unrelated pack
grant_db = PermissionGrantDB(resource_uid=self.resources['pack_1'].get_uid(),
resource_type=ResourceType.PACK,
permission_types=[PermissionType.ACTION_VIEW])
grant_db = PermissionGrant.add_or_update(grant_db)
permission_grants = [str(grant_db.id)]
role_db = RoleDB(name='custom_role_unrelated_pack_action_grant',
permission_grants=permission_grants)
role_db = Role.add_or_update(role_db)
self.roles['custom_role_unrelated_pack_action_grant'] = role_db
# Custom role - one grant of unrelated permission type to parent action pack
grant_db = PermissionGrantDB(resource_uid=self.resources['pack_2'].get_uid(),
resource_type=ResourceType.PACK,
permission_types=[PermissionType.RULE_VIEW])
grant_db = PermissionGrant.add_or_update(grant_db)
permission_grants = [str(grant_db.id)]
role_db = RoleDB(name='custom_role_pack_action_grant_unrelated_permission',
permission_grants=permission_grants)
role_db = Role.add_or_update(role_db)
self.roles['custom_role_pack_action_grant_unrelated_permission'] = role_db
# Custom role - one grant of "action_view" to the parent pack of the action the execution
# belongs to
grant_db = PermissionGrantDB(resource_uid=self.resources['pack_2'].get_uid(),
resource_type=ResourceType.PACK,
permission_types=[PermissionType.ACTION_VIEW])
grant_db = PermissionGrant.add_or_update(grant_db)
permission_grants = [str(grant_db.id)]
role_db = RoleDB(name='custom_role_pack_action_view_grant',
permission_grants=permission_grants)
role_db = Role.add_or_update(role_db)
self.roles['custom_role_pack_action_view_grant'] = role_db
# Custom role - one grant of "action_view" to the action the execution belongs to
grant_db = PermissionGrantDB(resource_uid=self.resources['action_1'].get_uid(),
resource_type=ResourceType.ACTION,
permission_types=[PermissionType.ACTION_VIEW])
grant_db = PermissionGrant.add_or_update(grant_db)
permission_grants = [str(grant_db.id)]
role_db = RoleDB(name='custom_role_action_view_grant',
permission_grants=permission_grants)
role_db = Role.add_or_update(role_db)
self.roles['custom_role_action_view_grant'] = role_db
# Custom role - one grant of "action_execute" to the parent pack of the action the
# execution belongs to
grant_db = PermissionGrantDB(resource_uid=self.resources['pack_2'].get_uid(),
resource_type=ResourceType.PACK,
permission_types=[PermissionType.ACTION_EXECUTE])
grant_db = PermissionGrant.add_or_update(grant_db)
permission_grants = [str(grant_db.id)]
role_db = RoleDB(name='custom_role_pack_action_execute_grant',
permission_grants=permission_grants)
role_db = Role.add_or_update(role_db)
self.roles['custom_role_pack_action_execute_grant'] = role_db
# Custom role - one grant of "action_execute" to the the action the execution belongs to
grant_db = PermissionGrantDB(resource_uid=self.resources['action_1'].get_uid(),
resource_type=ResourceType.ACTION,
permission_types=[PermissionType.ACTION_EXECUTE])
grant_db = PermissionGrant.add_or_update(grant_db)
permission_grants = [str(grant_db.id)]
role_db = RoleDB(name='custom_role_action_execute_grant',
permission_grants=permission_grants)
role_db = Role.add_or_update(role_db)
self.roles['custom_role_action_execute_grant'] = role_db
# Custom role - "action_all" grant on a parent action pack the execution belongs to
grant_db = PermissionGrantDB(resource_uid=self.resources['pack_2'].get_uid(),
resource_type=ResourceType.PACK,
permission_types=[PermissionType.ACTION_ALL])
grant_db = PermissionGrant.add_or_update(grant_db)
permission_grants = [str(grant_db.id)]
role_4_db = RoleDB(name='custom_role_pack_action_all_grant',
permission_grants=permission_grants)
role_4_db = Role.add_or_update(role_4_db)
self.roles['custom_role_pack_action_all_grant'] = role_4_db
# Custom role - "action_all" grant on action the execution belongs to
grant_db = PermissionGrantDB(resource_uid=self.resources['action_1'].get_uid(),
resource_type=ResourceType.ACTION,
permission_types=[PermissionType.ACTION_ALL])
grant_db = PermissionGrant.add_or_update(grant_db)
permission_grants = [str(grant_db.id)]
role_4_db = RoleDB(name='custom_role_action_all_grant',
permission_grants=permission_grants)
role_4_db = Role.add_or_update(role_4_db)
self.roles['custom_role_action_all_grant'] = role_4_db
# Create some mock role assignments
user_db = self.users['custom_role_unrelated_pack_action_grant']
role_assignment_db = UserRoleAssignmentDB(
user=user_db.name,
role=self.roles['custom_role_unrelated_pack_action_grant'].name)
UserRoleAssignment.add_or_update(role_assignment_db)
user_db = self.users['custom_role_pack_action_grant_unrelated_permission']
role_assignment_db = UserRoleAssignmentDB(
user=user_db.name,
role=self.roles['custom_role_pack_action_grant_unrelated_permission'].name)
UserRoleAssignment.add_or_update(role_assignment_db)
user_db = self.users['custom_role_pack_action_view_grant']
role_assignment_db = UserRoleAssignmentDB(
user=user_db.name,
role=self.roles['custom_role_pack_action_view_grant'].name)
UserRoleAssignment.add_or_update(role_assignment_db)
user_db = self.users['custom_role_action_view_grant']
role_assignment_db = UserRoleAssignmentDB(
user=user_db.name,
role=self.roles['custom_role_action_view_grant'].name)
UserRoleAssignment.add_or_update(role_assignment_db)
user_db = self.users['custom_role_pack_action_execute_grant']
role_assignment_db = UserRoleAssignmentDB(
user=user_db.name,
role=self.roles['custom_role_pack_action_execute_grant'].name)
UserRoleAssignment.add_or_update(role_assignment_db)
user_db = self.users['custom_role_action_execute_grant']
role_assignment_db = UserRoleAssignmentDB(
user=user_db.name,
role=self.roles['custom_role_action_execute_grant'].name)
UserRoleAssignment.add_or_update(role_assignment_db)
user_db = self.users['custom_role_pack_action_all_grant']
role_assignment_db = UserRoleAssignmentDB(
user=user_db.name,
role=self.roles['custom_role_pack_action_all_grant'].name)
UserRoleAssignment.add_or_update(role_assignment_db)
user_db = self.users['custom_role_action_all_grant']
role_assignment_db = UserRoleAssignmentDB(
user=user_db.name,
role=self.roles['custom_role_action_all_grant'].name)
UserRoleAssignment.add_or_update(role_assignment_db)
def test_migrate_executions(self):
ActionExecutionDB._meta["allow_inheritance"] = True
LiveActionDB._meta["allow_inheritance"] = True
class ActionExecutionDB_OldFieldType(ActionExecutionDB):
result = stormbase.EscapedDynamicField(default={})
class LiveActionDB_OldFieldType(LiveActionDB):
result = stormbase.EscapedDynamicField(default={})
execution_dbs = ActionExecution.query(
__raw__={"result": {
"$not": {
"$type": "binData",
},
}})
self.assertEqual(len(execution_dbs), 0)
execution_dbs = ActionExecution.query(__raw__={
"result": {
"$type": "object",
},
})
self.assertEqual(len(execution_dbs), 0)
# 1. Insert data in old format
liveaction_1_db = LiveActionDB_OldFieldType()
liveaction_1_db.action = "foo.bar"
liveaction_1_db.status = action_constants.LIVEACTION_STATUS_FAILED
liveaction_1_db.result = MOCK_RESULT_1
liveaction_1_db.start_timestamp = datetime.datetime.utcnow().replace(
tzinfo=datetime.timezone.utc)
liveaction_1_db = LiveAction.add_or_update(liveaction_1_db,
publish=False)
execution_1_db = ActionExecutionDB_OldFieldType()
execution_1_db.action = {"a": 1}
execution_1_db.runner = {"a": 1}
execution_1_db.liveaction = {"id": liveaction_1_db.id}
execution_1_db.status = action_constants.LIVEACTION_STATUS_FAILED
execution_1_db.result = MOCK_RESULT_1
execution_1_db.start_timestamp = datetime.datetime.utcnow().replace(
tzinfo=datetime.timezone.utc)
execution_1_db = ActionExecution.add_or_update(execution_1_db,
publish=False)
# This execution is not in a final state yet so it should not be migrated
liveaction_2_db = LiveActionDB_OldFieldType()
liveaction_2_db.action = "foo.bar2"
liveaction_2_db.status = action_constants.LIVEACTION_STATUS_RUNNING
liveaction_2_db.result = MOCK_RESULT_2
liveaction_2_db.start_timestamp = datetime.datetime.utcnow().replace(
tzinfo=datetime.timezone.utc)
liveaction_2_db = LiveAction.add_or_update(liveaction_2_db,
publish=False)
execution_2_db = ActionExecutionDB_OldFieldType()
execution_2_db.action = {"a": 2}
execution_2_db.runner = {"a": 2}
execution_2_db.liveaction = {"id": liveaction_2_db.id}
execution_2_db.status = action_constants.LIVEACTION_STATUS_RUNNING
execution_2_db.result = MOCK_RESULT_2
execution_2_db.start_timestamp = datetime.datetime.utcnow().replace(
tzinfo=datetime.timezone.utc)
execution_2_db = ActionExecution.add_or_update(execution_2_db,
publish=False)
# This object is older than the default threshold so it should not be migrated
execution_3_db = ActionExecutionDB_OldFieldType()
execution_3_db.action = {"a": 2}
execution_3_db.runner = {"a": 2}
execution_3_db.liveaction = {"id": liveaction_2_db.id}
execution_3_db.status = action_constants.LIVEACTION_STATUS_SUCCEEDED
execution_3_db.result = MOCK_RESULT_1
execution_3_db.start_timestamp = datetime.datetime.utcfromtimestamp(
0).replace(tzinfo=datetime.timezone.utc)
execution_3_db = ActionExecution.add_or_update(execution_3_db,
publish=False)
# Verify data has been inserted in old format
execution_dbs = ActionExecution.query(__raw__={
"result": {
"$type": "object",
},
})
self.assertEqual(len(execution_dbs), 3)
execution_dbs = ActionExecution.query(
__raw__={"result": {
"$not": {
"$type": "binData",
},
}})
self.assertEqual(len(execution_dbs), 3)
execution_dbs = ActionExecution.query(__raw__={
"result": {
"$type": "binData",
},
})
self.assertEqual(len(execution_dbs), 0)
liveaction_dbs = LiveAction.query(__raw__={
"result": {
"$type": "object",
},
})
self.assertEqual(len(liveaction_dbs), 2)
liveaction_dbs = LiveAction.query(
__raw__={"result": {
"$not": {
"$type": "binData",
},
}})
self.assertEqual(len(liveaction_dbs), 2)
liveaction_dbs = LiveAction.query(__raw__={
"result": {
"$type": "binData",
},
})
self.assertEqual(len(liveaction_dbs), 0)
# Update inserted documents and remove special _cls field added by mongoengine. We need to
# do that here due to how mongoengine works with subclasses.
ActionExecution.query(__raw__={
"result": {
"$type": "object",
},
}).update(set___cls="ActionExecutionDB")
LiveAction.query(__raw__={
"result": {
"$type": "object",
},
}).update(set___cls="LiveActionDB")
# 2. Run migration
start_dt = datetime.datetime.utcnow().replace(
tzinfo=datetime.timezone.utc) - datetime.timedelta(hours=2)
end_dt = datetime.datetime.utcnow().replace(
tzinfo=datetime.timezone.utc)
migration_module.migrate_executions(start_dt=start_dt, end_dt=end_dt)
# 3. Verify data has been migrated - only 1 item should have been migrated since it's in a
# completed state
execution_dbs = ActionExecution.query(__raw__={
"result": {
"$type": "object",
},
})
self.assertEqual(len(execution_dbs), 2)
execution_dbs = ActionExecution.query(__raw__={
"result": {
"$type": "binData",
},
})
self.assertEqual(len(execution_dbs), 1)
execution_db_1_retrieved = ActionExecution.get_by_id(execution_1_db.id)
self.assertEqual(execution_db_1_retrieved.result, MOCK_RESULT_1)
execution_db_2_retrieved = ActionExecution.get_by_id(execution_2_db.id)
self.assertEqual(execution_db_2_retrieved.result, MOCK_RESULT_2)
liveaction_db_1_retrieved = LiveAction.get_by_id(liveaction_1_db.id)
self.assertEqual(liveaction_db_1_retrieved.result, MOCK_RESULT_1)
liveaction_db_2_retrieved = LiveAction.get_by_id(liveaction_2_db.id)
self.assertEqual(liveaction_db_2_retrieved.result, MOCK_RESULT_2)
