def test_liveaction_gets_deleted(self): now = date_utils.get_datetime_utc_now() start_ts = now - timedelta(days=15) end_ts = now - timedelta(days=14) liveaction_model = copy.deepcopy(self.models['liveactions']['liveaction4.yaml']) liveaction_model['start_timestamp'] = start_ts liveaction_model['end_timestamp'] = end_ts liveaction_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED liveaction = LiveAction.add_or_update(liveaction_model) # Write one execution before cut-off threshold exec_model = copy.deepcopy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['end_timestamp'] = end_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED exec_model['id'] = bson.ObjectId() exec_model['liveaction']['id'] = str(liveaction.id) ActionExecution.add_or_update(exec_model) liveactions = LiveAction.get_all() executions = ActionExecution.get_all() self.assertEqual(len(liveactions), 1) self.assertEqual(len(executions), 1) purge_executions(logger=LOG, timestamp=now - timedelta(days=10)) liveactions = LiveAction.get_all() executions = ActionExecution.get_all() self.assertEqual(len(executions), 0) self.assertEqual(len(liveactions), 0)
def test_crud_complete(self): # Create the DB record. obj = ActionExecutionAPI(**copy.deepcopy(self.fake_history_workflow)) ActionExecution.add_or_update(ActionExecutionAPI.to_model(obj)) model = ActionExecution.get_by_id(obj.id) self.assertEqual(str(model.id), obj.id) self.assertDictEqual(model.trigger, self.fake_history_workflow['trigger']) self.assertDictEqual(model.trigger_type, self.fake_history_workflow['trigger_type']) self.assertDictEqual(model.trigger_instance, self.fake_history_workflow['trigger_instance']) self.assertDictEqual(model.rule, self.fake_history_workflow['rule']) self.assertDictEqual(model.action, self.fake_history_workflow['action']) self.assertDictEqual(model.runner, self.fake_history_workflow['runner']) doc = copy.deepcopy(self.fake_history_workflow['liveaction']) doc['start_timestamp'] = doc['start_timestamp'] doc['end_timestamp'] = doc['end_timestamp'] self.assertDictEqual(model.liveaction, doc) self.assertIsNone(getattr(model, 'parent', None)) self.assertListEqual(model.children, self.fake_history_workflow['children']) # Update the DB record. children = [str(bson.ObjectId()), str(bson.ObjectId())] model.children = children ActionExecution.add_or_update(model) model = ActionExecution.get_by_id(obj.id) self.assertListEqual(model.children, children) # Delete the DB record. ActionExecution.delete(model) self.assertRaises(ValueError, ActionExecution.get_by_id, obj.id)
def test_no_timestamp_doesnt_delete_things(self): now = date_utils.get_datetime_utc_now() exec_model = copy.deepcopy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = now - timedelta(days=15) exec_model['end_timestamp'] = now - timedelta(days=14) exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) # Insert corresponding stdout and stderr db mock models self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=3) execs = ActionExecution.get_all() self.assertEqual(len(execs), 1) stdout_dbs = ActionExecutionOutput.query(output_type='stdout') self.assertEqual(len(stdout_dbs), 3) stderr_dbs = ActionExecutionOutput.query(output_type='stderr') self.assertEqual(len(stderr_dbs), 3) expected_msg = 'Specify a valid timestamp' self.assertRaisesRegexp(ValueError, expected_msg, purge_executions, logger=LOG, timestamp=None) execs = ActionExecution.get_all() self.assertEqual(len(execs), 1) stdout_dbs = ActionExecutionOutput.query(output_type='stdout') self.assertEqual(len(stdout_dbs), 3) stderr_dbs = ActionExecutionOutput.query(output_type='stderr') self.assertEqual(len(stderr_dbs), 3)
def test_crud_partial(self): # Create the DB record. obj = ActionExecutionAPI( **copy.deepcopy(self.fake_history_subtasks[0])) ActionExecution.add_or_update(ActionExecutionAPI.to_model(obj)) model = ActionExecution.get_by_id(obj.id) self.assertEqual(str(model.id), obj.id) self.assertDictEqual(model.trigger, {}) self.assertDictEqual(model.trigger_type, {}) self.assertDictEqual(model.trigger_instance, {}) self.assertDictEqual(model.rule, {}) self.assertDictEqual(model.action, self.fake_history_subtasks[0]['action']) self.assertDictEqual(model.runner, self.fake_history_subtasks[0]['runner']) doc = copy.deepcopy(self.fake_history_subtasks[0]['liveaction']) doc['start_timestamp'] = doc['start_timestamp'] doc['end_timestamp'] = doc['end_timestamp'] self.assertDictEqual(model.liveaction, doc) self.assertEqual(model.parent, self.fake_history_subtasks[0]['parent']) self.assertListEqual(model.children, []) # Update the DB record. children = [str(bson.ObjectId()), str(bson.ObjectId())] model.children = children ActionExecution.add_or_update(model) model = ActionExecution.get_by_id(obj.id) self.assertListEqual(model.children, children) # Delete the DB record. ActionExecution.delete(model) self.assertRaises(StackStormDBObjectNotFoundError, ActionExecution.get_by_id, obj.id)
def test_no_timestamp_doesnt_delete_things(self): now = date_utils.get_datetime_utc_now() exec_model = copy.deepcopy( self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = now - timedelta(days=15) exec_model['end_timestamp'] = now - timedelta(days=14) exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) # Insert corresponding stdout and stderr db mock models self._insert_mock_stdout_and_stderr_objects_for_execution( exec_model['id'], count=3) execs = ActionExecution.get_all() self.assertEqual(len(execs), 1) stdout_dbs = ActionExecutionOutput.query(output_type='stdout') self.assertEqual(len(stdout_dbs), 3) stderr_dbs = ActionExecutionOutput.query(output_type='stderr') self.assertEqual(len(stderr_dbs), 3) expected_msg = 'Specify a valid timestamp' self.assertRaisesRegexp(ValueError, expected_msg, purge_executions, logger=LOG, timestamp=None) execs = ActionExecution.get_all() self.assertEqual(len(execs), 1) stdout_dbs = ActionExecutionOutput.query(output_type='stdout') self.assertEqual(len(stdout_dbs), 3) stderr_dbs = ActionExecutionOutput.query(output_type='stderr') self.assertEqual(len(stderr_dbs), 3)
def test_liveaction_gets_deleted(self): now = date_utils.get_datetime_utc_now() start_ts = now - timedelta(days=15) end_ts = now - timedelta(days=14) liveaction_model = copy.deepcopy( self.models['liveactions']['liveaction4.yaml']) liveaction_model['start_timestamp'] = start_ts liveaction_model['end_timestamp'] = end_ts liveaction_model[ 'status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED liveaction = LiveAction.add_or_update(liveaction_model) # Write one execution before cut-off threshold exec_model = copy.deepcopy( self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['end_timestamp'] = end_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED exec_model['id'] = bson.ObjectId() exec_model['liveaction']['id'] = str(liveaction.id) ActionExecution.add_or_update(exec_model) liveactions = LiveAction.get_all() executions = ActionExecution.get_all() self.assertEqual(len(liveactions), 1) self.assertEqual(len(executions), 1) purge_executions(logger=LOG, timestamp=now - timedelta(days=10)) liveactions = LiveAction.get_all() executions = ActionExecution.get_all() self.assertEqual(len(executions), 0) self.assertEqual(len(liveactions), 0)
def test_crud_partial(self): # Create the DB record. obj = ActionExecutionAPI(**copy.deepcopy(self.fake_history_subtasks[0])) ActionExecution.add_or_update(ActionExecutionAPI.to_model(obj)) model = ActionExecution.get_by_id(obj.id) self.assertEqual(str(model.id), obj.id) self.assertDictEqual(model.trigger, {}) self.assertDictEqual(model.trigger_type, {}) self.assertDictEqual(model.trigger_instance, {}) self.assertDictEqual(model.rule, {}) self.assertDictEqual(model.action, self.fake_history_subtasks[0]['action']) self.assertDictEqual(model.runner, self.fake_history_subtasks[0]['runner']) doc = copy.deepcopy(self.fake_history_subtasks[0]['liveaction']) doc['start_timestamp'] = doc['start_timestamp'] doc['end_timestamp'] = doc['end_timestamp'] self.assertDictEqual(model.liveaction, doc) self.assertEqual(model.parent, self.fake_history_subtasks[0]['parent']) self.assertListEqual(model.children, []) # Update the DB record. children = [str(bson.ObjectId()), str(bson.ObjectId())] model.children = children ActionExecution.add_or_update(model) model = ActionExecution.get_by_id(obj.id) self.assertListEqual(model.children, children) # Delete the DB record. ActionExecution.delete(model) self.assertRaises(StackStormDBObjectNotFoundError, ActionExecution.get_by_id, obj.id)
def test_purge_executions_with_timestamp(self): now = date_utils.get_datetime_utc_now() # Write one execution after cut-off threshold exec_model = copy.deepcopy( self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = now - timedelta(days=15) exec_model['end_timestamp'] = now - timedelta(days=14) exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) # Write one execution before cut-off threshold exec_model = copy.deepcopy( self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = now - timedelta(days=22) exec_model['end_timestamp'] = now - timedelta(days=21) exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) execs = ActionExecution.get_all() purge_executions(logger=LOG, timestamp=now - timedelta(days=20)) execs = ActionExecution.get_all() self.assertEqual(len(execs), 1)
def create_execution_object(liveaction, publish=True): action_db = action_utils.get_action_by_ref(liveaction.action) runner = RunnerType.get_by_name(action_db.runner_type['name']) attrs = { 'action': vars(ActionAPI.from_model(action_db)), 'parameters': liveaction['parameters'], 'runner': vars(RunnerTypeAPI.from_model(runner)) } attrs.update(_decompose_liveaction(liveaction)) if 'rule' in liveaction.context: rule = reference.get_model_from_ref(Rule, liveaction.context.get('rule', {})) attrs['rule'] = vars(RuleAPI.from_model(rule)) if 'trigger_instance' in liveaction.context: trigger_instance_id = liveaction.context.get('trigger_instance', {}) trigger_instance_id = trigger_instance_id.get('id', None) trigger_instance = TriggerInstance.get_by_id(trigger_instance_id) trigger = reference.get_model_by_resource_ref( db_api=Trigger, ref=trigger_instance.trigger) trigger_type = reference.get_model_by_resource_ref(db_api=TriggerType, ref=trigger.type) trigger_instance = reference.get_model_from_ref( TriggerInstance, liveaction.context.get('trigger_instance', {})) attrs['trigger_instance'] = vars( TriggerInstanceAPI.from_model(trigger_instance)) attrs['trigger'] = vars(TriggerAPI.from_model(trigger)) attrs['trigger_type'] = vars(TriggerTypeAPI.from_model(trigger_type)) parent = _get_parent_execution(liveaction) if parent: attrs['parent'] = str(parent.id) attrs['log'] = [_create_execution_log_entry(liveaction['status'])] execution = ActionExecutionDB(**attrs) execution = ActionExecution.add_or_update(execution, publish=False) # Update the web_url field in execution. Unfortunately, we need # the execution id for constructing the URL which we only get # after the model is written to disk. execution.web_url = _get_web_url_for_execution(str(execution.id)) execution = ActionExecution.add_or_update(execution, publish=publish) if parent: if str(execution.id) not in parent.children: parent.children.append(str(execution.id)) ActionExecution.add_or_update(parent) return execution
def create_execution_object(liveaction, publish=True): action_db = action_utils.get_action_by_ref(liveaction.action) runner = RunnerType.get_by_name(action_db.runner_type['name']) attrs = { 'action': vars(ActionAPI.from_model(action_db)), 'parameters': liveaction['parameters'], 'runner': vars(RunnerTypeAPI.from_model(runner)) } attrs.update(_decompose_liveaction(liveaction)) if 'rule' in liveaction.context: rule = reference.get_model_from_ref(Rule, liveaction.context.get('rule', {})) attrs['rule'] = vars(RuleAPI.from_model(rule)) if 'trigger_instance' in liveaction.context: trigger_instance_id = liveaction.context.get('trigger_instance', {}) trigger_instance_id = trigger_instance_id.get('id', None) trigger_instance = TriggerInstance.get_by_id(trigger_instance_id) trigger = reference.get_model_by_resource_ref(db_api=Trigger, ref=trigger_instance.trigger) trigger_type = reference.get_model_by_resource_ref(db_api=TriggerType, ref=trigger.type) trigger_instance = reference.get_model_from_ref( TriggerInstance, liveaction.context.get('trigger_instance', {})) attrs['trigger_instance'] = vars(TriggerInstanceAPI.from_model(trigger_instance)) attrs['trigger'] = vars(TriggerAPI.from_model(trigger)) attrs['trigger_type'] = vars(TriggerTypeAPI.from_model(trigger_type)) parent = _get_parent_execution(liveaction) if parent: attrs['parent'] = str(parent.id) attrs['log'] = [_create_execution_log_entry(liveaction['status'])] execution = ActionExecutionDB(**attrs) execution = ActionExecution.add_or_update(execution, publish=False) # Update the web_url field in execution. Unfortunately, we need # the execution id for constructing the URL which we only get # after the model is written to disk. execution.web_url = _get_web_url_for_execution(str(execution.id)) execution = ActionExecution.add_or_update(execution, publish=publish) if parent: if str(execution.id) not in parent.children: parent.children.append(str(execution.id)) ActionExecution.add_or_update(parent) return execution
def test_no_timestamp_doesnt_delete_things(self): now = date_utils.get_datetime_utc_now() exec_model = copy.copy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = now - timedelta(days=15) exec_model['end_timestamp'] = now - timedelta(days=14) exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) execs = ActionExecution.get_all() self.assertEqual(len(execs), 1) purge_executions() execs = ActionExecution.get_all() self.assertEqual(len(execs), 1)
def test_garbage_collection(self): now = date_utils.get_datetime_utc_now() status = action_constants.LIVEACTION_STATUS_SUCCEEDED # Insert come mock ActionExecutionDB objects with start_timestamp < TTL defined in the # config old_executions_count = 15 ttl_days = 30 timestamp = (now - datetime.timedelta(days=ttl_days)) for index in range(0, old_executions_count): action_execution_db = ActionExecutionDB( start_timestamp=timestamp, end_timestamp=timestamp, status=status, action={'ref': 'core.local'}, runner={'name': 'run-local'}, liveaction={'ref': 'foo'}) ActionExecution.add_or_update(action_execution_db) # Insert come mock ActionExecutionDB objects with start_timestamp > TTL defined in the # config new_executions_count = 5 ttl_days = 2 timestamp = (now - datetime.timedelta(days=ttl_days)) for index in range(0, new_executions_count): action_execution_db = ActionExecutionDB( start_timestamp=timestamp, end_timestamp=timestamp, status=status, action={'ref': 'core.local'}, runner={'name': 'run-local'}, liveaction={'ref': 'foo'}) ActionExecution.add_or_update(action_execution_db) execs = ActionExecution.get_all() self.assertEqual(len(execs), (old_executions_count + new_executions_count)) # Start garbage collector process = self._start_garbage_collector() # Give it some time to perform garbage collection and kill it eventlet.sleep(5) process.send_signal(signal.SIGKILL) self.remove_process(process=process) # Old execution should have been garbage collected execs = ActionExecution.get_all() self.assertEqual(len(execs), (new_executions_count))
def test_purge_incomplete(self): now = date_utils.get_datetime_utc_now() start_ts = now - timedelta(days=15) # Write executions before cut-off threshold exec_model = copy.deepcopy( self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_SCHEDULED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) exec_model = copy.deepcopy( self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_RUNNING exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) exec_model = copy.deepcopy( self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_DELAYED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) exec_model = copy.deepcopy( self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_CANCELING exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) exec_model = copy.deepcopy( self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_REQUESTED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) self.assertEqual(len(ActionExecution.get_all()), 5) purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=False) self.assertEqual(len(ActionExecution.get_all()), 5) purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=True) self.assertEqual(len(ActionExecution.get_all()), 0)
def assign_parent(child): candidates = [v for k, v in cls.refs.iteritems() if v.action["name"] == "chain"] if candidates: parent = random.choice(candidates) child["parent"] = str(parent.id) parent.children.append(child["id"]) cls.refs[str(parent.id)] = ActionExecution.add_or_update(parent)
def test_update_execution(self): """Test ActionExecutionDb update """ self.assertTrue(self.executions['execution_1'].end_timestamp is None) self.executions['execution_1'].end_timestamp = date_utils.get_datetime_utc_now() updated = ActionExecution.add_or_update(self.executions['execution_1']) self.assertTrue(updated.end_timestamp == self.executions['execution_1'].end_timestamp)
def assign_parent(child): candidates = [v for k, v in cls.refs.items() if v.action["name"] == "chain"] if candidates: parent = random.choice(candidates) child["parent"] = str(parent.id) parent.children.append(child["id"]) cls.refs[str(parent.id)] = ActionExecution.add_or_update(parent)
def assign_parent(child): candidates = [v for k, v in cls.refs.items() if v.action['name'] == 'chain'] if candidates: parent = random.choice(candidates) child['parent'] = str(parent.id) parent.children.append(child['id']) cls.refs[str(parent.id)] = ActionExecution.add_or_update(parent)
def update_execution(liveaction_db, publish=True): execution = ActionExecution.get(liveaction__id=str(liveaction_db.id)) decomposed = _decompose_liveaction(liveaction_db) for k, v in six.iteritems(decomposed): setattr(execution, k, v) execution = ActionExecution.add_or_update(execution, publish=publish) return execution
def test_get_output_finished_execution(self): # Test the execution output API endpoint for execution which has finished for status in action_constants.LIVEACTION_COMPLETED_STATES: # Insert mock execution and output objects status = action_constants.LIVEACTION_STATUS_SUCCEEDED timestamp = date_utils.get_datetime_utc_now() action_execution_db = ActionExecutionDB( start_timestamp=timestamp, end_timestamp=timestamp, status=status, action={"ref": "core.local"}, runner={"name": "local-shell-cmd"}, liveaction={"ref": "foo"}, ) action_execution_db = ActionExecution.add_or_update( action_execution_db) for i in range(1, 6): stdout_db = ActionExecutionOutputDB( execution_id=str(action_execution_db.id), action_ref="core.local", runner_ref="dummy", timestamp=timestamp, output_type="stdout", data="stdout %s\n" % (i), ) ActionExecutionOutput.add_or_update(stdout_db) for i in range(10, 15): stderr_db = ActionExecutionOutputDB( execution_id=str(action_execution_db.id), action_ref="core.local", runner_ref="dummy", timestamp=timestamp, output_type="stderr", data="stderr %s\n" % (i), ) ActionExecutionOutput.add_or_update(stderr_db) resp = self.app.get( "/v1/executions/%s/output" % (str(action_execution_db.id)), expect_errors=False, ) self.assertEqual(resp.status_int, 200) events = self._parse_response(resp.text) self.assertEqual(len(events), 11) self.assertEqual(events[0][1]["data"], "stdout 1\n") self.assertEqual(events[9][1]["data"], "stderr 14\n") self.assertEqual(events[10][0], "EOF") # Verify "last" short-hand id works resp = self.app.get("/v1/executions/last/output", expect_errors=False) self.assertEqual(resp.status_int, 200) events = self._parse_response(resp.text) self.assertEqual(len(events), 11) self.assertEqual(events[10][0], "EOF")
def test_garbage_collection(self): now = date_utils.get_datetime_utc_now() status = action_constants.LIVEACTION_STATUS_SUCCEEDED # Insert come mock ActionExecutionDB objects with start_timestamp < TTL defined in the # config old_executions_count = 15 ttl_days = 30 timestamp = (now - datetime.timedelta(days=ttl_days)) for index in range(0, old_executions_count): action_execution_db = ActionExecutionDB(start_timestamp=timestamp, end_timestamp=timestamp, status=status, action={'ref': 'core.local'}, runner={'name': 'run-local'}, liveaction={'ref': 'foo'}) ActionExecution.add_or_update(action_execution_db) # Insert come mock ActionExecutionDB objects with start_timestamp > TTL defined in the # config new_executions_count = 5 ttl_days = 2 timestamp = (now - datetime.timedelta(days=ttl_days)) for index in range(0, new_executions_count): action_execution_db = ActionExecutionDB(start_timestamp=timestamp, end_timestamp=timestamp, status=status, action={'ref': 'core.local'}, runner={'name': 'run-local'}, liveaction={'ref': 'foo'}) ActionExecution.add_or_update(action_execution_db) execs = ActionExecution.get_all() self.assertEqual(len(execs), (old_executions_count + new_executions_count)) # Start garbage collector process = self._start_garbage_collector() # Give it some time to perform garbage collection and kill it eventlet.sleep(5) process.send_signal(signal.SIGKILL) self.remove_process(process=process) # Old execution should have been garbage collected execs = ActionExecution.get_all() self.assertEqual(len(execs), (new_executions_count))
def test_datetime_range(self): base = date_utils.add_utc_tz(datetime.datetime(2014, 12, 25, 0, 0, 0)) for i in range(60): timestamp = base + datetime.timedelta(seconds=i) doc = copy.deepcopy(self.fake_history_subtasks[0]) doc['id'] = str(bson.ObjectId()) doc['start_timestamp'] = isotime.format(timestamp) obj = ActionExecutionAPI(**doc) ActionExecution.add_or_update(ActionExecutionAPI.to_model(obj)) dt_range = '2014-12-25T00:00:10Z..2014-12-25T00:00:19Z' objs = ActionExecution.query(start_timestamp=dt_range) self.assertEqual(len(objs), 10) dt_range = '2014-12-25T00:00:19Z..2014-12-25T00:00:10Z' objs = ActionExecution.query(start_timestamp=dt_range) self.assertEqual(len(objs), 10)
def test_no_timestamp_doesnt_delete_things(self): now = date_utils.get_datetime_utc_now() exec_model = copy.deepcopy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = now - timedelta(days=15) exec_model['end_timestamp'] = now - timedelta(days=14) exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) execs = ActionExecution.get_all() self.assertEqual(len(execs), 1) expected_msg = 'Specify a valid timestamp' self.assertRaisesRegexp(ValueError, expected_msg, purge_executions, logger=LOG, timestamp=None) execs = ActionExecution.get_all() self.assertEqual(len(execs), 1)
def publish_action_finished(action_execution_db): # Insert mock output object output_params['data'] = 'stdout pre finish 1\n' output_db = ActionExecutionOutputDB(**output_params) ActionExecutionOutput.add_or_update(output_db) # Transition execution to completed state so the connection closes action_execution_db.status = action_constants.LIVEACTION_STATUS_SUCCEEDED action_execution_db = ActionExecution.add_or_update(action_execution_db)
def test_purge_executions_with_action_ref(self): now = date_utils.get_datetime_utc_now() exec_model = copy.deepcopy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = now - timedelta(days=15) exec_model['end_timestamp'] = now - timedelta(days=14) exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) execs = ActionExecution.get_all() self.assertEqual(len(execs), 1) purge_executions(logger=LOG, action_ref='core.localzzz', timestamp=now - timedelta(days=10)) execs = ActionExecution.get_all() self.assertEqual(len(execs), 1) purge_executions(logger=LOG, action_ref='core.local', timestamp=now - timedelta(days=10)) execs = ActionExecution.get_all() self.assertEqual(len(execs), 0)
def test_purge_executions_with_action_ref(self): now = date_utils.get_datetime_utc_now() exec_model = copy.copy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = now - timedelta(days=15) exec_model['end_timestamp'] = now - timedelta(days=14) exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) execs = ActionExecution.get_all() self.assertEqual(len(execs), 1) purge_executions(action_ref='core.localzzz', timestamp=now - timedelta(days=10)) execs = ActionExecution.get_all() self.assertEqual(len(execs), 1) purge_executions(action_ref='core.local', timestamp=now - timedelta(days=10)) execs = ActionExecution.get_all() self.assertEqual(len(execs), 0)
def setUpClass(cls): super(TestActionExecutionFilters, cls).setUpClass() cls.dt_base = date_utils.add_utc_tz(datetime.datetime(2014, 12, 25, 0, 0, 0)) cls.num_records = 100 cls.refs = {} cls.start_timestamps = [] cls.fake_types = [ { "trigger": copy.deepcopy(fixture.ARTIFACTS["trigger"]), "trigger_type": copy.deepcopy(fixture.ARTIFACTS["trigger_type"]), "trigger_instance": copy.deepcopy( fixture.ARTIFACTS["trigger_instance"] ), "rule": copy.deepcopy(fixture.ARTIFACTS["rule"]), "action": copy.deepcopy(fixture.ARTIFACTS["actions"]["chain"]), "runner": copy.deepcopy(fixture.ARTIFACTS["runners"]["action-chain"]), "liveaction": copy.deepcopy( fixture.ARTIFACTS["liveactions"]["workflow"] ), "context": copy.deepcopy(fixture.ARTIFACTS["context"]), "children": [], }, { "action": copy.deepcopy(fixture.ARTIFACTS["actions"]["local"]), "runner": copy.deepcopy(fixture.ARTIFACTS["runners"]["run-local"]), "liveaction": copy.deepcopy(fixture.ARTIFACTS["liveactions"]["task1"]), }, ] def assign_parent(child): candidates = [v for k, v in cls.refs.items() if v.action["name"] == "chain"] if candidates: parent = random.choice(candidates) child["parent"] = str(parent.id) parent.children.append(child["id"]) cls.refs[str(parent.id)] = ActionExecution.add_or_update(parent) for i in range(cls.num_records): obj_id = str(bson.ObjectId()) timestamp = cls.dt_base + datetime.timedelta(seconds=i) fake_type = random.choice(cls.fake_types) data = copy.deepcopy(fake_type) data["id"] = obj_id data["start_timestamp"] = isotime.format(timestamp, offset=False) data["end_timestamp"] = isotime.format(timestamp, offset=False) data["status"] = data["liveaction"]["status"] data["result"] = data["liveaction"]["result"] if fake_type["action"]["name"] == "local" and random.choice([True, False]): assign_parent(data) wb_obj = ActionExecutionAPI(**data) db_obj = ActionExecutionAPI.to_model(wb_obj) cls.refs[obj_id] = ActionExecution.add_or_update(db_obj) cls.start_timestamps.append(timestamp) cls.start_timestamps = sorted(cls.start_timestamps)
def create_execution_object(liveaction, publish=True): action_db = action_utils.get_action_by_ref(liveaction.action) runner = RunnerType.get_by_name(action_db.runner_type['name']) attrs = { 'action': vars(ActionAPI.from_model(action_db)), 'runner': vars(RunnerTypeAPI.from_model(runner)) } attrs.update(_decompose_liveaction(liveaction)) if 'rule' in liveaction.context: rule = reference.get_model_from_ref(Rule, liveaction.context.get('rule', {})) attrs['rule'] = vars(RuleAPI.from_model(rule)) if 'trigger_instance' in liveaction.context: trigger_instance_id = liveaction.context.get('trigger_instance', {}) trigger_instance_id = trigger_instance_id.get('id', None) trigger_instance = TriggerInstance.get_by_id(trigger_instance_id) trigger = reference.get_model_by_resource_ref( db_api=Trigger, ref=trigger_instance.trigger) trigger_type = reference.get_model_by_resource_ref(db_api=TriggerType, ref=trigger.type) trigger_instance = reference.get_model_from_ref( TriggerInstance, liveaction.context.get('trigger_instance', {})) attrs['trigger_instance'] = vars( TriggerInstanceAPI.from_model(trigger_instance)) attrs['trigger'] = vars(TriggerAPI.from_model(trigger)) attrs['trigger_type'] = vars(TriggerTypeAPI.from_model(trigger_type)) parent = ActionExecution.get( liveaction__id=liveaction.context.get('parent', '')) if parent: attrs['parent'] = str(parent.id) execution = ActionExecutionDB(**attrs) execution = ActionExecution.add_or_update(execution, publish=publish) if parent: if str(execution.id) not in parent.children: parent.children.append(str(execution.id)) ActionExecution.add_or_update(parent) return execution
def test_purge_executions_with_timestamp(self): now = date_utils.get_datetime_utc_now() # Write one execution after cut-off threshold exec_model = copy.deepcopy( self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = now - timedelta(days=15) exec_model['end_timestamp'] = now - timedelta(days=14) exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) # Insert corresponding stdout and stderr db mock models self._insert_mock_stdout_and_stderr_objects_for_execution( exec_model['id'], count=3) # Write one execution before cut-off threshold exec_model = copy.deepcopy( self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = now - timedelta(days=22) exec_model['end_timestamp'] = now - timedelta(days=21) exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) # Insert corresponding stdout and stderr db mock models self._insert_mock_stdout_and_stderr_objects_for_execution( exec_model['id'], count=3) execs = ActionExecution.get_all() self.assertEqual(len(execs), 2) stdout_dbs = ActionExecutionOutput.query(output_type='stdout') self.assertEqual(len(stdout_dbs), 6) stderr_dbs = ActionExecutionOutput.query(output_type='stderr') self.assertEqual(len(stderr_dbs), 6) purge_executions(logger=LOG, timestamp=now - timedelta(days=20)) execs = ActionExecution.get_all() self.assertEqual(len(execs), 1) stdout_dbs = ActionExecutionOutput.query(output_type='stdout') self.assertEqual(len(stdout_dbs), 3) stderr_dbs = ActionExecutionOutput.query(output_type='stderr') self.assertEqual(len(stderr_dbs), 3)
def test_purge_executions_with_action_ref(self): now = date_utils.get_datetime_utc_now() exec_model = copy.deepcopy( self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = now - timedelta(days=15) exec_model['end_timestamp'] = now - timedelta(days=14) exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) # Insert corresponding stdout and stderr db mock models self._insert_mock_stdout_and_stderr_objects_for_execution( exec_model['id'], count=3) execs = ActionExecution.get_all() self.assertEqual(len(execs), 1) stdout_dbs = ActionExecutionOutput.query(output_type='stdout') self.assertEqual(len(stdout_dbs), 3) stderr_dbs = ActionExecutionOutput.query(output_type='stderr') self.assertEqual(len(stderr_dbs), 3) # Invalid action reference, nothing should be deleted purge_executions(logger=LOG, action_ref='core.localzzz', timestamp=now - timedelta(days=10)) execs = ActionExecution.get_all() self.assertEqual(len(execs), 1) stdout_dbs = ActionExecutionOutput.query(output_type='stdout') self.assertEqual(len(stdout_dbs), 3) stderr_dbs = ActionExecutionOutput.query(output_type='stderr') self.assertEqual(len(stderr_dbs), 3) purge_executions(logger=LOG, action_ref='core.local', timestamp=now - timedelta(days=10)) execs = ActionExecution.get_all() self.assertEqual(len(execs), 0) stdout_dbs = ActionExecutionOutput.query(output_type='stdout') self.assertEqual(len(stdout_dbs), 0) stderr_dbs = ActionExecutionOutput.query(output_type='stderr') self.assertEqual(len(stderr_dbs), 0)
def test_sort_by_start_timestamp(self): base = date_utils.add_utc_tz(datetime.datetime(2014, 12, 25, 0, 0, 0)) for i in range(60): timestamp = base + datetime.timedelta(seconds=i) doc = copy.deepcopy(self.fake_history_subtasks[0]) doc["id"] = str(bson.ObjectId()) doc["start_timestamp"] = isotime.format(timestamp) obj = ActionExecutionAPI(**doc) ActionExecution.add_or_update(ActionExecutionAPI.to_model(obj)) dt_range = "2014-12-25T00:00:10Z..2014-12-25T00:00:19Z" objs = ActionExecution.query(start_timestamp=dt_range, order_by=["start_timestamp"]) self.assertLess(objs[0]["start_timestamp"], objs[9]["start_timestamp"]) dt_range = "2014-12-25T00:00:19Z..2014-12-25T00:00:10Z" objs = ActionExecution.query(start_timestamp=dt_range, order_by=["-start_timestamp"]) self.assertLess(objs[9]["start_timestamp"], objs[0]["start_timestamp"])
def test_get_output_finished_execution(self): # Test the execution output API endpoint for execution which has finished for status in action_constants.LIVEACTION_COMPLETED_STATES: # Insert mock execution and output objects status = action_constants.LIVEACTION_STATUS_SUCCEEDED timestamp = date_utils.get_datetime_utc_now() action_execution_db = ActionExecutionDB( start_timestamp=timestamp, end_timestamp=timestamp, status=status, action={'ref': 'core.local'}, runner={'name': 'local-shell-cmd'}, liveaction={'ref': 'foo'}) action_execution_db = ActionExecution.add_or_update( action_execution_db) for i in range(1, 6): stdout_db = ActionExecutionOutputDB(execution_id=str( action_execution_db.id), action_ref='core.local', runner_ref='dummy', timestamp=timestamp, output_type='stdout', data='stdout %s\n' % (i)) ActionExecutionOutput.add_or_update(stdout_db) for i in range(10, 15): stderr_db = ActionExecutionOutputDB(execution_id=str( action_execution_db.id), action_ref='core.local', runner_ref='dummy', timestamp=timestamp, output_type='stderr', data='stderr %s\n' % (i)) ActionExecutionOutput.add_or_update(stderr_db) resp = self.app.get('/v1/executions/%s/output' % (str(action_execution_db.id)), expect_errors=False) self.assertEqual(resp.status_int, 200) events = self._parse_response(resp.text) self.assertEqual(len(events), 11) self.assertEqual(events[0][1]['data'], 'stdout 1\n') self.assertEqual(events[9][1]['data'], 'stderr 14\n') self.assertEqual(events[10][0], 'EOF') # Verify "last" short-hand id works resp = self.app.get('/v1/executions/last/output', expect_errors=False) self.assertEqual(resp.status_int, 200) events = self._parse_response(resp.text) self.assertEqual(len(events), 11) self.assertEqual(events[10][0], 'EOF')
def create_execution_object(liveaction, publish=True): action_db = action_utils.get_action_by_ref(liveaction.action) runner = RunnerType.get_by_name(action_db.runner_type['name']) attrs = { 'action': vars(ActionAPI.from_model(action_db)), 'parameters': liveaction['parameters'], 'runner': vars(RunnerTypeAPI.from_model(runner)) } attrs.update(_decompose_liveaction(liveaction)) if 'rule' in liveaction.context: rule = reference.get_model_from_ref(Rule, liveaction.context.get('rule', {})) attrs['rule'] = vars(RuleAPI.from_model(rule)) if 'trigger_instance' in liveaction.context: trigger_instance_id = liveaction.context.get('trigger_instance', {}) trigger_instance_id = trigger_instance_id.get('id', None) trigger_instance = TriggerInstance.get_by_id(trigger_instance_id) trigger = reference.get_model_by_resource_ref(db_api=Trigger, ref=trigger_instance.trigger) trigger_type = reference.get_model_by_resource_ref(db_api=TriggerType, ref=trigger.type) trigger_instance = reference.get_model_from_ref( TriggerInstance, liveaction.context.get('trigger_instance', {})) attrs['trigger_instance'] = vars(TriggerInstanceAPI.from_model(trigger_instance)) attrs['trigger'] = vars(TriggerAPI.from_model(trigger)) attrs['trigger_type'] = vars(TriggerTypeAPI.from_model(trigger_type)) parent = _get_parent_execution(liveaction) if parent: attrs['parent'] = str(parent.id) execution = ActionExecutionDB(**attrs) execution = ActionExecution.add_or_update(execution, publish=publish) if parent: if str(execution.id) not in parent.children: parent.children.append(str(execution.id)) ActionExecution.add_or_update(parent) return execution
def publish_action_finished(action_execution_db): # Insert mock output object output_params['data'] = 'stdout pre finish 1\n' output_db = ActionExecutionOutputDB(**output_params) ActionExecutionOutput.add_or_update(output_db) eventlet.sleep(1.0) # Transition execution to completed state so the connection closes action_execution_db.status = action_constants.LIVEACTION_STATUS_SUCCEEDED action_execution_db = ActionExecution.add_or_update(action_execution_db)
def create_execution_object(liveaction, action_db=None, runnertype_db=None, publish=True): if not action_db: action_db = action_utils.get_action_by_ref(liveaction.action) if not runnertype_db: runnertype_db = RunnerType.get_by_name(action_db.runner_type['name']) attrs = { 'action': vars(ActionAPI.from_model(action_db)), 'parameters': liveaction['parameters'], 'runner': vars(RunnerTypeAPI.from_model(runnertype_db)) } attrs.update(_decompose_liveaction(liveaction)) if 'rule' in liveaction.context: rule = reference.get_model_from_ref(Rule, liveaction.context.get('rule', {})) attrs['rule'] = vars(RuleAPI.from_model(rule)) if 'trigger_instance' in liveaction.context: trigger_instance_id = liveaction.context.get('trigger_instance', {}) trigger_instance_id = trigger_instance_id.get('id', None) trigger_instance = TriggerInstance.get_by_id(trigger_instance_id) trigger = reference.get_model_by_resource_ref(db_api=Trigger, ref=trigger_instance.trigger) trigger_type = reference.get_model_by_resource_ref(db_api=TriggerType, ref=trigger.type) trigger_instance = reference.get_model_from_ref( TriggerInstance, liveaction.context.get('trigger_instance', {})) attrs['trigger_instance'] = vars(TriggerInstanceAPI.from_model(trigger_instance)) attrs['trigger'] = vars(TriggerAPI.from_model(trigger)) attrs['trigger_type'] = vars(TriggerTypeAPI.from_model(trigger_type)) parent = _get_parent_execution(liveaction) if parent: attrs['parent'] = str(parent.id) attrs['log'] = [_create_execution_log_entry(liveaction['status'])] # TODO: This object initialization takes 20-30or so ms execution = ActionExecutionDB(**attrs) # TODO: Do 100% research this is fully safe and unique in distributed setups execution.id = ObjectId() execution.web_url = _get_web_url_for_execution(str(execution.id)) # NOTE: User input data is already validate as part of the API request, # other data is set by us. Skipping validation here makes operation 10%-30% faster execution = ActionExecution.add_or_update(execution, publish=publish, validate=False) if parent and str(execution.id) not in parent.children: values = {} values['push__children'] = str(execution.id) ActionExecution.update(parent, **values) return execution
def setUpClass(cls): super(TestActionExecutionFilters, cls).setUpClass() cls.dt_base = date_utils.add_utc_tz(datetime.datetime(2014, 12, 25, 0, 0, 0)) cls.num_records = 100 cls.refs = {} cls.start_timestamps = [] cls.fake_types = [ { 'trigger': copy.deepcopy(fixture.ARTIFACTS['trigger']), 'trigger_type': copy.deepcopy(fixture.ARTIFACTS['trigger_type']), 'trigger_instance': copy.deepcopy(fixture.ARTIFACTS['trigger_instance']), 'rule': copy.deepcopy(fixture.ARTIFACTS['rule']), 'action': copy.deepcopy(fixture.ARTIFACTS['actions']['chain']), 'runner': copy.deepcopy(fixture.ARTIFACTS['runners']['action-chain']), 'liveaction': copy.deepcopy(fixture.ARTIFACTS['liveactions']['workflow']), 'context': copy.deepcopy(fixture.ARTIFACTS['context']), 'children': [] }, { 'action': copy.deepcopy(fixture.ARTIFACTS['actions']['local']), 'runner': copy.deepcopy(fixture.ARTIFACTS['runners']['run-local']), 'liveaction': copy.deepcopy(fixture.ARTIFACTS['liveactions']['task1']) } ] def assign_parent(child): candidates = [v for k, v in cls.refs.items() if v.action['name'] == 'chain'] if candidates: parent = random.choice(candidates) child['parent'] = str(parent.id) parent.children.append(child['id']) cls.refs[str(parent.id)] = ActionExecution.add_or_update(parent) for i in range(cls.num_records): obj_id = str(bson.ObjectId()) timestamp = cls.dt_base + datetime.timedelta(seconds=i) fake_type = random.choice(cls.fake_types) data = copy.deepcopy(fake_type) data['id'] = obj_id data['start_timestamp'] = isotime.format(timestamp, offset=False) data['end_timestamp'] = isotime.format(timestamp, offset=False) data['status'] = data['liveaction']['status'] data['result'] = data['liveaction']['result'] if fake_type['action']['name'] == 'local' and random.choice([True, False]): assign_parent(data) wb_obj = ActionExecutionAPI(**data) db_obj = ActionExecutionAPI.to_model(wb_obj) cls.refs[obj_id] = ActionExecution.add_or_update(db_obj) cls.start_timestamps.append(timestamp) cls.start_timestamps = sorted(cls.start_timestamps)
def setUpClass(cls): super(TestActionExecutionFilters, cls).setUpClass() cls.dt_base = date_utils.add_utc_tz(datetime.datetime(2014, 12, 25, 0, 0, 0)) cls.num_records = 100 cls.refs = {} cls.start_timestamps = [] cls.fake_types = [ { "trigger": copy.deepcopy(fixture.ARTIFACTS["trigger"]), "trigger_type": copy.deepcopy(fixture.ARTIFACTS["trigger_type"]), "trigger_instance": copy.deepcopy(fixture.ARTIFACTS["trigger_instance"]), "rule": copy.deepcopy(fixture.ARTIFACTS["rule"]), "action": copy.deepcopy(fixture.ARTIFACTS["actions"]["chain"]), "runner": copy.deepcopy(fixture.ARTIFACTS["runners"]["action-chain"]), "liveaction": copy.deepcopy(fixture.ARTIFACTS["liveactions"]["workflow"]), "context": copy.deepcopy(fixture.ARTIFACTS["context"]), "children": [], }, { "action": copy.deepcopy(fixture.ARTIFACTS["actions"]["local"]), "runner": copy.deepcopy(fixture.ARTIFACTS["runners"]["run-local"]), "liveaction": copy.deepcopy(fixture.ARTIFACTS["liveactions"]["task1"]), }, ] def assign_parent(child): candidates = [v for k, v in cls.refs.iteritems() if v.action["name"] == "chain"] if candidates: parent = random.choice(candidates) child["parent"] = str(parent.id) parent.children.append(child["id"]) cls.refs[str(parent.id)] = ActionExecution.add_or_update(parent) for i in range(cls.num_records): obj_id = str(bson.ObjectId()) timestamp = cls.dt_base + datetime.timedelta(seconds=i) fake_type = random.choice(cls.fake_types) data = copy.deepcopy(fake_type) data["id"] = obj_id data["start_timestamp"] = isotime.format(timestamp, offset=False) data["end_timestamp"] = isotime.format(timestamp, offset=False) data["status"] = data["liveaction"]["status"] data["result"] = data["liveaction"]["result"] if fake_type["action"]["name"] == "local" and random.choice([True, False]): assign_parent(data) wb_obj = ActionExecutionAPI(**data) db_obj = ActionExecutionAPI.to_model(wb_obj) cls.refs[obj_id] = ActionExecution.add_or_update(db_obj) cls.start_timestamps.append(timestamp) cls.start_timestamps = sorted(cls.start_timestamps)
def test_sort_by_start_timestamp(self): base = isotime.add_utc_tz(datetime.datetime(2014, 12, 25, 0, 0, 0)) for i in range(60): timestamp = base + datetime.timedelta(seconds=i) doc = copy.deepcopy(self.fake_history_subtasks[0]) doc['id'] = str(bson.ObjectId()) doc['start_timestamp'] = isotime.format(timestamp) obj = ActionExecutionAPI(**doc) ActionExecution.add_or_update(ActionExecutionAPI.to_model(obj)) dt_range = '2014-12-25T00:00:10Z..2014-12-25T00:00:19Z' objs = ActionExecution.query(start_timestamp=dt_range, order_by=['start_timestamp']) self.assertLess(objs[0]['start_timestamp'], objs[9]['start_timestamp']) dt_range = '2014-12-25T00:00:19Z..2014-12-25T00:00:10Z' objs = ActionExecution.query(start_timestamp=dt_range, order_by=['-start_timestamp']) self.assertLess(objs[9]['start_timestamp'], objs[0]['start_timestamp'])
def test_no_timestamp_doesnt_delete_things(self): now = date_utils.get_datetime_utc_now() exec_model = copy.deepcopy( self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = now - timedelta(days=15) exec_model['end_timestamp'] = now - timedelta(days=14) exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) execs = ActionExecution.get_all() self.assertEqual(len(execs), 1) expected_msg = 'Specify a valid timestamp' self.assertRaisesRegexp(ValueError, expected_msg, purge_executions, logger=LOG, timestamp=None) execs = ActionExecution.get_all() self.assertEqual(len(execs), 1)
def test_purge_incomplete(self): now = date_utils.get_datetime_utc_now() start_ts = now - timedelta(days=15) # Write executions before cut-off threshold exec_model = copy.deepcopy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_SCHEDULED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) exec_model = copy.deepcopy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_RUNNING exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) exec_model = copy.deepcopy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_DELAYED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) exec_model = copy.deepcopy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_CANCELING exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) exec_model = copy.deepcopy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_REQUESTED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) self.assertEqual(len(ActionExecution.get_all()), 5) purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=False) self.assertEqual(len(ActionExecution.get_all()), 5) purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=True) self.assertEqual(len(ActionExecution.get_all()), 0)
def test_purge_executions_with_timestamp(self): now = date_utils.get_datetime_utc_now() # Write one execution after cut-off threshold exec_model = copy.deepcopy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = now - timedelta(days=15) exec_model['end_timestamp'] = now - timedelta(days=14) exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) # Insert corresponding stdout and stderr db mock models self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=3) # Write one execution before cut-off threshold exec_model = copy.deepcopy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = now - timedelta(days=22) exec_model['end_timestamp'] = now - timedelta(days=21) exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) # Insert corresponding stdout and stderr db mock models self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=3) execs = ActionExecution.get_all() self.assertEqual(len(execs), 2) stdout_dbs = ActionExecutionOutput.query(output_type='stdout') self.assertEqual(len(stdout_dbs), 6) stderr_dbs = ActionExecutionOutput.query(output_type='stderr') self.assertEqual(len(stderr_dbs), 6) purge_executions(logger=LOG, timestamp=now - timedelta(days=20)) execs = ActionExecution.get_all() self.assertEqual(len(execs), 1) stdout_dbs = ActionExecutionOutput.query(output_type='stdout') self.assertEqual(len(stdout_dbs), 3) stderr_dbs = ActionExecutionOutput.query(output_type='stderr') self.assertEqual(len(stderr_dbs), 3)
def test_purge_executions_with_timestamp(self): now = date_utils.get_datetime_utc_now() # Write one execution after cut-off threshold exec_model = copy.deepcopy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = now - timedelta(days=15) exec_model['end_timestamp'] = now - timedelta(days=14) exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) # Write one execution before cut-off threshold exec_model = copy.deepcopy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = now - timedelta(days=22) exec_model['end_timestamp'] = now - timedelta(days=21) exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) execs = ActionExecution.get_all() purge_executions(logger=LOG, timestamp=now - timedelta(days=20)) execs = ActionExecution.get_all() self.assertEqual(len(execs), 1)
def test_purge_executions_with_action_ref(self): now = date_utils.get_datetime_utc_now() exec_model = copy.deepcopy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = now - timedelta(days=15) exec_model['end_timestamp'] = now - timedelta(days=14) exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) # Insert corresponding stdout and stderr db mock models self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=3) execs = ActionExecution.get_all() self.assertEqual(len(execs), 1) stdout_dbs = ActionExecutionOutput.query(output_type='stdout') self.assertEqual(len(stdout_dbs), 3) stderr_dbs = ActionExecutionOutput.query(output_type='stderr') self.assertEqual(len(stderr_dbs), 3) # Invalid action reference, nothing should be deleted purge_executions(logger=LOG, action_ref='core.localzzz', timestamp=now - timedelta(days=10)) execs = ActionExecution.get_all() self.assertEqual(len(execs), 1) stdout_dbs = ActionExecutionOutput.query(output_type='stdout') self.assertEqual(len(stdout_dbs), 3) stderr_dbs = ActionExecutionOutput.query(output_type='stderr') self.assertEqual(len(stderr_dbs), 3) purge_executions(logger=LOG, action_ref='core.local', timestamp=now - timedelta(days=10)) execs = ActionExecution.get_all() self.assertEqual(len(execs), 0) stdout_dbs = ActionExecutionOutput.query(output_type='stdout') self.assertEqual(len(stdout_dbs), 0) stderr_dbs = ActionExecutionOutput.query(output_type='stderr') self.assertEqual(len(stderr_dbs), 0)
def test_get_output_finished_execution(self): # Test the execution output API endpoint for execution which has finished for status in action_constants.LIVEACTION_COMPLETED_STATES: # Insert mock execution and output objects status = action_constants.LIVEACTION_STATUS_SUCCEEDED timestamp = date_utils.get_datetime_utc_now() action_execution_db = ActionExecutionDB(start_timestamp=timestamp, end_timestamp=timestamp, status=status, action={'ref': 'core.local'}, runner={'name': 'local-shell-cmd'}, liveaction={'ref': 'foo'}) action_execution_db = ActionExecution.add_or_update(action_execution_db) for i in range(1, 6): stdout_db = ActionExecutionOutputDB(execution_id=str(action_execution_db.id), action_ref='core.local', runner_ref='dummy', timestamp=timestamp, output_type='stdout', data='stdout %s\n' % (i)) ActionExecutionOutput.add_or_update(stdout_db) for i in range(10, 15): stderr_db = ActionExecutionOutputDB(execution_id=str(action_execution_db.id), action_ref='core.local', runner_ref='dummy', timestamp=timestamp, output_type='stderr', data='stderr %s\n' % (i)) ActionExecutionOutput.add_or_update(stderr_db) resp = self.app.get('/v1/executions/%s/output' % (str(action_execution_db.id)), expect_errors=False) self.assertEqual(resp.status_int, 200) events = self._parse_response(resp.text) self.assertEqual(len(events), 11) self.assertEqual(events[0][1]['data'], 'stdout 1\n') self.assertEqual(events[9][1]['data'], 'stderr 14\n') self.assertEqual(events[10][0], 'EOF') # Verify "last" short-hand id works resp = self.app.get('/v1/executions/last/output', expect_errors=False) self.assertEqual(resp.status_int, 200) events = self._parse_response(resp.text) self.assertEqual(len(events), 11) self.assertEqual(events[10][0], 'EOF')
def test_crud_complete(self): # Create the DB record. obj = ActionExecutionAPI(**copy.deepcopy(self.fake_history_workflow)) ActionExecution.add_or_update(ActionExecutionAPI.to_model(obj)) model = ActionExecution.get_by_id(obj.id) self.assertEqual(str(model.id), obj.id) self.assertDictEqual(model.trigger, self.fake_history_workflow["trigger"]) self.assertDictEqual(model.trigger_type, self.fake_history_workflow["trigger_type"]) self.assertDictEqual(model.trigger_instance, self.fake_history_workflow["trigger_instance"]) self.assertDictEqual(model.rule, self.fake_history_workflow["rule"]) self.assertDictEqual(model.action, self.fake_history_workflow["action"]) self.assertDictEqual(model.runner, self.fake_history_workflow["runner"]) doc = copy.deepcopy(self.fake_history_workflow["liveaction"]) doc["start_timestamp"] = doc["start_timestamp"] doc["end_timestamp"] = doc["end_timestamp"] self.assertDictEqual(model.liveaction, doc) self.assertIsNone(getattr(model, "parent", None)) self.assertListEqual(model.children, self.fake_history_workflow["children"]) # Update the DB record. children = [str(bson.ObjectId()), str(bson.ObjectId())] model.children = children ActionExecution.add_or_update(model) model = ActionExecution.get_by_id(obj.id) self.assertListEqual(model.children, children) # Delete the DB record. ActionExecution.delete(model) self.assertRaises(StackStormDBObjectNotFoundError, ActionExecution.get_by_id, obj.id)
def setUp(self): super(ExecutionPermissionsResolverTestCase, self).setUp() # Create some mock users user_1_db = UserDB(name='custom_role_unrelated_pack_action_grant') user_1_db = User.add_or_update(user_1_db) self.users['custom_role_unrelated_pack_action_grant'] = user_1_db user_2_db = UserDB( name='custom_role_pack_action_grant_unrelated_permission') user_2_db = User.add_or_update(user_2_db) self.users[ 'custom_role_pack_action_grant_unrelated_permission'] = user_2_db user_3_db = UserDB(name='custom_role_pack_action_view_grant') user_3_db = User.add_or_update(user_3_db) self.users['custom_role_pack_action_view_grant'] = user_3_db user_4_db = UserDB(name='custom_role_action_view_grant') user_4_db = User.add_or_update(user_4_db) self.users['custom_role_action_view_grant'] = user_4_db user_5_db = UserDB(name='custom_role_pack_action_execute_grant') user_5_db = User.add_or_update(user_5_db) self.users['custom_role_pack_action_execute_grant'] = user_5_db user_6_db = UserDB(name='custom_role_action_execute_grant') user_6_db = User.add_or_update(user_6_db) self.users['custom_role_action_execute_grant'] = user_6_db user_7_db = UserDB(name='custom_role_pack_action_all_grant') user_7_db = User.add_or_update(user_7_db) self.users['custom_role_pack_action_all_grant'] = user_7_db user_8_db = UserDB(name='custom_role_action_all_grant') user_8_db = User.add_or_update(user_8_db) self.users['custom_role_action_all_grant'] = user_8_db user_9_db = UserDB(name='custom_role_execution_list_grant') user_9_db = User.add_or_update(user_5_db) self.users['custom_role_execution_list_grant'] = user_9_db # Create some mock resources on which permissions can be granted action_1_db = ActionDB(pack='test_pack_2', name='action1', entry_point='', runner_type={'name': 'run-local'}) action_1_db = Action.add_or_update(action_1_db) self.resources['action_1'] = action_1_db runner = {'name': 'run-python'} liveaction = {'action': 'test_pack_2.action1'} status = action_constants.LIVEACTION_STATUS_REQUESTED action = {'uid': action_1_db.get_uid(), 'pack': 'test_pack_2'} exec_1_db = ActionExecutionDB(action=action, runner=runner, liveaction=liveaction, status=status) exec_1_db = ActionExecution.add_or_update(exec_1_db) self.resources['exec_1'] = exec_1_db # Create some mock roles with associated permission grants # Custom role - one grant to an unrelated pack grant_db = PermissionGrantDB( resource_uid=self.resources['pack_1'].get_uid(), resource_type=ResourceType.PACK, permission_types=[PermissionType.ACTION_VIEW]) grant_db = PermissionGrant.add_or_update(grant_db) permission_grants = [str(grant_db.id)] role_db = RoleDB(name='custom_role_unrelated_pack_action_grant', permission_grants=permission_grants) role_db = Role.add_or_update(role_db) self.roles['custom_role_unrelated_pack_action_grant'] = role_db # Custom role - one grant of unrelated permission type to parent action pack grant_db = PermissionGrantDB( resource_uid=self.resources['pack_2'].get_uid(), resource_type=ResourceType.PACK, permission_types=[PermissionType.RULE_VIEW]) grant_db = PermissionGrant.add_or_update(grant_db) permission_grants = [str(grant_db.id)] role_db = RoleDB( name='custom_role_pack_action_grant_unrelated_permission', permission_grants=permission_grants) role_db = Role.add_or_update(role_db) self.roles[ 'custom_role_pack_action_grant_unrelated_permission'] = role_db # Custom role - one grant of "action_view" to the parent pack of the action the execution # belongs to grant_db = PermissionGrantDB( resource_uid=self.resources['pack_2'].get_uid(), resource_type=ResourceType.PACK, permission_types=[PermissionType.ACTION_VIEW]) grant_db = PermissionGrant.add_or_update(grant_db) permission_grants = [str(grant_db.id)] role_db = RoleDB(name='custom_role_pack_action_view_grant', permission_grants=permission_grants) role_db = Role.add_or_update(role_db) self.roles['custom_role_pack_action_view_grant'] = role_db # Custom role - one grant of "action_view" to the action the execution belongs to grant_db = PermissionGrantDB( resource_uid=self.resources['action_1'].get_uid(), resource_type=ResourceType.ACTION, permission_types=[PermissionType.ACTION_VIEW]) grant_db = PermissionGrant.add_or_update(grant_db) permission_grants = [str(grant_db.id)] role_db = RoleDB(name='custom_role_action_view_grant', permission_grants=permission_grants) role_db = Role.add_or_update(role_db) self.roles['custom_role_action_view_grant'] = role_db # Custom role - one grant of "action_execute" to the parent pack of the action the # execution belongs to grant_db = PermissionGrantDB( resource_uid=self.resources['pack_2'].get_uid(), resource_type=ResourceType.PACK, permission_types=[PermissionType.ACTION_EXECUTE]) grant_db = PermissionGrant.add_or_update(grant_db) permission_grants = [str(grant_db.id)] role_db = RoleDB(name='custom_role_pack_action_execute_grant', permission_grants=permission_grants) role_db = Role.add_or_update(role_db) self.roles['custom_role_pack_action_execute_grant'] = role_db # Custom role - one grant of "action_execute" to the the action the execution belongs to grant_db = PermissionGrantDB( resource_uid=self.resources['action_1'].get_uid(), resource_type=ResourceType.ACTION, permission_types=[PermissionType.ACTION_EXECUTE]) grant_db = PermissionGrant.add_or_update(grant_db) permission_grants = [str(grant_db.id)] role_db = RoleDB(name='custom_role_action_execute_grant', permission_grants=permission_grants) role_db = Role.add_or_update(role_db) self.roles['custom_role_action_execute_grant'] = role_db # Custom role - "action_all" grant on a parent action pack the execution belongs to grant_db = PermissionGrantDB( resource_uid=self.resources['pack_2'].get_uid(), resource_type=ResourceType.PACK, permission_types=[PermissionType.ACTION_ALL]) grant_db = PermissionGrant.add_or_update(grant_db) permission_grants = [str(grant_db.id)] role_4_db = RoleDB(name='custom_role_pack_action_all_grant', permission_grants=permission_grants) role_4_db = Role.add_or_update(role_4_db) self.roles['custom_role_pack_action_all_grant'] = role_4_db # Custom role - "action_all" grant on action the execution belongs to grant_db = PermissionGrantDB( resource_uid=self.resources['action_1'].get_uid(), resource_type=ResourceType.ACTION, permission_types=[PermissionType.ACTION_ALL]) grant_db = PermissionGrant.add_or_update(grant_db) permission_grants = [str(grant_db.id)] role_4_db = RoleDB(name='custom_role_action_all_grant', permission_grants=permission_grants) role_4_db = Role.add_or_update(role_4_db) self.roles['custom_role_action_all_grant'] = role_4_db # Custom role - "execution_list" grant grant_db = PermissionGrantDB( resource_uid=None, resource_type=None, permission_types=[PermissionType.EXECUTION_LIST]) grant_db = PermissionGrant.add_or_update(grant_db) permission_grants = [str(grant_db.id)] role_5_db = RoleDB(name='custom_role_execution_list_grant', permission_grants=permission_grants) role_5_db = Role.add_or_update(role_5_db) self.roles['custom_role_execution_list_grant'] = role_5_db # Create some mock role assignments user_db = self.users['custom_role_unrelated_pack_action_grant'] role_assignment_db = UserRoleAssignmentDB( user=user_db.name, role=self.roles['custom_role_unrelated_pack_action_grant'].name, source='assignments/%s.yaml' % user_db.name) UserRoleAssignment.add_or_update(role_assignment_db) user_db = self.users[ 'custom_role_pack_action_grant_unrelated_permission'] role_assignment_db = UserRoleAssignmentDB( user=user_db.name, role=self. roles['custom_role_pack_action_grant_unrelated_permission'].name, source='assignments/%s.yaml' % user_db.name) UserRoleAssignment.add_or_update(role_assignment_db) user_db = self.users['custom_role_pack_action_view_grant'] role_assignment_db = UserRoleAssignmentDB( user=user_db.name, role=self.roles['custom_role_pack_action_view_grant'].name, source='assignments/%s.yaml' % user_db.name) UserRoleAssignment.add_or_update(role_assignment_db) user_db = self.users['custom_role_action_view_grant'] role_assignment_db = UserRoleAssignmentDB( user=user_db.name, role=self.roles['custom_role_action_view_grant'].name, source='assignments/%s.yaml' % user_db.name) UserRoleAssignment.add_or_update(role_assignment_db) user_db = self.users['custom_role_pack_action_execute_grant'] role_assignment_db = UserRoleAssignmentDB( user=user_db.name, role=self.roles['custom_role_pack_action_execute_grant'].name, source='assignments/%s.yaml' % user_db.name) UserRoleAssignment.add_or_update(role_assignment_db) user_db = self.users['custom_role_action_execute_grant'] role_assignment_db = UserRoleAssignmentDB( user=user_db.name, role=self.roles['custom_role_action_execute_grant'].name, source='assignments/%s.yaml' % user_db.name) UserRoleAssignment.add_or_update(role_assignment_db) user_db = self.users['custom_role_pack_action_all_grant'] role_assignment_db = UserRoleAssignmentDB( user=user_db.name, role=self.roles['custom_role_pack_action_all_grant'].name, source='assignments/%s.yaml' % user_db.name) UserRoleAssignment.add_or_update(role_assignment_db) user_db = self.users['custom_role_action_all_grant'] role_assignment_db = UserRoleAssignmentDB( user=user_db.name, role=self.roles['custom_role_action_all_grant'].name, source='assignments/%s.yaml' % user_db.name) UserRoleAssignment.add_or_update(role_assignment_db) user_db = self.users['custom_role_execution_list_grant'] role_assignment_db = UserRoleAssignmentDB( user=user_db.name, role=self.roles['custom_role_execution_list_grant'].name, source='assignments/%s.yaml' % user_db.name) UserRoleAssignment.add_or_update(role_assignment_db)
def test_get_output_running_execution(self): # Retrieve lister instance to avoid race with listener connection not being established # early enough for tests to pass. # NOTE: This only affects tests where listeners are not pre-initialized. listener = get_listener(name='execution_output') eventlet.sleep(1.0) # Test the execution output API endpoint for execution which is running (blocking) status = action_constants.LIVEACTION_STATUS_RUNNING timestamp = date_utils.get_datetime_utc_now() action_execution_db = ActionExecutionDB( start_timestamp=timestamp, end_timestamp=timestamp, status=status, action={'ref': 'core.local'}, runner={'name': 'local-shell-cmd'}, liveaction={'ref': 'foo'}) action_execution_db = ActionExecution.add_or_update( action_execution_db) output_params = dict(execution_id=str(action_execution_db.id), action_ref='core.local', runner_ref='dummy', timestamp=timestamp, output_type='stdout', data='stdout before start\n') # Insert mock output object output_db = ActionExecutionOutputDB(**output_params) ActionExecutionOutput.add_or_update(output_db, publish=False) def insert_mock_data(): output_params['data'] = 'stdout mid 1\n' output_db = ActionExecutionOutputDB(**output_params) ActionExecutionOutput.add_or_update(output_db) # Since the API endpoint is blocking (connection is kept open until action finishes), we # spawn an eventlet which eventually finishes the action. def publish_action_finished(action_execution_db): # Insert mock output object output_params['data'] = 'stdout pre finish 1\n' output_db = ActionExecutionOutputDB(**output_params) ActionExecutionOutput.add_or_update(output_db) eventlet.sleep(1.0) # Transition execution to completed state so the connection closes action_execution_db.status = action_constants.LIVEACTION_STATUS_SUCCEEDED action_execution_db = ActionExecution.add_or_update( action_execution_db) eventlet.spawn_after(0.2, insert_mock_data) eventlet.spawn_after(1.5, publish_action_finished, action_execution_db) # Retrieve data while execution is running - endpoint return new data once it's available # and block until the execution finishes resp = self.app.get('/v1/executions/%s/output' % (str(action_execution_db.id)), expect_errors=False) self.assertEqual(resp.status_int, 200) events = self._parse_response(resp.text) self.assertEqual(len(events), 4) self.assertEqual(events[0][1]['data'], 'stdout before start\n') self.assertEqual(events[1][1]['data'], 'stdout mid 1\n') self.assertEqual(events[2][1]['data'], 'stdout pre finish 1\n') self.assertEqual(events[3][0], 'EOF') # Once the execution is in completed state, existing output should be returned immediately resp = self.app.get('/v1/executions/%s/output' % (str(action_execution_db.id)), expect_errors=False) self.assertEqual(resp.status_int, 200) events = self._parse_response(resp.text) self.assertEqual(len(events), 4) self.assertEqual(events[0][1]['data'], 'stdout before start\n') self.assertEqual(events[1][1]['data'], 'stdout mid 1\n') self.assertEqual(events[2][1]['data'], 'stdout pre finish 1\n') self.assertEqual(events[3][0], 'EOF') listener.shutdown()
def _save_execution(execution): return ActionExecution.add_or_update(execution)
def test_garbage_collection(self): now = date_utils.get_datetime_utc_now() status = action_constants.LIVEACTION_STATUS_SUCCEEDED # Insert come mock ActionExecutionDB objects with start_timestamp < TTL defined in the # config old_executions_count = 15 ttl_days = 30 # > 20 timestamp = (now - datetime.timedelta(days=ttl_days)) for index in range(0, old_executions_count): action_execution_db = ActionExecutionDB(start_timestamp=timestamp, end_timestamp=timestamp, status=status, action={'ref': 'core.local'}, runner={'name': 'local-shell-cmd'}, liveaction={'ref': 'foo'}) ActionExecution.add_or_update(action_execution_db) stdout_db = ActionExecutionOutputDB(execution_id=str(action_execution_db.id), action_ref='core.local', runner_ref='dummy', timestamp=timestamp, output_type='stdout', data='stdout') ActionExecutionOutput.add_or_update(stdout_db) stderr_db = ActionExecutionOutputDB(execution_id=str(action_execution_db.id), action_ref='core.local', runner_ref='dummy', timestamp=timestamp, output_type='stderr', data='stderr') ActionExecutionOutput.add_or_update(stderr_db) # Insert come mock ActionExecutionDB objects with start_timestamp > TTL defined in the # config new_executions_count = 5 ttl_days = 2 # < 20 timestamp = (now - datetime.timedelta(days=ttl_days)) for index in range(0, new_executions_count): action_execution_db = ActionExecutionDB(start_timestamp=timestamp, end_timestamp=timestamp, status=status, action={'ref': 'core.local'}, runner={'name': 'local-shell-cmd'}, liveaction={'ref': 'foo'}) ActionExecution.add_or_update(action_execution_db) stdout_db = ActionExecutionOutputDB(execution_id=str(action_execution_db.id), action_ref='core.local', runner_ref='dummy', timestamp=timestamp, output_type='stdout', data='stdout') ActionExecutionOutput.add_or_update(stdout_db) stderr_db = ActionExecutionOutputDB(execution_id=str(action_execution_db.id), action_ref='core.local', runner_ref='dummy', timestamp=timestamp, output_type='stderr', data='stderr') ActionExecutionOutput.add_or_update(stderr_db) # Insert some mock output objects where start_timestamp > action_executions_output_ttl new_output_count = 5 ttl_days = 15 # > 10 and < 20 timestamp = (now - datetime.timedelta(days=ttl_days)) for index in range(0, new_output_count): action_execution_db = ActionExecutionDB(start_timestamp=timestamp, end_timestamp=timestamp, status=status, action={'ref': 'core.local'}, runner={'name': 'local-shell-cmd'}, liveaction={'ref': 'foo'}) ActionExecution.add_or_update(action_execution_db) stdout_db = ActionExecutionOutputDB(execution_id=str(action_execution_db.id), action_ref='core.local', runner_ref='dummy', timestamp=timestamp, output_type='stdout', data='stdout') ActionExecutionOutput.add_or_update(stdout_db) stderr_db = ActionExecutionOutputDB(execution_id=str(action_execution_db.id), action_ref='core.local', runner_ref='dummy', timestamp=timestamp, output_type='stderr', data='stderr') ActionExecutionOutput.add_or_update(stderr_db) execs = ActionExecution.get_all() self.assertEqual(len(execs), (old_executions_count + new_executions_count + new_output_count)) stdout_dbs = ActionExecutionOutput.query(output_type='stdout') self.assertEqual(len(stdout_dbs), (old_executions_count + new_executions_count + new_output_count)) stderr_dbs = ActionExecutionOutput.query(output_type='stderr') self.assertEqual(len(stderr_dbs), (old_executions_count + new_executions_count + new_output_count)) # Start garbage collector process = self._start_garbage_collector() # Give it some time to perform garbage collection and kill it eventlet.sleep(15) process.send_signal(signal.SIGKILL) self.remove_process(process=process) # Old executions and corresponding objects should have been garbage collected execs = ActionExecution.get_all() self.assertEqual(len(execs), (new_executions_count + new_output_count)) # Collection for output objects older than 10 days is also enabled, so those objects # should be deleted as well stdout_dbs = ActionExecutionOutput.query(output_type='stdout') self.assertEqual(len(stdout_dbs), (new_executions_count)) stderr_dbs = ActionExecutionOutput.query(output_type='stderr') self.assertEqual(len(stderr_dbs), (new_executions_count))
def test_purge_incomplete(self): now = date_utils.get_datetime_utc_now() start_ts = now - timedelta(days=15) # Write executions before cut-off threshold exec_model = copy.deepcopy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_SCHEDULED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) # Insert corresponding stdout and stderr db mock models self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=1) exec_model = copy.deepcopy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_RUNNING exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) # Insert corresponding stdout and stderr db mock models self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=1) exec_model = copy.deepcopy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_DELAYED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) # Insert corresponding stdout and stderr db mock models self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=1) exec_model = copy.deepcopy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_CANCELING exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) # Insert corresponding stdout and stderr db mock models self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=1) exec_model = copy.deepcopy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_REQUESTED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) # Insert corresponding stdout and stderr db mock models self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=1) self.assertEqual(len(ActionExecution.get_all()), 5) stdout_dbs = ActionExecutionOutput.query(output_type='stdout') self.assertEqual(len(stdout_dbs), 5) stderr_dbs = ActionExecutionOutput.query(output_type='stderr') self.assertEqual(len(stderr_dbs), 5) # Incompleted executions shouldnt be purged purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=False) self.assertEqual(len(ActionExecution.get_all()), 5) stdout_dbs = ActionExecutionOutput.query(output_type='stdout') self.assertEqual(len(stdout_dbs), 5) stderr_dbs = ActionExecutionOutput.query(output_type='stderr') self.assertEqual(len(stderr_dbs), 5) purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=True) self.assertEqual(len(ActionExecution.get_all()), 0) stdout_dbs = ActionExecutionOutput.query(output_type='stdout') self.assertEqual(len(stdout_dbs), 0) stderr_dbs = ActionExecutionOutput.query(output_type='stderr') self.assertEqual(len(stderr_dbs), 0)
def test_get_output_running_execution(self): # Retrieve lister instance to avoid race with listener connection not being established # early enough for tests to pass. # NOTE: This only affects tests where listeners are not pre-initialized. listener = get_listener(name='execution_output') eventlet.sleep(1.0) # Test the execution output API endpoint for execution which is running (blocking) status = action_constants.LIVEACTION_STATUS_RUNNING timestamp = date_utils.get_datetime_utc_now() action_execution_db = ActionExecutionDB(start_timestamp=timestamp, end_timestamp=timestamp, status=status, action={'ref': 'core.local'}, runner={'name': 'local-shell-cmd'}, liveaction={'ref': 'foo'}) action_execution_db = ActionExecution.add_or_update(action_execution_db) output_params = dict(execution_id=str(action_execution_db.id), action_ref='core.local', runner_ref='dummy', timestamp=timestamp, output_type='stdout', data='stdout before start\n') # Insert mock output object output_db = ActionExecutionOutputDB(**output_params) ActionExecutionOutput.add_or_update(output_db, publish=False) def insert_mock_data(): output_params['data'] = 'stdout mid 1\n' output_db = ActionExecutionOutputDB(**output_params) ActionExecutionOutput.add_or_update(output_db) # Since the API endpoint is blocking (connection is kept open until action finishes), we # spawn an eventlet which eventually finishes the action. def publish_action_finished(action_execution_db): # Insert mock output object output_params['data'] = 'stdout pre finish 1\n' output_db = ActionExecutionOutputDB(**output_params) ActionExecutionOutput.add_or_update(output_db) eventlet.sleep(1.0) # Transition execution to completed state so the connection closes action_execution_db.status = action_constants.LIVEACTION_STATUS_SUCCEEDED action_execution_db = ActionExecution.add_or_update(action_execution_db) eventlet.spawn_after(0.2, insert_mock_data) eventlet.spawn_after(1.5, publish_action_finished, action_execution_db) # Retrieve data while execution is running - endpoint return new data once it's available # and block until the execution finishes resp = self.app.get('/v1/executions/%s/output' % (str(action_execution_db.id)), expect_errors=False) self.assertEqual(resp.status_int, 200) events = self._parse_response(resp.text) self.assertEqual(len(events), 4) self.assertEqual(events[0][1]['data'], 'stdout before start\n') self.assertEqual(events[1][1]['data'], 'stdout mid 1\n') self.assertEqual(events[2][1]['data'], 'stdout pre finish 1\n') self.assertEqual(events[3][0], 'EOF') # Once the execution is in completed state, existing output should be returned immediately resp = self.app.get('/v1/executions/%s/output' % (str(action_execution_db.id)), expect_errors=False) self.assertEqual(resp.status_int, 200) events = self._parse_response(resp.text) self.assertEqual(len(events), 4) self.assertEqual(events[0][1]['data'], 'stdout before start\n') self.assertEqual(events[1][1]['data'], 'stdout mid 1\n') self.assertEqual(events[2][1]['data'], 'stdout pre finish 1\n') self.assertEqual(events[3][0], 'EOF') listener.shutdown()
def setUp(self): super(ExecutionPermissionsResolverTestCase, self).setUp() # Create some mock users user_1_db = UserDB(name='custom_role_unrelated_pack_action_grant') user_1_db = User.add_or_update(user_1_db) self.users['custom_role_unrelated_pack_action_grant'] = user_1_db user_2_db = UserDB(name='custom_role_pack_action_grant_unrelated_permission') user_2_db = User.add_or_update(user_2_db) self.users['custom_role_pack_action_grant_unrelated_permission'] = user_2_db user_3_db = UserDB(name='custom_role_pack_action_view_grant') user_3_db = User.add_or_update(user_3_db) self.users['custom_role_pack_action_view_grant'] = user_3_db user_4_db = UserDB(name='custom_role_action_view_grant') user_4_db = User.add_or_update(user_4_db) self.users['custom_role_action_view_grant'] = user_4_db user_5_db = UserDB(name='custom_role_pack_action_execute_grant') user_5_db = User.add_or_update(user_5_db) self.users['custom_role_pack_action_execute_grant'] = user_5_db user_6_db = UserDB(name='custom_role_action_execute_grant') user_6_db = User.add_or_update(user_6_db) self.users['custom_role_action_execute_grant'] = user_6_db user_7_db = UserDB(name='custom_role_pack_action_all_grant') user_7_db = User.add_or_update(user_7_db) self.users['custom_role_pack_action_all_grant'] = user_7_db user_8_db = UserDB(name='custom_role_action_all_grant') user_8_db = User.add_or_update(user_8_db) self.users['custom_role_action_all_grant'] = user_8_db # Create some mock resources on which permissions can be granted action_1_db = ActionDB(pack='test_pack_2', name='action1', entry_point='', runner_type={'name': 'run-local'}) action_1_db = Action.add_or_update(action_1_db) self.resources['action_1'] = action_1_db runner = {'name': 'run-python'} liveaction = {'action': 'test_pack_2.action1'} status = action_constants.LIVEACTION_STATUS_REQUESTED action = {'uid': action_1_db.get_uid(), 'pack': 'test_pack_2'} exec_1_db = ActionExecutionDB(action=action, runner=runner, liveaction=liveaction, status=status) exec_1_db = ActionExecution.add_or_update(exec_1_db) self.resources['exec_1'] = exec_1_db # Create some mock roles with associated permission grants # Custom role - one grant to an unrelated pack grant_db = PermissionGrantDB(resource_uid=self.resources['pack_1'].get_uid(), resource_type=ResourceType.PACK, permission_types=[PermissionType.ACTION_VIEW]) grant_db = PermissionGrant.add_or_update(grant_db) permission_grants = [str(grant_db.id)] role_db = RoleDB(name='custom_role_unrelated_pack_action_grant', permission_grants=permission_grants) role_db = Role.add_or_update(role_db) self.roles['custom_role_unrelated_pack_action_grant'] = role_db # Custom role - one grant of unrelated permission type to parent action pack grant_db = PermissionGrantDB(resource_uid=self.resources['pack_2'].get_uid(), resource_type=ResourceType.PACK, permission_types=[PermissionType.RULE_VIEW]) grant_db = PermissionGrant.add_or_update(grant_db) permission_grants = [str(grant_db.id)] role_db = RoleDB(name='custom_role_pack_action_grant_unrelated_permission', permission_grants=permission_grants) role_db = Role.add_or_update(role_db) self.roles['custom_role_pack_action_grant_unrelated_permission'] = role_db # Custom role - one grant of "action_view" to the parent pack of the action the execution # belongs to grant_db = PermissionGrantDB(resource_uid=self.resources['pack_2'].get_uid(), resource_type=ResourceType.PACK, permission_types=[PermissionType.ACTION_VIEW]) grant_db = PermissionGrant.add_or_update(grant_db) permission_grants = [str(grant_db.id)] role_db = RoleDB(name='custom_role_pack_action_view_grant', permission_grants=permission_grants) role_db = Role.add_or_update(role_db) self.roles['custom_role_pack_action_view_grant'] = role_db # Custom role - one grant of "action_view" to the action the execution belongs to grant_db = PermissionGrantDB(resource_uid=self.resources['action_1'].get_uid(), resource_type=ResourceType.ACTION, permission_types=[PermissionType.ACTION_VIEW]) grant_db = PermissionGrant.add_or_update(grant_db) permission_grants = [str(grant_db.id)] role_db = RoleDB(name='custom_role_action_view_grant', permission_grants=permission_grants) role_db = Role.add_or_update(role_db) self.roles['custom_role_action_view_grant'] = role_db # Custom role - one grant of "action_execute" to the parent pack of the action the # execution belongs to grant_db = PermissionGrantDB(resource_uid=self.resources['pack_2'].get_uid(), resource_type=ResourceType.PACK, permission_types=[PermissionType.ACTION_EXECUTE]) grant_db = PermissionGrant.add_or_update(grant_db) permission_grants = [str(grant_db.id)] role_db = RoleDB(name='custom_role_pack_action_execute_grant', permission_grants=permission_grants) role_db = Role.add_or_update(role_db) self.roles['custom_role_pack_action_execute_grant'] = role_db # Custom role - one grant of "action_execute" to the the action the execution belongs to grant_db = PermissionGrantDB(resource_uid=self.resources['action_1'].get_uid(), resource_type=ResourceType.ACTION, permission_types=[PermissionType.ACTION_EXECUTE]) grant_db = PermissionGrant.add_or_update(grant_db) permission_grants = [str(grant_db.id)] role_db = RoleDB(name='custom_role_action_execute_grant', permission_grants=permission_grants) role_db = Role.add_or_update(role_db) self.roles['custom_role_action_execute_grant'] = role_db # Custom role - "action_all" grant on a parent action pack the execution belongs to grant_db = PermissionGrantDB(resource_uid=self.resources['pack_2'].get_uid(), resource_type=ResourceType.PACK, permission_types=[PermissionType.ACTION_ALL]) grant_db = PermissionGrant.add_or_update(grant_db) permission_grants = [str(grant_db.id)] role_4_db = RoleDB(name='custom_role_pack_action_all_grant', permission_grants=permission_grants) role_4_db = Role.add_or_update(role_4_db) self.roles['custom_role_pack_action_all_grant'] = role_4_db # Custom role - "action_all" grant on action the execution belongs to grant_db = PermissionGrantDB(resource_uid=self.resources['action_1'].get_uid(), resource_type=ResourceType.ACTION, permission_types=[PermissionType.ACTION_ALL]) grant_db = PermissionGrant.add_or_update(grant_db) permission_grants = [str(grant_db.id)] role_4_db = RoleDB(name='custom_role_action_all_grant', permission_grants=permission_grants) role_4_db = Role.add_or_update(role_4_db) self.roles['custom_role_action_all_grant'] = role_4_db # Create some mock role assignments user_db = self.users['custom_role_unrelated_pack_action_grant'] role_assignment_db = UserRoleAssignmentDB( user=user_db.name, role=self.roles['custom_role_unrelated_pack_action_grant'].name) UserRoleAssignment.add_or_update(role_assignment_db) user_db = self.users['custom_role_pack_action_grant_unrelated_permission'] role_assignment_db = UserRoleAssignmentDB( user=user_db.name, role=self.roles['custom_role_pack_action_grant_unrelated_permission'].name) UserRoleAssignment.add_or_update(role_assignment_db) user_db = self.users['custom_role_pack_action_view_grant'] role_assignment_db = UserRoleAssignmentDB( user=user_db.name, role=self.roles['custom_role_pack_action_view_grant'].name) UserRoleAssignment.add_or_update(role_assignment_db) user_db = self.users['custom_role_action_view_grant'] role_assignment_db = UserRoleAssignmentDB( user=user_db.name, role=self.roles['custom_role_action_view_grant'].name) UserRoleAssignment.add_or_update(role_assignment_db) user_db = self.users['custom_role_pack_action_execute_grant'] role_assignment_db = UserRoleAssignmentDB( user=user_db.name, role=self.roles['custom_role_pack_action_execute_grant'].name) UserRoleAssignment.add_or_update(role_assignment_db) user_db = self.users['custom_role_action_execute_grant'] role_assignment_db = UserRoleAssignmentDB( user=user_db.name, role=self.roles['custom_role_action_execute_grant'].name) UserRoleAssignment.add_or_update(role_assignment_db) user_db = self.users['custom_role_pack_action_all_grant'] role_assignment_db = UserRoleAssignmentDB( user=user_db.name, role=self.roles['custom_role_pack_action_all_grant'].name) UserRoleAssignment.add_or_update(role_assignment_db) user_db = self.users['custom_role_action_all_grant'] role_assignment_db = UserRoleAssignmentDB( user=user_db.name, role=self.roles['custom_role_action_all_grant'].name) UserRoleAssignment.add_or_update(role_assignment_db)
def test_migrate_executions(self): ActionExecutionDB._meta["allow_inheritance"] = True LiveActionDB._meta["allow_inheritance"] = True class ActionExecutionDB_OldFieldType(ActionExecutionDB): result = stormbase.EscapedDynamicField(default={}) class LiveActionDB_OldFieldType(LiveActionDB): result = stormbase.EscapedDynamicField(default={}) execution_dbs = ActionExecution.query( __raw__={"result": { "$not": { "$type": "binData", }, }}) self.assertEqual(len(execution_dbs), 0) execution_dbs = ActionExecution.query(__raw__={ "result": { "$type": "object", }, }) self.assertEqual(len(execution_dbs), 0) # 1. Insert data in old format liveaction_1_db = LiveActionDB_OldFieldType() liveaction_1_db.action = "foo.bar" liveaction_1_db.status = action_constants.LIVEACTION_STATUS_FAILED liveaction_1_db.result = MOCK_RESULT_1 liveaction_1_db.start_timestamp = datetime.datetime.utcnow().replace( tzinfo=datetime.timezone.utc) liveaction_1_db = LiveAction.add_or_update(liveaction_1_db, publish=False) execution_1_db = ActionExecutionDB_OldFieldType() execution_1_db.action = {"a": 1} execution_1_db.runner = {"a": 1} execution_1_db.liveaction = {"id": liveaction_1_db.id} execution_1_db.status = action_constants.LIVEACTION_STATUS_FAILED execution_1_db.result = MOCK_RESULT_1 execution_1_db.start_timestamp = datetime.datetime.utcnow().replace( tzinfo=datetime.timezone.utc) execution_1_db = ActionExecution.add_or_update(execution_1_db, publish=False) # This execution is not in a final state yet so it should not be migrated liveaction_2_db = LiveActionDB_OldFieldType() liveaction_2_db.action = "foo.bar2" liveaction_2_db.status = action_constants.LIVEACTION_STATUS_RUNNING liveaction_2_db.result = MOCK_RESULT_2 liveaction_2_db.start_timestamp = datetime.datetime.utcnow().replace( tzinfo=datetime.timezone.utc) liveaction_2_db = LiveAction.add_or_update(liveaction_2_db, publish=False) execution_2_db = ActionExecutionDB_OldFieldType() execution_2_db.action = {"a": 2} execution_2_db.runner = {"a": 2} execution_2_db.liveaction = {"id": liveaction_2_db.id} execution_2_db.status = action_constants.LIVEACTION_STATUS_RUNNING execution_2_db.result = MOCK_RESULT_2 execution_2_db.start_timestamp = datetime.datetime.utcnow().replace( tzinfo=datetime.timezone.utc) execution_2_db = ActionExecution.add_or_update(execution_2_db, publish=False) # This object is older than the default threshold so it should not be migrated execution_3_db = ActionExecutionDB_OldFieldType() execution_3_db.action = {"a": 2} execution_3_db.runner = {"a": 2} execution_3_db.liveaction = {"id": liveaction_2_db.id} execution_3_db.status = action_constants.LIVEACTION_STATUS_SUCCEEDED execution_3_db.result = MOCK_RESULT_1 execution_3_db.start_timestamp = datetime.datetime.utcfromtimestamp( 0).replace(tzinfo=datetime.timezone.utc) execution_3_db = ActionExecution.add_or_update(execution_3_db, publish=False) # Verify data has been inserted in old format execution_dbs = ActionExecution.query(__raw__={ "result": { "$type": "object", }, }) self.assertEqual(len(execution_dbs), 3) execution_dbs = ActionExecution.query( __raw__={"result": { "$not": { "$type": "binData", }, }}) self.assertEqual(len(execution_dbs), 3) execution_dbs = ActionExecution.query(__raw__={ "result": { "$type": "binData", }, }) self.assertEqual(len(execution_dbs), 0) liveaction_dbs = LiveAction.query(__raw__={ "result": { "$type": "object", }, }) self.assertEqual(len(liveaction_dbs), 2) liveaction_dbs = LiveAction.query( __raw__={"result": { "$not": { "$type": "binData", }, }}) self.assertEqual(len(liveaction_dbs), 2) liveaction_dbs = LiveAction.query(__raw__={ "result": { "$type": "binData", }, }) self.assertEqual(len(liveaction_dbs), 0) # Update inserted documents and remove special _cls field added by mongoengine. We need to # do that here due to how mongoengine works with subclasses. ActionExecution.query(__raw__={ "result": { "$type": "object", }, }).update(set___cls="ActionExecutionDB") LiveAction.query(__raw__={ "result": { "$type": "object", }, }).update(set___cls="LiveActionDB") # 2. Run migration start_dt = datetime.datetime.utcnow().replace( tzinfo=datetime.timezone.utc) - datetime.timedelta(hours=2) end_dt = datetime.datetime.utcnow().replace( tzinfo=datetime.timezone.utc) migration_module.migrate_executions(start_dt=start_dt, end_dt=end_dt) # 3. Verify data has been migrated - only 1 item should have been migrated since it's in a # completed state execution_dbs = ActionExecution.query(__raw__={ "result": { "$type": "object", }, }) self.assertEqual(len(execution_dbs), 2) execution_dbs = ActionExecution.query(__raw__={ "result": { "$type": "binData", }, }) self.assertEqual(len(execution_dbs), 1) execution_db_1_retrieved = ActionExecution.get_by_id(execution_1_db.id) self.assertEqual(execution_db_1_retrieved.result, MOCK_RESULT_1) execution_db_2_retrieved = ActionExecution.get_by_id(execution_2_db.id) self.assertEqual(execution_db_2_retrieved.result, MOCK_RESULT_2) liveaction_db_1_retrieved = LiveAction.get_by_id(liveaction_1_db.id) self.assertEqual(liveaction_db_1_retrieved.result, MOCK_RESULT_1) liveaction_db_2_retrieved = LiveAction.get_by_id(liveaction_2_db.id) self.assertEqual(liveaction_db_2_retrieved.result, MOCK_RESULT_2)