Exemple #1
0
    def test_liveaction_gets_deleted(self):
        now = date_utils.get_datetime_utc_now()
        start_ts = now - timedelta(days=15)
        end_ts = now - timedelta(days=14)

        liveaction_model = copy.deepcopy(
            self.models['liveactions']['liveaction4.yaml'])
        liveaction_model['start_timestamp'] = start_ts
        liveaction_model['end_timestamp'] = end_ts
        liveaction_model[
            'status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        liveaction = LiveAction.add_or_update(liveaction_model)

        # Write one execution before cut-off threshold
        exec_model = copy.deepcopy(
            self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['end_timestamp'] = end_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        exec_model['liveaction']['id'] = str(liveaction.id)
        ActionExecution.add_or_update(exec_model)

        liveactions = LiveAction.get_all()
        executions = ActionExecution.get_all()
        self.assertEqual(len(liveactions), 1)
        self.assertEqual(len(executions), 1)

        purge_executions(logger=LOG, timestamp=now - timedelta(days=10))

        liveactions = LiveAction.get_all()
        executions = ActionExecution.get_all()
        self.assertEqual(len(executions), 0)
        self.assertEqual(len(liveactions), 0)
Exemple #2
0
    def test_no_timestamp_doesnt_delete_things(self):
        now = date_utils.get_datetime_utc_now()
        exec_model = copy.deepcopy(
            self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = now - timedelta(days=15)
        exec_model['end_timestamp'] = now - timedelta(days=14)
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Insert corresponding stdout and stderr db mock models
        self._insert_mock_stdout_and_stderr_objects_for_execution(
            exec_model['id'], count=3)

        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 3)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 3)

        expected_msg = 'Specify a valid timestamp'
        self.assertRaisesRegexp(ValueError,
                                expected_msg,
                                purge_executions,
                                logger=LOG,
                                timestamp=None)
        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 3)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 3)
    def test_purge_executions_with_timestamp(self):
        now = date_utils.get_datetime_utc_now()

        # Write one execution after cut-off threshold
        exec_model = copy.deepcopy(
            self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = now - timedelta(days=15)
        exec_model['end_timestamp'] = now - timedelta(days=14)
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Write one execution before cut-off threshold
        exec_model = copy.deepcopy(
            self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = now - timedelta(days=22)
        exec_model['end_timestamp'] = now - timedelta(days=21)
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        execs = ActionExecution.get_all()
        purge_executions(logger=LOG, timestamp=now - timedelta(days=20))
        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)
Exemple #4
0
    def test_no_timestamp_doesnt_delete_things(self):
        now = date_utils.get_datetime_utc_now()
        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = now - timedelta(days=15)
        exec_model['end_timestamp'] = now - timedelta(days=14)
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Insert corresponding stdout and stderr db mock models
        self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=3)

        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 3)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 3)

        expected_msg = 'Specify a valid timestamp'
        self.assertRaisesRegexp(ValueError, expected_msg, purge_executions,
                                logger=LOG, timestamp=None)
        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 3)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 3)
Exemple #5
0
    def test_liveaction_gets_deleted(self):
        now = date_utils.get_datetime_utc_now()
        start_ts = now - timedelta(days=15)
        end_ts = now - timedelta(days=14)

        liveaction_model = copy.deepcopy(self.models['liveactions']['liveaction4.yaml'])
        liveaction_model['start_timestamp'] = start_ts
        liveaction_model['end_timestamp'] = end_ts
        liveaction_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        liveaction = LiveAction.add_or_update(liveaction_model)

        # Write one execution before cut-off threshold
        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['end_timestamp'] = end_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        exec_model['liveaction']['id'] = str(liveaction.id)
        ActionExecution.add_or_update(exec_model)

        liveactions = LiveAction.get_all()
        executions = ActionExecution.get_all()
        self.assertEqual(len(liveactions), 1)
        self.assertEqual(len(executions), 1)

        purge_executions(logger=LOG, timestamp=now - timedelta(days=10))

        liveactions = LiveAction.get_all()
        executions = ActionExecution.get_all()
        self.assertEqual(len(executions), 0)
        self.assertEqual(len(liveactions), 0)
Exemple #6
0
    def test_no_timestamp_doesnt_delete_things(self):
        now = date_utils.get_datetime_utc_now()
        exec_model = copy.copy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = now - timedelta(days=15)
        exec_model['end_timestamp'] = now - timedelta(days=14)
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)
        purge_executions()
        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)
    def test_purge_incomplete(self):
        now = date_utils.get_datetime_utc_now()
        start_ts = now - timedelta(days=15)

        # Write executions before cut-off threshold
        exec_model = copy.deepcopy(
            self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SCHEDULED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        exec_model = copy.deepcopy(
            self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_RUNNING
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        exec_model = copy.deepcopy(
            self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_DELAYED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        exec_model = copy.deepcopy(
            self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_CANCELING
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        exec_model = copy.deepcopy(
            self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_REQUESTED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        self.assertEqual(len(ActionExecution.get_all()), 5)
        purge_executions(logger=LOG,
                         timestamp=now - timedelta(days=10),
                         purge_incomplete=False)
        self.assertEqual(len(ActionExecution.get_all()), 5)
        purge_executions(logger=LOG,
                         timestamp=now - timedelta(days=10),
                         purge_incomplete=True)
        self.assertEqual(len(ActionExecution.get_all()), 0)
Exemple #8
0
    def test_garbage_collection(self):
        now = date_utils.get_datetime_utc_now()
        status = action_constants.LIVEACTION_STATUS_SUCCEEDED

        # Insert come mock ActionExecutionDB objects with start_timestamp < TTL defined in the
        # config
        old_executions_count = 15
        ttl_days = 30
        timestamp = (now - datetime.timedelta(days=ttl_days))
        for index in range(0, old_executions_count):
            action_execution_db = ActionExecutionDB(
                start_timestamp=timestamp,
                end_timestamp=timestamp,
                status=status,
                action={'ref': 'core.local'},
                runner={'name': 'run-local'},
                liveaction={'ref': 'foo'})
            ActionExecution.add_or_update(action_execution_db)

        # Insert come mock ActionExecutionDB objects with start_timestamp > TTL defined in the
        # config
        new_executions_count = 5
        ttl_days = 2
        timestamp = (now - datetime.timedelta(days=ttl_days))
        for index in range(0, new_executions_count):
            action_execution_db = ActionExecutionDB(
                start_timestamp=timestamp,
                end_timestamp=timestamp,
                status=status,
                action={'ref': 'core.local'},
                runner={'name': 'run-local'},
                liveaction={'ref': 'foo'})
            ActionExecution.add_or_update(action_execution_db)

        execs = ActionExecution.get_all()
        self.assertEqual(len(execs),
                         (old_executions_count + new_executions_count))

        # Start garbage collector
        process = self._start_garbage_collector()

        # Give it some time to perform garbage collection and kill it
        eventlet.sleep(5)
        process.send_signal(signal.SIGKILL)
        self.remove_process(process=process)

        # Old execution should have been garbage collected
        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), (new_executions_count))
    def test_garbage_collection(self):
        now = date_utils.get_datetime_utc_now()
        status = action_constants.LIVEACTION_STATUS_SUCCEEDED

        # Insert come mock ActionExecutionDB objects with start_timestamp < TTL defined in the
        # config
        old_executions_count = 15
        ttl_days = 30
        timestamp = (now - datetime.timedelta(days=ttl_days))
        for index in range(0, old_executions_count):
            action_execution_db = ActionExecutionDB(start_timestamp=timestamp,
                                                    end_timestamp=timestamp,
                                                    status=status,
                                                    action={'ref': 'core.local'},
                                                    runner={'name': 'run-local'},
                                                    liveaction={'ref': 'foo'})
            ActionExecution.add_or_update(action_execution_db)

        # Insert come mock ActionExecutionDB objects with start_timestamp > TTL defined in the
        # config
        new_executions_count = 5
        ttl_days = 2
        timestamp = (now - datetime.timedelta(days=ttl_days))
        for index in range(0, new_executions_count):
            action_execution_db = ActionExecutionDB(start_timestamp=timestamp,
                                                    end_timestamp=timestamp,
                                                    status=status,
                                                    action={'ref': 'core.local'},
                                                    runner={'name': 'run-local'},
                                                    liveaction={'ref': 'foo'})
            ActionExecution.add_or_update(action_execution_db)

        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), (old_executions_count + new_executions_count))

        # Start garbage collector
        process = self._start_garbage_collector()

        # Give it some time to perform garbage collection and kill it
        eventlet.sleep(5)
        process.send_signal(signal.SIGKILL)
        self.remove_process(process=process)

        # Old execution should have been garbage collected
        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), (new_executions_count))
    def test_no_timestamp_doesnt_delete_things(self):
        now = date_utils.get_datetime_utc_now()
        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = now - timedelta(days=15)
        exec_model['end_timestamp'] = now - timedelta(days=14)
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)

        expected_msg = 'Specify a valid timestamp'
        self.assertRaisesRegexp(ValueError, expected_msg, purge_executions,
                                logger=LOG, timestamp=None)
        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)
    def test_purge_executions_with_action_ref(self):
        now = date_utils.get_datetime_utc_now()
        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = now - timedelta(days=15)
        exec_model['end_timestamp'] = now - timedelta(days=14)
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)
        purge_executions(logger=LOG, action_ref='core.localzzz', timestamp=now - timedelta(days=10))
        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)

        purge_executions(logger=LOG, action_ref='core.local', timestamp=now - timedelta(days=10))
        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 0)
Exemple #12
0
    def test_purge_executions_with_action_ref(self):
        now = date_utils.get_datetime_utc_now()
        exec_model = copy.copy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = now - timedelta(days=15)
        exec_model['end_timestamp'] = now - timedelta(days=14)
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)
        purge_executions(action_ref='core.localzzz', timestamp=now - timedelta(days=10))
        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)

        purge_executions(action_ref='core.local', timestamp=now - timedelta(days=10))
        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 0)
Exemple #13
0
    def test_purge_executions_with_action_ref(self):
        now = date_utils.get_datetime_utc_now()
        exec_model = copy.deepcopy(
            self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = now - timedelta(days=15)
        exec_model['end_timestamp'] = now - timedelta(days=14)
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Insert corresponding stdout and stderr db mock models
        self._insert_mock_stdout_and_stderr_objects_for_execution(
            exec_model['id'], count=3)

        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 3)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 3)

        # Invalid action reference, nothing should be deleted
        purge_executions(logger=LOG,
                         action_ref='core.localzzz',
                         timestamp=now - timedelta(days=10))

        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 3)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 3)

        purge_executions(logger=LOG,
                         action_ref='core.local',
                         timestamp=now - timedelta(days=10))

        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 0)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 0)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 0)
Exemple #14
0
    def test_purge_executions_with_timestamp(self):
        now = date_utils.get_datetime_utc_now()

        # Write one execution after cut-off threshold
        exec_model = copy.deepcopy(
            self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = now - timedelta(days=15)
        exec_model['end_timestamp'] = now - timedelta(days=14)
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Insert corresponding stdout and stderr db mock models
        self._insert_mock_stdout_and_stderr_objects_for_execution(
            exec_model['id'], count=3)

        # Write one execution before cut-off threshold
        exec_model = copy.deepcopy(
            self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = now - timedelta(days=22)
        exec_model['end_timestamp'] = now - timedelta(days=21)
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Insert corresponding stdout and stderr db mock models
        self._insert_mock_stdout_and_stderr_objects_for_execution(
            exec_model['id'], count=3)

        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 2)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 6)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 6)

        purge_executions(logger=LOG, timestamp=now - timedelta(days=20))
        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 3)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 3)
    def test_no_timestamp_doesnt_delete_things(self):
        now = date_utils.get_datetime_utc_now()
        exec_model = copy.deepcopy(
            self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = now - timedelta(days=15)
        exec_model['end_timestamp'] = now - timedelta(days=14)
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)

        expected_msg = 'Specify a valid timestamp'
        self.assertRaisesRegexp(ValueError,
                                expected_msg,
                                purge_executions,
                                logger=LOG,
                                timestamp=None)
        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)
    def test_purge_incomplete(self):
        now = date_utils.get_datetime_utc_now()
        start_ts = now - timedelta(days=15)

        # Write executions before cut-off threshold
        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SCHEDULED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_RUNNING
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_DELAYED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_CANCELING
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_REQUESTED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        self.assertEqual(len(ActionExecution.get_all()), 5)
        purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=False)
        self.assertEqual(len(ActionExecution.get_all()), 5)
        purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=True)
        self.assertEqual(len(ActionExecution.get_all()), 0)
Exemple #17
0
    def test_purge_executions_with_timestamp(self):
        now = date_utils.get_datetime_utc_now()

        # Write one execution after cut-off threshold
        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = now - timedelta(days=15)
        exec_model['end_timestamp'] = now - timedelta(days=14)
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Insert corresponding stdout and stderr db mock models
        self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=3)

        # Write one execution before cut-off threshold
        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = now - timedelta(days=22)
        exec_model['end_timestamp'] = now - timedelta(days=21)
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Insert corresponding stdout and stderr db mock models
        self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=3)

        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 2)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 6)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 6)

        purge_executions(logger=LOG, timestamp=now - timedelta(days=20))
        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 3)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 3)
    def test_purge_executions_with_timestamp(self):
        now = date_utils.get_datetime_utc_now()

        # Write one execution after cut-off threshold
        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = now - timedelta(days=15)
        exec_model['end_timestamp'] = now - timedelta(days=14)
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Write one execution before cut-off threshold
        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = now - timedelta(days=22)
        exec_model['end_timestamp'] = now - timedelta(days=21)
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        execs = ActionExecution.get_all()
        purge_executions(logger=LOG, timestamp=now - timedelta(days=20))
        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)
Exemple #19
0
    def test_purge_executions_with_action_ref(self):
        now = date_utils.get_datetime_utc_now()
        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = now - timedelta(days=15)
        exec_model['end_timestamp'] = now - timedelta(days=14)
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Insert corresponding stdout and stderr db mock models
        self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=3)

        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 3)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 3)

        # Invalid action reference, nothing should be deleted
        purge_executions(logger=LOG, action_ref='core.localzzz', timestamp=now - timedelta(days=10))

        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 3)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 3)

        purge_executions(logger=LOG, action_ref='core.local', timestamp=now - timedelta(days=10))

        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 0)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 0)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 0)
Exemple #20
0
 def _get_all_executions_from_db(self):
     return ActionExecution.get_all()  # XXX: Paginated call.
Exemple #21
0
    def test_garbage_collection(self):
        now = date_utils.get_datetime_utc_now()
        status = action_constants.LIVEACTION_STATUS_SUCCEEDED

        # Insert come mock ActionExecutionDB objects with start_timestamp < TTL defined in the
        # config
        old_executions_count = 15
        ttl_days = 30  # > 20
        timestamp = (now - datetime.timedelta(days=ttl_days))
        for index in range(0, old_executions_count):
            action_execution_db = ActionExecutionDB(start_timestamp=timestamp,
                                                    end_timestamp=timestamp,
                                                    status=status,
                                                    action={'ref': 'core.local'},
                                                    runner={'name': 'local-shell-cmd'},
                                                    liveaction={'ref': 'foo'})
            ActionExecution.add_or_update(action_execution_db)

            stdout_db = ActionExecutionOutputDB(execution_id=str(action_execution_db.id),
                                                action_ref='core.local',
                                                runner_ref='dummy',
                                                timestamp=timestamp,
                                                output_type='stdout',
                                                data='stdout')
            ActionExecutionOutput.add_or_update(stdout_db)

            stderr_db = ActionExecutionOutputDB(execution_id=str(action_execution_db.id),
                                                action_ref='core.local',
                                                runner_ref='dummy',
                                                timestamp=timestamp,
                                                output_type='stderr',
                                                data='stderr')
            ActionExecutionOutput.add_or_update(stderr_db)

        # Insert come mock ActionExecutionDB objects with start_timestamp > TTL defined in the
        # config
        new_executions_count = 5
        ttl_days = 2  # < 20
        timestamp = (now - datetime.timedelta(days=ttl_days))
        for index in range(0, new_executions_count):
            action_execution_db = ActionExecutionDB(start_timestamp=timestamp,
                                                    end_timestamp=timestamp,
                                                    status=status,
                                                    action={'ref': 'core.local'},
                                                    runner={'name': 'local-shell-cmd'},
                                                    liveaction={'ref': 'foo'})
            ActionExecution.add_or_update(action_execution_db)

            stdout_db = ActionExecutionOutputDB(execution_id=str(action_execution_db.id),
                                                action_ref='core.local',
                                                runner_ref='dummy',
                                                timestamp=timestamp,
                                                output_type='stdout',
                                                data='stdout')
            ActionExecutionOutput.add_or_update(stdout_db)

            stderr_db = ActionExecutionOutputDB(execution_id=str(action_execution_db.id),
                                                action_ref='core.local',
                                                runner_ref='dummy',
                                                timestamp=timestamp,
                                                output_type='stderr',
                                                data='stderr')
            ActionExecutionOutput.add_or_update(stderr_db)

        # Insert some mock output objects where start_timestamp > action_executions_output_ttl
        new_output_count = 5
        ttl_days = 15  # > 10 and < 20
        timestamp = (now - datetime.timedelta(days=ttl_days))
        for index in range(0, new_output_count):
            action_execution_db = ActionExecutionDB(start_timestamp=timestamp,
                                                    end_timestamp=timestamp,
                                                    status=status,
                                                    action={'ref': 'core.local'},
                                                    runner={'name': 'local-shell-cmd'},
                                                    liveaction={'ref': 'foo'})
            ActionExecution.add_or_update(action_execution_db)

            stdout_db = ActionExecutionOutputDB(execution_id=str(action_execution_db.id),
                                                action_ref='core.local',
                                                runner_ref='dummy',
                                                timestamp=timestamp,
                                                output_type='stdout',
                                                data='stdout')
            ActionExecutionOutput.add_or_update(stdout_db)

            stderr_db = ActionExecutionOutputDB(execution_id=str(action_execution_db.id),
                                                action_ref='core.local',
                                                runner_ref='dummy',
                                                timestamp=timestamp,
                                                output_type='stderr',
                                                data='stderr')
            ActionExecutionOutput.add_or_update(stderr_db)

        execs = ActionExecution.get_all()
        self.assertEqual(len(execs),
                         (old_executions_count + new_executions_count + new_output_count))

        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs),
                         (old_executions_count + new_executions_count + new_output_count))

        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs),
                         (old_executions_count + new_executions_count + new_output_count))

        # Start garbage collector
        process = self._start_garbage_collector()

        # Give it some time to perform garbage collection and kill it
        eventlet.sleep(15)
        process.send_signal(signal.SIGKILL)
        self.remove_process(process=process)

        # Old executions and corresponding objects should have been garbage collected
        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), (new_executions_count + new_output_count))

        # Collection for output objects older than 10 days is also enabled, so those objects
        # should be deleted as well
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), (new_executions_count))

        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), (new_executions_count))
Exemple #22
0
    def test_purge_incomplete(self):
        now = date_utils.get_datetime_utc_now()
        start_ts = now - timedelta(days=15)

        # Write executions before cut-off threshold
        exec_model = copy.deepcopy(
            self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SCHEDULED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Insert corresponding stdout and stderr db mock models
        self._insert_mock_stdout_and_stderr_objects_for_execution(
            exec_model['id'], count=1)

        exec_model = copy.deepcopy(
            self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_RUNNING
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Insert corresponding stdout and stderr db mock models
        self._insert_mock_stdout_and_stderr_objects_for_execution(
            exec_model['id'], count=1)

        exec_model = copy.deepcopy(
            self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_DELAYED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Insert corresponding stdout and stderr db mock models
        self._insert_mock_stdout_and_stderr_objects_for_execution(
            exec_model['id'], count=1)

        exec_model = copy.deepcopy(
            self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_CANCELING
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Insert corresponding stdout and stderr db mock models
        self._insert_mock_stdout_and_stderr_objects_for_execution(
            exec_model['id'], count=1)

        exec_model = copy.deepcopy(
            self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_REQUESTED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Insert corresponding stdout and stderr db mock models
        self._insert_mock_stdout_and_stderr_objects_for_execution(
            exec_model['id'], count=1)

        self.assertEqual(len(ActionExecution.get_all()), 5)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 5)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 5)

        # Incompleted executions shouldnt be purged
        purge_executions(logger=LOG,
                         timestamp=now - timedelta(days=10),
                         purge_incomplete=False)
        self.assertEqual(len(ActionExecution.get_all()), 5)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 5)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 5)

        purge_executions(logger=LOG,
                         timestamp=now - timedelta(days=10),
                         purge_incomplete=True)
        self.assertEqual(len(ActionExecution.get_all()), 0)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 0)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 0)
Exemple #23
0
    def test_purge_incomplete(self):
        now = date_utils.get_datetime_utc_now()
        start_ts = now - timedelta(days=15)

        # Write executions before cut-off threshold
        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SCHEDULED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Insert corresponding stdout and stderr db mock models
        self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=1)

        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_RUNNING
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Insert corresponding stdout and stderr db mock models
        self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=1)

        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_DELAYED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Insert corresponding stdout and stderr db mock models
        self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=1)

        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_CANCELING
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Insert corresponding stdout and stderr db mock models
        self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=1)

        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_REQUESTED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Insert corresponding stdout and stderr db mock models
        self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=1)

        self.assertEqual(len(ActionExecution.get_all()), 5)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 5)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 5)

        # Incompleted executions shouldnt be purged
        purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=False)
        self.assertEqual(len(ActionExecution.get_all()), 5)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 5)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 5)

        purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=True)
        self.assertEqual(len(ActionExecution.get_all()), 0)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 0)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 0)
Exemple #24
0
    def test_garbage_collection(self):
        now = date_utils.get_datetime_utc_now()
        status = action_constants.LIVEACTION_STATUS_SUCCEEDED

        # Insert come mock ActionExecutionDB objects with start_timestamp < TTL defined in the
        # config
        old_executions_count = 15
        ttl_days = 30  # > 20
        timestamp = (now - datetime.timedelta(days=ttl_days))
        for index in range(0, old_executions_count):
            action_execution_db = ActionExecutionDB(
                start_timestamp=timestamp,
                end_timestamp=timestamp,
                status=status,
                action={'ref': 'core.local'},
                runner={'name': 'local-shell-cmd'},
                liveaction={'ref': 'foo'})
            ActionExecution.add_or_update(action_execution_db)

            stdout_db = ActionExecutionOutputDB(execution_id=str(
                action_execution_db.id),
                                                action_ref='core.local',
                                                runner_ref='dummy',
                                                timestamp=timestamp,
                                                output_type='stdout',
                                                data='stdout')
            ActionExecutionOutput.add_or_update(stdout_db)

            stderr_db = ActionExecutionOutputDB(execution_id=str(
                action_execution_db.id),
                                                action_ref='core.local',
                                                runner_ref='dummy',
                                                timestamp=timestamp,
                                                output_type='stderr',
                                                data='stderr')
            ActionExecutionOutput.add_or_update(stderr_db)

        # Insert come mock ActionExecutionDB objects with start_timestamp > TTL defined in the
        # config
        new_executions_count = 5
        ttl_days = 2  # < 20
        timestamp = (now - datetime.timedelta(days=ttl_days))
        for index in range(0, new_executions_count):
            action_execution_db = ActionExecutionDB(
                start_timestamp=timestamp,
                end_timestamp=timestamp,
                status=status,
                action={'ref': 'core.local'},
                runner={'name': 'local-shell-cmd'},
                liveaction={'ref': 'foo'})
            ActionExecution.add_or_update(action_execution_db)

            stdout_db = ActionExecutionOutputDB(execution_id=str(
                action_execution_db.id),
                                                action_ref='core.local',
                                                runner_ref='dummy',
                                                timestamp=timestamp,
                                                output_type='stdout',
                                                data='stdout')
            ActionExecutionOutput.add_or_update(stdout_db)

            stderr_db = ActionExecutionOutputDB(execution_id=str(
                action_execution_db.id),
                                                action_ref='core.local',
                                                runner_ref='dummy',
                                                timestamp=timestamp,
                                                output_type='stderr',
                                                data='stderr')
            ActionExecutionOutput.add_or_update(stderr_db)

        # Insert some mock output objects where start_timestamp > action_executions_output_ttl
        new_output_count = 5
        ttl_days = 15  # > 10 and < 20
        timestamp = (now - datetime.timedelta(days=ttl_days))
        for index in range(0, new_output_count):
            action_execution_db = ActionExecutionDB(
                start_timestamp=timestamp,
                end_timestamp=timestamp,
                status=status,
                action={'ref': 'core.local'},
                runner={'name': 'local-shell-cmd'},
                liveaction={'ref': 'foo'})
            ActionExecution.add_or_update(action_execution_db)

            stdout_db = ActionExecutionOutputDB(execution_id=str(
                action_execution_db.id),
                                                action_ref='core.local',
                                                runner_ref='dummy',
                                                timestamp=timestamp,
                                                output_type='stdout',
                                                data='stdout')
            ActionExecutionOutput.add_or_update(stdout_db)

            stderr_db = ActionExecutionOutputDB(execution_id=str(
                action_execution_db.id),
                                                action_ref='core.local',
                                                runner_ref='dummy',
                                                timestamp=timestamp,
                                                output_type='stderr',
                                                data='stderr')
            ActionExecutionOutput.add_or_update(stderr_db)

        execs = ActionExecution.get_all()
        self.assertEqual(
            len(execs),
            (old_executions_count + new_executions_count + new_output_count))

        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(
            len(stdout_dbs),
            (old_executions_count + new_executions_count + new_output_count))

        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(
            len(stderr_dbs),
            (old_executions_count + new_executions_count + new_output_count))

        # Start garbage collector
        process = self._start_garbage_collector()

        # Give it some time to perform garbage collection and kill it
        eventlet.sleep(15)
        process.send_signal(signal.SIGKILL)
        self.remove_process(process=process)

        # Old executions and corresponding objects should have been garbage collected
        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), (new_executions_count + new_output_count))

        # Collection for output objects older than 10 days is also enabled, so those objects
        # should be deleted as well
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), (new_executions_count))

        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), (new_executions_count))