Example #1
0
    def _purge_action_executions(self):
        """
        Purge action executions and corresponding live action, stdout and stderr object which match
        the criteria defined in the config.
        """
        LOG.info('Performing garbage collection for action executions and related objects')

        utc_now = get_datetime_utc_now()
        timestamp = (utc_now - datetime.timedelta(days=self._action_executions_ttl))

        # Another sanity check to make sure we don't delete new executions
        if timestamp > (utc_now - datetime.timedelta(days=MINIMUM_TTL_DAYS)):
            raise ValueError('Calculated timestamp would violate the minimum TTL constraint')

        timestamp_str = isotime.format(dt=timestamp)
        LOG.info('Deleting action executions older than: %s' % (timestamp_str))

        assert timestamp < utc_now

        try:
            purge_executions(logger=LOG, timestamp=timestamp)
        except Exception as e:
            LOG.exception('Failed to delete executions: %s' % (six.text_type(e)))

        return True
Example #2
0
File: base.py Project: vtdat/st2
    def _purge_action_executions(self):
        """
        Purge action executions and corresponding live action, stdout and stderr object which match
        the criteria defined in the config.
        """
        LOG.info(
            'Performing garbage collection for action executions and related objects'
        )

        utc_now = get_datetime_utc_now()
        timestamp = (utc_now -
                     datetime.timedelta(days=self._action_executions_ttl))

        # Another sanity check to make sure we don't delete new executions
        if timestamp > (utc_now - datetime.timedelta(days=MINIMUM_TTL_DAYS)):
            raise ValueError(
                'Calculated timestamp would violate the minimum TTL constraint'
            )

        timestamp_str = isotime.format(dt=timestamp)
        LOG.info('Deleting action executions older than: %s' % (timestamp_str))

        assert timestamp < utc_now

        try:
            purge_executions(logger=LOG, timestamp=timestamp)
        except Exception as e:
            LOG.exception('Failed to delete executions: %s' %
                          (six.text_type(e)))

        return True
Example #3
0
    def test_liveaction_gets_deleted(self):
        now = date_utils.get_datetime_utc_now()
        start_ts = now - timedelta(days=15)
        end_ts = now - timedelta(days=14)

        liveaction_model = copy.deepcopy(
            self.models['liveactions']['liveaction4.yaml'])
        liveaction_model['start_timestamp'] = start_ts
        liveaction_model['end_timestamp'] = end_ts
        liveaction_model[
            'status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        liveaction = LiveAction.add_or_update(liveaction_model)

        # Write one execution before cut-off threshold
        exec_model = copy.deepcopy(
            self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['end_timestamp'] = end_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        exec_model['liveaction']['id'] = str(liveaction.id)
        ActionExecution.add_or_update(exec_model)

        liveactions = LiveAction.get_all()
        executions = ActionExecution.get_all()
        self.assertEqual(len(liveactions), 1)
        self.assertEqual(len(executions), 1)

        purge_executions(logger=LOG, timestamp=now - timedelta(days=10))

        liveactions = LiveAction.get_all()
        executions = ActionExecution.get_all()
        self.assertEqual(len(executions), 0)
        self.assertEqual(len(liveactions), 0)
Example #4
0
def main():
    _register_cli_opts()
    common_setup(config=config, setup_db=True, register_mq_exchanges=False)

    # Get config values
    timestamp = cfg.CONF.timestamp
    action_ref = cfg.CONF.action_ref
    purge_incomplete = cfg.CONF.purge_incomplete

    if not timestamp:
        LOG.error('Please supply a timestamp for purging models. Aborting.')
        return 1
    else:
        timestamp = datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S.%fZ')
        timestamp = timestamp.replace(tzinfo=pytz.UTC)

    try:
        purge_executions(logger=LOG, timestamp=timestamp, action_ref=action_ref,
                         purge_incomplete=purge_incomplete)
    except Exception as e:
        LOG.exception(str(e))
        return FAILURE_EXIT_CODE
    finally:
        common_teardown()

    return SUCCESS_EXIT_CODE
Example #5
0
    def test_purge_executions_whole_model_is_not_loaded_in_memory(
            self, mock_ActionExecution, mock_LiveAction):
        # Verify that whole execution objects are not loaded in memory and we just retrieve the
        # id field
        self.assertEqual(mock_ActionExecution.query.call_count, 0)
        self.assertEqual(mock_LiveAction.query.call_count, 0)
        now = date_utils.get_datetime_utc_now()
        purge_executions(logger=LOG,
                         timestamp=now - timedelta(days=10),
                         purge_incomplete=True)

        self.assertEqual(mock_ActionExecution.query.call_count, 2)
        self.assertEqual(mock_LiveAction.query.call_count, 1)

        self.assertEqual(
            mock_ActionExecution.query.call_args_list[0][1]['only_fields'],
            ['id'])
        self.assertTrue(
            mock_ActionExecution.query.call_args_list[0][1]['no_dereference'])
        self.assertEqual(
            mock_ActionExecution.query.call_args_list[1][1]['only_fields'],
            ['id'])
        self.assertTrue(
            mock_ActionExecution.query.call_args_list[1][1]['no_dereference'])
        self.assertEqual(
            mock_LiveAction.query.call_args_list[0][1]['only_fields'], ['id'])
        self.assertTrue(
            mock_LiveAction.query.call_args_list[0][1]['no_dereference'])
Example #6
0
def main():
    _register_cli_opts()
    common_setup(config=config, setup_db=True, register_mq_exchanges=False)

    # Get config values
    timestamp = cfg.CONF.timestamp
    action_ref = cfg.CONF.action_ref
    purge_incomplete = cfg.CONF.purge_incomplete

    if not timestamp:
        LOG.error('Please supply a timestamp for purging models. Aborting.')
        return 1
    else:
        timestamp = datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S.%fZ')
        timestamp = timestamp.replace(tzinfo=pytz.UTC)

    try:
        purge_executions(logger=LOG,
                         timestamp=timestamp,
                         action_ref=action_ref,
                         purge_incomplete=purge_incomplete)
    except Exception as e:
        LOG.exception(str(e))
        return FAILURE_EXIT_CODE
    finally:
        common_teardown()

    return SUCCESS_EXIT_CODE
    def test_purge_executions_with_timestamp(self):
        now = date_utils.get_datetime_utc_now()

        # Write one execution after cut-off threshold
        exec_model = copy.deepcopy(
            self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = now - timedelta(days=15)
        exec_model['end_timestamp'] = now - timedelta(days=14)
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Write one execution before cut-off threshold
        exec_model = copy.deepcopy(
            self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = now - timedelta(days=22)
        exec_model['end_timestamp'] = now - timedelta(days=21)
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        execs = ActionExecution.get_all()
        purge_executions(logger=LOG, timestamp=now - timedelta(days=20))
        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)
Example #8
0
    def test_liveaction_gets_deleted(self):
        now = date_utils.get_datetime_utc_now()
        start_ts = now - timedelta(days=15)
        end_ts = now - timedelta(days=14)

        liveaction_model = copy.deepcopy(self.models['liveactions']['liveaction4.yaml'])
        liveaction_model['start_timestamp'] = start_ts
        liveaction_model['end_timestamp'] = end_ts
        liveaction_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        liveaction = LiveAction.add_or_update(liveaction_model)

        # Write one execution before cut-off threshold
        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['end_timestamp'] = end_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        exec_model['liveaction']['id'] = str(liveaction.id)
        ActionExecution.add_or_update(exec_model)

        liveactions = LiveAction.get_all()
        executions = ActionExecution.get_all()
        self.assertEqual(len(liveactions), 1)
        self.assertEqual(len(executions), 1)

        purge_executions(logger=LOG, timestamp=now - timedelta(days=10))

        liveactions = LiveAction.get_all()
        executions = ActionExecution.get_all()
        self.assertEqual(len(executions), 0)
        self.assertEqual(len(liveactions), 0)
Example #9
0
    def _purge_action_executions(self):
        """
        Purge action executions and corresponding live action, stdout and stderr object which match
        the criteria defined in the config.
        """
        utc_now = get_datetime_utc_now()
        timestamp = utc_now - datetime.timedelta(
            days=self._action_executions_ttl)

        # Another sanity check to make sure we don't delete new executions
        if timestamp > (utc_now - datetime.timedelta(days=MINIMUM_TTL_DAYS)):
            raise ValueError(
                "Calculated timestamp would violate the minimum TTL constraint"
            )

        timestamp_str = isotime.format(dt=timestamp)
        LOG.info("Deleting action executions older than: %s" % (timestamp_str))

        if timestamp >= utc_now:
            raise ValueError(f"Calculated timestamp ({timestamp}) is"
                             f" later than now in UTC ({utc_now}).")

        try:
            purge_executions(logger=LOG, timestamp=timestamp)
        except Exception as e:
            LOG.exception("Failed to delete executions: %s" %
                          (six.text_type(e)))

        return True
    def test_purge_incomplete(self):
        now = date_utils.get_datetime_utc_now()
        start_ts = now - timedelta(days=15)

        # Write executions before cut-off threshold
        exec_model = copy.deepcopy(
            self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SCHEDULED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        exec_model = copy.deepcopy(
            self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_RUNNING
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        exec_model = copy.deepcopy(
            self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_DELAYED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        exec_model = copy.deepcopy(
            self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_CANCELING
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        exec_model = copy.deepcopy(
            self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_REQUESTED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        self.assertEqual(len(ActionExecution.get_all()), 5)
        purge_executions(logger=LOG,
                         timestamp=now - timedelta(days=10),
                         purge_incomplete=False)
        self.assertEqual(len(ActionExecution.get_all()), 5)
        purge_executions(logger=LOG,
                         timestamp=now - timedelta(days=10),
                         purge_incomplete=True)
        self.assertEqual(len(ActionExecution.get_all()), 0)
    def test_purge_executions_with_action_ref(self):
        now = date_utils.get_datetime_utc_now()
        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = now - timedelta(days=15)
        exec_model['end_timestamp'] = now - timedelta(days=14)
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)
        purge_executions(logger=LOG, action_ref='core.localzzz', timestamp=now - timedelta(days=10))
        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)

        purge_executions(logger=LOG, action_ref='core.local', timestamp=now - timedelta(days=10))
        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 0)
Example #12
0
    def test_purge_executions_whole_model_is_not_loaded_in_memory(self, mock_ActionExecution,
                                                                  mock_LiveAction):
        # Verify that whole execution objects are not loaded in memory and we just retrieve the
        # id field
        self.assertEqual(mock_ActionExecution.query.call_count, 0)
        self.assertEqual(mock_LiveAction.query.call_count, 0)
        now = date_utils.get_datetime_utc_now()
        purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=True)

        self.assertEqual(mock_ActionExecution.query.call_count, 2)
        self.assertEqual(mock_LiveAction.query.call_count, 1)

        self.assertEqual(mock_ActionExecution.query.call_args_list[0][1]['only_fields'], ['id'])
        self.assertTrue(mock_ActionExecution.query.call_args_list[0][1]['no_dereference'])
        self.assertEqual(mock_ActionExecution.query.call_args_list[1][1]['only_fields'], ['id'])
        self.assertTrue(mock_ActionExecution.query.call_args_list[1][1]['no_dereference'])
        self.assertEqual(mock_LiveAction.query.call_args_list[0][1]['only_fields'], ['id'])
        self.assertTrue(mock_LiveAction.query.call_args_list[0][1]['no_dereference'])
Example #13
0
    def test_purge_executions_with_action_ref(self):
        now = date_utils.get_datetime_utc_now()
        exec_model = copy.deepcopy(
            self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = now - timedelta(days=15)
        exec_model['end_timestamp'] = now - timedelta(days=14)
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Insert corresponding stdout and stderr db mock models
        self._insert_mock_stdout_and_stderr_objects_for_execution(
            exec_model['id'], count=3)

        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 3)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 3)

        # Invalid action reference, nothing should be deleted
        purge_executions(logger=LOG,
                         action_ref='core.localzzz',
                         timestamp=now - timedelta(days=10))

        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 3)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 3)

        purge_executions(logger=LOG,
                         action_ref='core.local',
                         timestamp=now - timedelta(days=10))

        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 0)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 0)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 0)
Example #14
0
    def test_purge_executions_with_timestamp(self):
        now = date_utils.get_datetime_utc_now()

        # Write one execution after cut-off threshold
        exec_model = copy.deepcopy(
            self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = now - timedelta(days=15)
        exec_model['end_timestamp'] = now - timedelta(days=14)
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Insert corresponding stdout and stderr db mock models
        self._insert_mock_stdout_and_stderr_objects_for_execution(
            exec_model['id'], count=3)

        # Write one execution before cut-off threshold
        exec_model = copy.deepcopy(
            self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = now - timedelta(days=22)
        exec_model['end_timestamp'] = now - timedelta(days=21)
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Insert corresponding stdout and stderr db mock models
        self._insert_mock_stdout_and_stderr_objects_for_execution(
            exec_model['id'], count=3)

        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 2)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 6)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 6)

        purge_executions(logger=LOG, timestamp=now - timedelta(days=20))
        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 3)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 3)
    def test_purge_incomplete(self):
        now = date_utils.get_datetime_utc_now()
        start_ts = now - timedelta(days=15)

        # Write executions before cut-off threshold
        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SCHEDULED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_RUNNING
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_DELAYED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_CANCELING
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_REQUESTED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        self.assertEqual(len(ActionExecution.get_all()), 5)
        purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=False)
        self.assertEqual(len(ActionExecution.get_all()), 5)
        purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=True)
        self.assertEqual(len(ActionExecution.get_all()), 0)
Example #16
0
    def test_purge_executions_with_timestamp(self):
        now = date_utils.get_datetime_utc_now()

        # Write one execution after cut-off threshold
        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = now - timedelta(days=15)
        exec_model['end_timestamp'] = now - timedelta(days=14)
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Insert corresponding stdout and stderr db mock models
        self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=3)

        # Write one execution before cut-off threshold
        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = now - timedelta(days=22)
        exec_model['end_timestamp'] = now - timedelta(days=21)
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Insert corresponding stdout and stderr db mock models
        self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=3)

        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 2)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 6)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 6)

        purge_executions(logger=LOG, timestamp=now - timedelta(days=20))
        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 3)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 3)
    def test_purge_executions_with_timestamp(self):
        now = date_utils.get_datetime_utc_now()

        # Write one execution after cut-off threshold
        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = now - timedelta(days=15)
        exec_model['end_timestamp'] = now - timedelta(days=14)
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Write one execution before cut-off threshold
        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = now - timedelta(days=22)
        exec_model['end_timestamp'] = now - timedelta(days=21)
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        execs = ActionExecution.get_all()
        purge_executions(logger=LOG, timestamp=now - timedelta(days=20))
        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)
    def test_purge_executions_with_action_ref(self):
        now = date_utils.get_datetime_utc_now()
        exec_model = copy.deepcopy(
            self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = now - timedelta(days=15)
        exec_model['end_timestamp'] = now - timedelta(days=14)
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)
        purge_executions(logger=LOG,
                         action_ref='core.localzzz',
                         timestamp=now - timedelta(days=10))
        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)

        purge_executions(logger=LOG,
                         action_ref='core.local',
                         timestamp=now - timedelta(days=10))
        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 0)
Example #19
0
    def test_purge_executions_with_action_ref(self):
        now = date_utils.get_datetime_utc_now()
        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = now - timedelta(days=15)
        exec_model['end_timestamp'] = now - timedelta(days=14)
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Insert corresponding stdout and stderr db mock models
        self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=3)

        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 3)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 3)

        # Invalid action reference, nothing should be deleted
        purge_executions(logger=LOG, action_ref='core.localzzz', timestamp=now - timedelta(days=10))

        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 3)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 3)

        purge_executions(logger=LOG, action_ref='core.local', timestamp=now - timedelta(days=10))

        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 0)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 0)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 0)
Example #20
0
    def test_purge_incomplete(self):
        now = date_utils.get_datetime_utc_now()
        start_ts = now - timedelta(days=15)

        # Write executions before cut-off threshold
        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SCHEDULED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Insert corresponding stdout and stderr db mock models
        self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=1)

        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_RUNNING
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Insert corresponding stdout and stderr db mock models
        self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=1)

        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_DELAYED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Insert corresponding stdout and stderr db mock models
        self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=1)

        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_CANCELING
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Insert corresponding stdout and stderr db mock models
        self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=1)

        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_REQUESTED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Insert corresponding stdout and stderr db mock models
        self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=1)

        self.assertEqual(len(ActionExecution.get_all()), 5)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 5)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 5)

        # Incompleted executions shouldnt be purged
        purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=False)
        self.assertEqual(len(ActionExecution.get_all()), 5)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 5)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 5)

        purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=True)
        self.assertEqual(len(ActionExecution.get_all()), 0)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 0)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 0)
Example #21
0
    def test_purge_incomplete(self):
        now = date_utils.get_datetime_utc_now()
        start_ts = now - timedelta(days=15)

        # Write executions before cut-off threshold
        exec_model = copy.deepcopy(
            self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SCHEDULED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Insert corresponding stdout and stderr db mock models
        self._insert_mock_stdout_and_stderr_objects_for_execution(
            exec_model['id'], count=1)

        exec_model = copy.deepcopy(
            self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_RUNNING
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Insert corresponding stdout and stderr db mock models
        self._insert_mock_stdout_and_stderr_objects_for_execution(
            exec_model['id'], count=1)

        exec_model = copy.deepcopy(
            self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_DELAYED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Insert corresponding stdout and stderr db mock models
        self._insert_mock_stdout_and_stderr_objects_for_execution(
            exec_model['id'], count=1)

        exec_model = copy.deepcopy(
            self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_CANCELING
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Insert corresponding stdout and stderr db mock models
        self._insert_mock_stdout_and_stderr_objects_for_execution(
            exec_model['id'], count=1)

        exec_model = copy.deepcopy(
            self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_REQUESTED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Insert corresponding stdout and stderr db mock models
        self._insert_mock_stdout_and_stderr_objects_for_execution(
            exec_model['id'], count=1)

        self.assertEqual(len(ActionExecution.get_all()), 5)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 5)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 5)

        # Incompleted executions shouldnt be purged
        purge_executions(logger=LOG,
                         timestamp=now - timedelta(days=10),
                         purge_incomplete=False)
        self.assertEqual(len(ActionExecution.get_all()), 5)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 5)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 5)

        purge_executions(logger=LOG,
                         timestamp=now - timedelta(days=10),
                         purge_incomplete=True)
        self.assertEqual(len(ActionExecution.get_all()), 0)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 0)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 0)