def _purge_action_executions(self):
"""
Purge action executions and corresponding live action, stdout and stderr object which match
the criteria defined in the config.
"""
LOG.info('Performing garbage collection for action executions and related objects')
utc_now = get_datetime_utc_now()
timestamp = (utc_now - datetime.timedelta(days=self._action_executions_ttl))
# Another sanity check to make sure we don't delete new executions
if timestamp > (utc_now - datetime.timedelta(days=MINIMUM_TTL_DAYS)):
raise ValueError('Calculated timestamp would violate the minimum TTL constraint')
timestamp_str = isotime.format(dt=timestamp)
LOG.info('Deleting action executions older than: %s' % (timestamp_str))
assert timestamp < utc_now
try:
purge_executions(logger=LOG, timestamp=timestamp)
except Exception as e:
LOG.exception('Failed to delete executions: %s' % (six.text_type(e)))
return True
def _purge_action_executions(self):
"""
Purge action executions and corresponding live action, stdout and stderr object which match
the criteria defined in the config.
"""
LOG.info(
'Performing garbage collection for action executions and related objects'
)
utc_now = get_datetime_utc_now()
timestamp = (utc_now -
datetime.timedelta(days=self._action_executions_ttl))
# Another sanity check to make sure we don't delete new executions
if timestamp > (utc_now - datetime.timedelta(days=MINIMUM_TTL_DAYS)):
raise ValueError(
'Calculated timestamp would violate the minimum TTL constraint'
)
timestamp_str = isotime.format(dt=timestamp)
LOG.info('Deleting action executions older than: %s' % (timestamp_str))
assert timestamp < utc_now
try:
purge_executions(logger=LOG, timestamp=timestamp)
except Exception as e:
LOG.exception('Failed to delete executions: %s' %
(six.text_type(e)))
return True
def test_liveaction_gets_deleted(self):
now = date_utils.get_datetime_utc_now()
start_ts = now - timedelta(days=15)
end_ts = now - timedelta(days=14)
liveaction_model = copy.deepcopy(
self.models['liveactions']['liveaction4.yaml'])
liveaction_model['start_timestamp'] = start_ts
liveaction_model['end_timestamp'] = end_ts
liveaction_model[
'status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
liveaction = LiveAction.add_or_update(liveaction_model)
# Write one execution before cut-off threshold
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['end_timestamp'] = end_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
exec_model['liveaction']['id'] = str(liveaction.id)
ActionExecution.add_or_update(exec_model)
liveactions = LiveAction.get_all()
executions = ActionExecution.get_all()
self.assertEqual(len(liveactions), 1)
self.assertEqual(len(executions), 1)
purge_executions(logger=LOG, timestamp=now - timedelta(days=10))
liveactions = LiveAction.get_all()
executions = ActionExecution.get_all()
self.assertEqual(len(executions), 0)
self.assertEqual(len(liveactions), 0)
def main():
_register_cli_opts()
common_setup(config=config, setup_db=True, register_mq_exchanges=False)
# Get config values
timestamp = cfg.CONF.timestamp
action_ref = cfg.CONF.action_ref
purge_incomplete = cfg.CONF.purge_incomplete
if not timestamp:
LOG.error('Please supply a timestamp for purging models. Aborting.')
return 1
else:
timestamp = datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S.%fZ')
timestamp = timestamp.replace(tzinfo=pytz.UTC)
try:
purge_executions(logger=LOG, timestamp=timestamp, action_ref=action_ref,
purge_incomplete=purge_incomplete)
except Exception as e:
LOG.exception(str(e))
return FAILURE_EXIT_CODE
finally:
common_teardown()
return SUCCESS_EXIT_CODE
def test_purge_executions_whole_model_is_not_loaded_in_memory(
self, mock_ActionExecution, mock_LiveAction):
# Verify that whole execution objects are not loaded in memory and we just retrieve the
# id field
self.assertEqual(mock_ActionExecution.query.call_count, 0)
self.assertEqual(mock_LiveAction.query.call_count, 0)
now = date_utils.get_datetime_utc_now()
purge_executions(logger=LOG,
timestamp=now - timedelta(days=10),
purge_incomplete=True)
self.assertEqual(mock_ActionExecution.query.call_count, 2)
self.assertEqual(mock_LiveAction.query.call_count, 1)
self.assertEqual(
mock_ActionExecution.query.call_args_list[0][1]['only_fields'],
['id'])
self.assertTrue(
mock_ActionExecution.query.call_args_list[0][1]['no_dereference'])
self.assertEqual(
mock_ActionExecution.query.call_args_list[1][1]['only_fields'],
['id'])
self.assertTrue(
mock_ActionExecution.query.call_args_list[1][1]['no_dereference'])
self.assertEqual(
mock_LiveAction.query.call_args_list[0][1]['only_fields'], ['id'])
self.assertTrue(
mock_LiveAction.query.call_args_list[0][1]['no_dereference'])
def main():
_register_cli_opts()
common_setup(config=config, setup_db=True, register_mq_exchanges=False)
# Get config values
timestamp = cfg.CONF.timestamp
action_ref = cfg.CONF.action_ref
purge_incomplete = cfg.CONF.purge_incomplete
if not timestamp:
LOG.error('Please supply a timestamp for purging models. Aborting.')
return 1
else:
timestamp = datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S.%fZ')
timestamp = timestamp.replace(tzinfo=pytz.UTC)
try:
purge_executions(logger=LOG,
timestamp=timestamp,
action_ref=action_ref,
purge_incomplete=purge_incomplete)
except Exception as e:
LOG.exception(str(e))
return FAILURE_EXIT_CODE
finally:
common_teardown()
return SUCCESS_EXIT_CODE
def test_purge_executions_with_timestamp(self):
now = date_utils.get_datetime_utc_now()
# Write one execution after cut-off threshold
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=15)
exec_model['end_timestamp'] = now - timedelta(days=14)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Write one execution before cut-off threshold
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=22)
exec_model['end_timestamp'] = now - timedelta(days=21)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
execs = ActionExecution.get_all()
purge_executions(logger=LOG, timestamp=now - timedelta(days=20))
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
def test_liveaction_gets_deleted(self):
now = date_utils.get_datetime_utc_now()
start_ts = now - timedelta(days=15)
end_ts = now - timedelta(days=14)
liveaction_model = copy.deepcopy(self.models['liveactions']['liveaction4.yaml'])
liveaction_model['start_timestamp'] = start_ts
liveaction_model['end_timestamp'] = end_ts
liveaction_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
liveaction = LiveAction.add_or_update(liveaction_model)
# Write one execution before cut-off threshold
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['end_timestamp'] = end_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
exec_model['liveaction']['id'] = str(liveaction.id)
ActionExecution.add_or_update(exec_model)
liveactions = LiveAction.get_all()
executions = ActionExecution.get_all()
self.assertEqual(len(liveactions), 1)
self.assertEqual(len(executions), 1)
purge_executions(logger=LOG, timestamp=now - timedelta(days=10))
liveactions = LiveAction.get_all()
executions = ActionExecution.get_all()
self.assertEqual(len(executions), 0)
self.assertEqual(len(liveactions), 0)
def _purge_action_executions(self):
"""
Purge action executions and corresponding live action, stdout and stderr object which match
the criteria defined in the config.
"""
utc_now = get_datetime_utc_now()
timestamp = utc_now - datetime.timedelta(
days=self._action_executions_ttl)
# Another sanity check to make sure we don't delete new executions
if timestamp > (utc_now - datetime.timedelta(days=MINIMUM_TTL_DAYS)):
raise ValueError(
"Calculated timestamp would violate the minimum TTL constraint"
)
timestamp_str = isotime.format(dt=timestamp)
LOG.info("Deleting action executions older than: %s" % (timestamp_str))
if timestamp >= utc_now:
raise ValueError(f"Calculated timestamp ({timestamp}) is"
f" later than now in UTC ({utc_now}).")
try:
purge_executions(logger=LOG, timestamp=timestamp)
except Exception as e:
LOG.exception("Failed to delete executions: %s" %
(six.text_type(e)))
return True
def test_purge_incomplete(self):
now = date_utils.get_datetime_utc_now()
start_ts = now - timedelta(days=15)
# Write executions before cut-off threshold
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_SCHEDULED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_RUNNING
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_DELAYED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_CANCELING
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_REQUESTED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
self.assertEqual(len(ActionExecution.get_all()), 5)
purge_executions(logger=LOG,
timestamp=now - timedelta(days=10),
purge_incomplete=False)
self.assertEqual(len(ActionExecution.get_all()), 5)
purge_executions(logger=LOG,
timestamp=now - timedelta(days=10),
purge_incomplete=True)
self.assertEqual(len(ActionExecution.get_all()), 0)
def test_purge_executions_with_action_ref(self):
now = date_utils.get_datetime_utc_now()
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=15)
exec_model['end_timestamp'] = now - timedelta(days=14)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
purge_executions(logger=LOG, action_ref='core.localzzz', timestamp=now - timedelta(days=10))
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
purge_executions(logger=LOG, action_ref='core.local', timestamp=now - timedelta(days=10))
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 0)
def test_purge_executions_whole_model_is_not_loaded_in_memory(self, mock_ActionExecution,
mock_LiveAction):
# Verify that whole execution objects are not loaded in memory and we just retrieve the
# id field
self.assertEqual(mock_ActionExecution.query.call_count, 0)
self.assertEqual(mock_LiveAction.query.call_count, 0)
now = date_utils.get_datetime_utc_now()
purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=True)
self.assertEqual(mock_ActionExecution.query.call_count, 2)
self.assertEqual(mock_LiveAction.query.call_count, 1)
self.assertEqual(mock_ActionExecution.query.call_args_list[0][1]['only_fields'], ['id'])
self.assertTrue(mock_ActionExecution.query.call_args_list[0][1]['no_dereference'])
self.assertEqual(mock_ActionExecution.query.call_args_list[1][1]['only_fields'], ['id'])
self.assertTrue(mock_ActionExecution.query.call_args_list[1][1]['no_dereference'])
self.assertEqual(mock_LiveAction.query.call_args_list[0][1]['only_fields'], ['id'])
self.assertTrue(mock_LiveAction.query.call_args_list[0][1]['no_dereference'])
def test_purge_executions_with_action_ref(self):
now = date_utils.get_datetime_utc_now()
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=15)
exec_model['end_timestamp'] = now - timedelta(days=14)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(
exec_model['id'], count=3)
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 3)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 3)
# Invalid action reference, nothing should be deleted
purge_executions(logger=LOG,
action_ref='core.localzzz',
timestamp=now - timedelta(days=10))
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 3)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 3)
purge_executions(logger=LOG,
action_ref='core.local',
timestamp=now - timedelta(days=10))
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 0)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 0)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 0)
def test_purge_executions_with_timestamp(self):
now = date_utils.get_datetime_utc_now()
# Write one execution after cut-off threshold
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=15)
exec_model['end_timestamp'] = now - timedelta(days=14)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(
exec_model['id'], count=3)
# Write one execution before cut-off threshold
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=22)
exec_model['end_timestamp'] = now - timedelta(days=21)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(
exec_model['id'], count=3)
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 2)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 6)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 6)
purge_executions(logger=LOG, timestamp=now - timedelta(days=20))
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 3)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 3)
def test_purge_incomplete(self):
now = date_utils.get_datetime_utc_now()
start_ts = now - timedelta(days=15)
# Write executions before cut-off threshold
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_SCHEDULED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_RUNNING
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_DELAYED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_CANCELING
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_REQUESTED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
self.assertEqual(len(ActionExecution.get_all()), 5)
purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=False)
self.assertEqual(len(ActionExecution.get_all()), 5)
purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=True)
self.assertEqual(len(ActionExecution.get_all()), 0)
def test_purge_executions_with_timestamp(self):
now = date_utils.get_datetime_utc_now()
# Write one execution after cut-off threshold
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=15)
exec_model['end_timestamp'] = now - timedelta(days=14)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=3)
# Write one execution before cut-off threshold
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=22)
exec_model['end_timestamp'] = now - timedelta(days=21)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=3)
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 2)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 6)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 6)
purge_executions(logger=LOG, timestamp=now - timedelta(days=20))
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 3)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 3)
def test_purge_executions_with_timestamp(self):
now = date_utils.get_datetime_utc_now()
# Write one execution after cut-off threshold
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=15)
exec_model['end_timestamp'] = now - timedelta(days=14)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Write one execution before cut-off threshold
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=22)
exec_model['end_timestamp'] = now - timedelta(days=21)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
execs = ActionExecution.get_all()
purge_executions(logger=LOG, timestamp=now - timedelta(days=20))
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
def test_purge_executions_with_action_ref(self):
now = date_utils.get_datetime_utc_now()
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=15)
exec_model['end_timestamp'] = now - timedelta(days=14)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
purge_executions(logger=LOG,
action_ref='core.localzzz',
timestamp=now - timedelta(days=10))
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
purge_executions(logger=LOG,
action_ref='core.local',
timestamp=now - timedelta(days=10))
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 0)
def test_purge_executions_with_action_ref(self):
now = date_utils.get_datetime_utc_now()
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = now - timedelta(days=15)
exec_model['end_timestamp'] = now - timedelta(days=14)
exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=3)
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 3)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 3)
# Invalid action reference, nothing should be deleted
purge_executions(logger=LOG, action_ref='core.localzzz', timestamp=now - timedelta(days=10))
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 1)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 3)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 3)
purge_executions(logger=LOG, action_ref='core.local', timestamp=now - timedelta(days=10))
execs = ActionExecution.get_all()
self.assertEqual(len(execs), 0)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 0)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 0)
def test_purge_incomplete(self):
now = date_utils.get_datetime_utc_now()
start_ts = now - timedelta(days=15)
# Write executions before cut-off threshold
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_SCHEDULED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=1)
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_RUNNING
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=1)
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_DELAYED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=1)
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_CANCELING
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=1)
exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_REQUESTED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=1)
self.assertEqual(len(ActionExecution.get_all()), 5)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 5)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 5)
# Incompleted executions shouldnt be purged
purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=False)
self.assertEqual(len(ActionExecution.get_all()), 5)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 5)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 5)
purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=True)
self.assertEqual(len(ActionExecution.get_all()), 0)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 0)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 0)
def test_purge_incomplete(self):
now = date_utils.get_datetime_utc_now()
start_ts = now - timedelta(days=15)
# Write executions before cut-off threshold
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_SCHEDULED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(
exec_model['id'], count=1)
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_RUNNING
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(
exec_model['id'], count=1)
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_DELAYED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(
exec_model['id'], count=1)
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_CANCELING
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(
exec_model['id'], count=1)
exec_model = copy.deepcopy(
self.models['executions']['execution1.yaml'])
exec_model['start_timestamp'] = start_ts
exec_model['status'] = action_constants.LIVEACTION_STATUS_REQUESTED
exec_model['id'] = bson.ObjectId()
ActionExecution.add_or_update(exec_model)
# Insert corresponding stdout and stderr db mock models
self._insert_mock_stdout_and_stderr_objects_for_execution(
exec_model['id'], count=1)
self.assertEqual(len(ActionExecution.get_all()), 5)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 5)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 5)
# Incompleted executions shouldnt be purged
purge_executions(logger=LOG,
timestamp=now - timedelta(days=10),
purge_incomplete=False)
self.assertEqual(len(ActionExecution.get_all()), 5)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 5)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 5)
purge_executions(logger=LOG,
timestamp=now - timedelta(days=10),
purge_incomplete=True)
self.assertEqual(len(ActionExecution.get_all()), 0)
stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
self.assertEqual(len(stdout_dbs), 0)
stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
self.assertEqual(len(stderr_dbs), 0)
