def _purge_action_executions(self): """ Purge action executions and corresponding live action, stdout and stderr object which match the criteria defined in the config. """ LOG.info('Performing garbage collection for action executions and related objects') utc_now = get_datetime_utc_now() timestamp = (utc_now - datetime.timedelta(days=self._action_executions_ttl)) # Another sanity check to make sure we don't delete new executions if timestamp > (utc_now - datetime.timedelta(days=MINIMUM_TTL_DAYS)): raise ValueError('Calculated timestamp would violate the minimum TTL constraint') timestamp_str = isotime.format(dt=timestamp) LOG.info('Deleting action executions older than: %s' % (timestamp_str)) assert timestamp < utc_now try: purge_executions(logger=LOG, timestamp=timestamp) except Exception as e: LOG.exception('Failed to delete executions: %s' % (six.text_type(e))) return True
def _purge_action_executions(self): """ Purge action executions and corresponding live action, stdout and stderr object which match the criteria defined in the config. """ LOG.info( 'Performing garbage collection for action executions and related objects' ) utc_now = get_datetime_utc_now() timestamp = (utc_now - datetime.timedelta(days=self._action_executions_ttl)) # Another sanity check to make sure we don't delete new executions if timestamp > (utc_now - datetime.timedelta(days=MINIMUM_TTL_DAYS)): raise ValueError( 'Calculated timestamp would violate the minimum TTL constraint' ) timestamp_str = isotime.format(dt=timestamp) LOG.info('Deleting action executions older than: %s' % (timestamp_str)) assert timestamp < utc_now try: purge_executions(logger=LOG, timestamp=timestamp) except Exception as e: LOG.exception('Failed to delete executions: %s' % (six.text_type(e))) return True
def test_liveaction_gets_deleted(self): now = date_utils.get_datetime_utc_now() start_ts = now - timedelta(days=15) end_ts = now - timedelta(days=14) liveaction_model = copy.deepcopy( self.models['liveactions']['liveaction4.yaml']) liveaction_model['start_timestamp'] = start_ts liveaction_model['end_timestamp'] = end_ts liveaction_model[ 'status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED liveaction = LiveAction.add_or_update(liveaction_model) # Write one execution before cut-off threshold exec_model = copy.deepcopy( self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['end_timestamp'] = end_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED exec_model['id'] = bson.ObjectId() exec_model['liveaction']['id'] = str(liveaction.id) ActionExecution.add_or_update(exec_model) liveactions = LiveAction.get_all() executions = ActionExecution.get_all() self.assertEqual(len(liveactions), 1) self.assertEqual(len(executions), 1) purge_executions(logger=LOG, timestamp=now - timedelta(days=10)) liveactions = LiveAction.get_all() executions = ActionExecution.get_all() self.assertEqual(len(executions), 0) self.assertEqual(len(liveactions), 0)
def main(): _register_cli_opts() common_setup(config=config, setup_db=True, register_mq_exchanges=False) # Get config values timestamp = cfg.CONF.timestamp action_ref = cfg.CONF.action_ref purge_incomplete = cfg.CONF.purge_incomplete if not timestamp: LOG.error('Please supply a timestamp for purging models. Aborting.') return 1 else: timestamp = datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S.%fZ') timestamp = timestamp.replace(tzinfo=pytz.UTC) try: purge_executions(logger=LOG, timestamp=timestamp, action_ref=action_ref, purge_incomplete=purge_incomplete) except Exception as e: LOG.exception(str(e)) return FAILURE_EXIT_CODE finally: common_teardown() return SUCCESS_EXIT_CODE
def test_purge_executions_whole_model_is_not_loaded_in_memory( self, mock_ActionExecution, mock_LiveAction): # Verify that whole execution objects are not loaded in memory and we just retrieve the # id field self.assertEqual(mock_ActionExecution.query.call_count, 0) self.assertEqual(mock_LiveAction.query.call_count, 0) now = date_utils.get_datetime_utc_now() purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=True) self.assertEqual(mock_ActionExecution.query.call_count, 2) self.assertEqual(mock_LiveAction.query.call_count, 1) self.assertEqual( mock_ActionExecution.query.call_args_list[0][1]['only_fields'], ['id']) self.assertTrue( mock_ActionExecution.query.call_args_list[0][1]['no_dereference']) self.assertEqual( mock_ActionExecution.query.call_args_list[1][1]['only_fields'], ['id']) self.assertTrue( mock_ActionExecution.query.call_args_list[1][1]['no_dereference']) self.assertEqual( mock_LiveAction.query.call_args_list[0][1]['only_fields'], ['id']) self.assertTrue( mock_LiveAction.query.call_args_list[0][1]['no_dereference'])
def main(): _register_cli_opts() common_setup(config=config, setup_db=True, register_mq_exchanges=False) # Get config values timestamp = cfg.CONF.timestamp action_ref = cfg.CONF.action_ref purge_incomplete = cfg.CONF.purge_incomplete if not timestamp: LOG.error('Please supply a timestamp for purging models. Aborting.') return 1 else: timestamp = datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S.%fZ') timestamp = timestamp.replace(tzinfo=pytz.UTC) try: purge_executions(logger=LOG, timestamp=timestamp, action_ref=action_ref, purge_incomplete=purge_incomplete) except Exception as e: LOG.exception(str(e)) return FAILURE_EXIT_CODE finally: common_teardown() return SUCCESS_EXIT_CODE
def test_purge_executions_with_timestamp(self): now = date_utils.get_datetime_utc_now() # Write one execution after cut-off threshold exec_model = copy.deepcopy( self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = now - timedelta(days=15) exec_model['end_timestamp'] = now - timedelta(days=14) exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) # Write one execution before cut-off threshold exec_model = copy.deepcopy( self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = now - timedelta(days=22) exec_model['end_timestamp'] = now - timedelta(days=21) exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) execs = ActionExecution.get_all() purge_executions(logger=LOG, timestamp=now - timedelta(days=20)) execs = ActionExecution.get_all() self.assertEqual(len(execs), 1)
def test_liveaction_gets_deleted(self): now = date_utils.get_datetime_utc_now() start_ts = now - timedelta(days=15) end_ts = now - timedelta(days=14) liveaction_model = copy.deepcopy(self.models['liveactions']['liveaction4.yaml']) liveaction_model['start_timestamp'] = start_ts liveaction_model['end_timestamp'] = end_ts liveaction_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED liveaction = LiveAction.add_or_update(liveaction_model) # Write one execution before cut-off threshold exec_model = copy.deepcopy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['end_timestamp'] = end_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED exec_model['id'] = bson.ObjectId() exec_model['liveaction']['id'] = str(liveaction.id) ActionExecution.add_or_update(exec_model) liveactions = LiveAction.get_all() executions = ActionExecution.get_all() self.assertEqual(len(liveactions), 1) self.assertEqual(len(executions), 1) purge_executions(logger=LOG, timestamp=now - timedelta(days=10)) liveactions = LiveAction.get_all() executions = ActionExecution.get_all() self.assertEqual(len(executions), 0) self.assertEqual(len(liveactions), 0)
def _purge_action_executions(self): """ Purge action executions and corresponding live action, stdout and stderr object which match the criteria defined in the config. """ utc_now = get_datetime_utc_now() timestamp = utc_now - datetime.timedelta( days=self._action_executions_ttl) # Another sanity check to make sure we don't delete new executions if timestamp > (utc_now - datetime.timedelta(days=MINIMUM_TTL_DAYS)): raise ValueError( "Calculated timestamp would violate the minimum TTL constraint" ) timestamp_str = isotime.format(dt=timestamp) LOG.info("Deleting action executions older than: %s" % (timestamp_str)) if timestamp >= utc_now: raise ValueError(f"Calculated timestamp ({timestamp}) is" f" later than now in UTC ({utc_now}).") try: purge_executions(logger=LOG, timestamp=timestamp) except Exception as e: LOG.exception("Failed to delete executions: %s" % (six.text_type(e))) return True
def test_purge_incomplete(self): now = date_utils.get_datetime_utc_now() start_ts = now - timedelta(days=15) # Write executions before cut-off threshold exec_model = copy.deepcopy( self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_SCHEDULED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) exec_model = copy.deepcopy( self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_RUNNING exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) exec_model = copy.deepcopy( self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_DELAYED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) exec_model = copy.deepcopy( self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_CANCELING exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) exec_model = copy.deepcopy( self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_REQUESTED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) self.assertEqual(len(ActionExecution.get_all()), 5) purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=False) self.assertEqual(len(ActionExecution.get_all()), 5) purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=True) self.assertEqual(len(ActionExecution.get_all()), 0)
def test_purge_executions_with_action_ref(self): now = date_utils.get_datetime_utc_now() exec_model = copy.deepcopy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = now - timedelta(days=15) exec_model['end_timestamp'] = now - timedelta(days=14) exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) execs = ActionExecution.get_all() self.assertEqual(len(execs), 1) purge_executions(logger=LOG, action_ref='core.localzzz', timestamp=now - timedelta(days=10)) execs = ActionExecution.get_all() self.assertEqual(len(execs), 1) purge_executions(logger=LOG, action_ref='core.local', timestamp=now - timedelta(days=10)) execs = ActionExecution.get_all() self.assertEqual(len(execs), 0)
def test_purge_executions_whole_model_is_not_loaded_in_memory(self, mock_ActionExecution, mock_LiveAction): # Verify that whole execution objects are not loaded in memory and we just retrieve the # id field self.assertEqual(mock_ActionExecution.query.call_count, 0) self.assertEqual(mock_LiveAction.query.call_count, 0) now = date_utils.get_datetime_utc_now() purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=True) self.assertEqual(mock_ActionExecution.query.call_count, 2) self.assertEqual(mock_LiveAction.query.call_count, 1) self.assertEqual(mock_ActionExecution.query.call_args_list[0][1]['only_fields'], ['id']) self.assertTrue(mock_ActionExecution.query.call_args_list[0][1]['no_dereference']) self.assertEqual(mock_ActionExecution.query.call_args_list[1][1]['only_fields'], ['id']) self.assertTrue(mock_ActionExecution.query.call_args_list[1][1]['no_dereference']) self.assertEqual(mock_LiveAction.query.call_args_list[0][1]['only_fields'], ['id']) self.assertTrue(mock_LiveAction.query.call_args_list[0][1]['no_dereference'])
def test_purge_executions_with_action_ref(self): now = date_utils.get_datetime_utc_now() exec_model = copy.deepcopy( self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = now - timedelta(days=15) exec_model['end_timestamp'] = now - timedelta(days=14) exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) # Insert corresponding stdout and stderr db mock models self._insert_mock_stdout_and_stderr_objects_for_execution( exec_model['id'], count=3) execs = ActionExecution.get_all() self.assertEqual(len(execs), 1) stdout_dbs = ActionExecutionOutput.query(output_type='stdout') self.assertEqual(len(stdout_dbs), 3) stderr_dbs = ActionExecutionOutput.query(output_type='stderr') self.assertEqual(len(stderr_dbs), 3) # Invalid action reference, nothing should be deleted purge_executions(logger=LOG, action_ref='core.localzzz', timestamp=now - timedelta(days=10)) execs = ActionExecution.get_all() self.assertEqual(len(execs), 1) stdout_dbs = ActionExecutionOutput.query(output_type='stdout') self.assertEqual(len(stdout_dbs), 3) stderr_dbs = ActionExecutionOutput.query(output_type='stderr') self.assertEqual(len(stderr_dbs), 3) purge_executions(logger=LOG, action_ref='core.local', timestamp=now - timedelta(days=10)) execs = ActionExecution.get_all() self.assertEqual(len(execs), 0) stdout_dbs = ActionExecutionOutput.query(output_type='stdout') self.assertEqual(len(stdout_dbs), 0) stderr_dbs = ActionExecutionOutput.query(output_type='stderr') self.assertEqual(len(stderr_dbs), 0)
def test_purge_executions_with_timestamp(self): now = date_utils.get_datetime_utc_now() # Write one execution after cut-off threshold exec_model = copy.deepcopy( self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = now - timedelta(days=15) exec_model['end_timestamp'] = now - timedelta(days=14) exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) # Insert corresponding stdout and stderr db mock models self._insert_mock_stdout_and_stderr_objects_for_execution( exec_model['id'], count=3) # Write one execution before cut-off threshold exec_model = copy.deepcopy( self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = now - timedelta(days=22) exec_model['end_timestamp'] = now - timedelta(days=21) exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) # Insert corresponding stdout and stderr db mock models self._insert_mock_stdout_and_stderr_objects_for_execution( exec_model['id'], count=3) execs = ActionExecution.get_all() self.assertEqual(len(execs), 2) stdout_dbs = ActionExecutionOutput.query(output_type='stdout') self.assertEqual(len(stdout_dbs), 6) stderr_dbs = ActionExecutionOutput.query(output_type='stderr') self.assertEqual(len(stderr_dbs), 6) purge_executions(logger=LOG, timestamp=now - timedelta(days=20)) execs = ActionExecution.get_all() self.assertEqual(len(execs), 1) stdout_dbs = ActionExecutionOutput.query(output_type='stdout') self.assertEqual(len(stdout_dbs), 3) stderr_dbs = ActionExecutionOutput.query(output_type='stderr') self.assertEqual(len(stderr_dbs), 3)
def test_purge_incomplete(self): now = date_utils.get_datetime_utc_now() start_ts = now - timedelta(days=15) # Write executions before cut-off threshold exec_model = copy.deepcopy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_SCHEDULED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) exec_model = copy.deepcopy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_RUNNING exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) exec_model = copy.deepcopy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_DELAYED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) exec_model = copy.deepcopy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_CANCELING exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) exec_model = copy.deepcopy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_REQUESTED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) self.assertEqual(len(ActionExecution.get_all()), 5) purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=False) self.assertEqual(len(ActionExecution.get_all()), 5) purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=True) self.assertEqual(len(ActionExecution.get_all()), 0)
def test_purge_executions_with_timestamp(self): now = date_utils.get_datetime_utc_now() # Write one execution after cut-off threshold exec_model = copy.deepcopy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = now - timedelta(days=15) exec_model['end_timestamp'] = now - timedelta(days=14) exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) # Insert corresponding stdout and stderr db mock models self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=3) # Write one execution before cut-off threshold exec_model = copy.deepcopy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = now - timedelta(days=22) exec_model['end_timestamp'] = now - timedelta(days=21) exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) # Insert corresponding stdout and stderr db mock models self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=3) execs = ActionExecution.get_all() self.assertEqual(len(execs), 2) stdout_dbs = ActionExecutionOutput.query(output_type='stdout') self.assertEqual(len(stdout_dbs), 6) stderr_dbs = ActionExecutionOutput.query(output_type='stderr') self.assertEqual(len(stderr_dbs), 6) purge_executions(logger=LOG, timestamp=now - timedelta(days=20)) execs = ActionExecution.get_all() self.assertEqual(len(execs), 1) stdout_dbs = ActionExecutionOutput.query(output_type='stdout') self.assertEqual(len(stdout_dbs), 3) stderr_dbs = ActionExecutionOutput.query(output_type='stderr') self.assertEqual(len(stderr_dbs), 3)
def test_purge_executions_with_timestamp(self): now = date_utils.get_datetime_utc_now() # Write one execution after cut-off threshold exec_model = copy.deepcopy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = now - timedelta(days=15) exec_model['end_timestamp'] = now - timedelta(days=14) exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) # Write one execution before cut-off threshold exec_model = copy.deepcopy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = now - timedelta(days=22) exec_model['end_timestamp'] = now - timedelta(days=21) exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) execs = ActionExecution.get_all() purge_executions(logger=LOG, timestamp=now - timedelta(days=20)) execs = ActionExecution.get_all() self.assertEqual(len(execs), 1)
def test_purge_executions_with_action_ref(self): now = date_utils.get_datetime_utc_now() exec_model = copy.deepcopy( self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = now - timedelta(days=15) exec_model['end_timestamp'] = now - timedelta(days=14) exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) execs = ActionExecution.get_all() self.assertEqual(len(execs), 1) purge_executions(logger=LOG, action_ref='core.localzzz', timestamp=now - timedelta(days=10)) execs = ActionExecution.get_all() self.assertEqual(len(execs), 1) purge_executions(logger=LOG, action_ref='core.local', timestamp=now - timedelta(days=10)) execs = ActionExecution.get_all() self.assertEqual(len(execs), 0)
def test_purge_executions_with_action_ref(self): now = date_utils.get_datetime_utc_now() exec_model = copy.deepcopy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = now - timedelta(days=15) exec_model['end_timestamp'] = now - timedelta(days=14) exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) # Insert corresponding stdout and stderr db mock models self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=3) execs = ActionExecution.get_all() self.assertEqual(len(execs), 1) stdout_dbs = ActionExecutionOutput.query(output_type='stdout') self.assertEqual(len(stdout_dbs), 3) stderr_dbs = ActionExecutionOutput.query(output_type='stderr') self.assertEqual(len(stderr_dbs), 3) # Invalid action reference, nothing should be deleted purge_executions(logger=LOG, action_ref='core.localzzz', timestamp=now - timedelta(days=10)) execs = ActionExecution.get_all() self.assertEqual(len(execs), 1) stdout_dbs = ActionExecutionOutput.query(output_type='stdout') self.assertEqual(len(stdout_dbs), 3) stderr_dbs = ActionExecutionOutput.query(output_type='stderr') self.assertEqual(len(stderr_dbs), 3) purge_executions(logger=LOG, action_ref='core.local', timestamp=now - timedelta(days=10)) execs = ActionExecution.get_all() self.assertEqual(len(execs), 0) stdout_dbs = ActionExecutionOutput.query(output_type='stdout') self.assertEqual(len(stdout_dbs), 0) stderr_dbs = ActionExecutionOutput.query(output_type='stderr') self.assertEqual(len(stderr_dbs), 0)
def test_purge_incomplete(self): now = date_utils.get_datetime_utc_now() start_ts = now - timedelta(days=15) # Write executions before cut-off threshold exec_model = copy.deepcopy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_SCHEDULED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) # Insert corresponding stdout and stderr db mock models self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=1) exec_model = copy.deepcopy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_RUNNING exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) # Insert corresponding stdout and stderr db mock models self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=1) exec_model = copy.deepcopy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_DELAYED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) # Insert corresponding stdout and stderr db mock models self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=1) exec_model = copy.deepcopy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_CANCELING exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) # Insert corresponding stdout and stderr db mock models self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=1) exec_model = copy.deepcopy(self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_REQUESTED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) # Insert corresponding stdout and stderr db mock models self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=1) self.assertEqual(len(ActionExecution.get_all()), 5) stdout_dbs = ActionExecutionOutput.query(output_type='stdout') self.assertEqual(len(stdout_dbs), 5) stderr_dbs = ActionExecutionOutput.query(output_type='stderr') self.assertEqual(len(stderr_dbs), 5) # Incompleted executions shouldnt be purged purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=False) self.assertEqual(len(ActionExecution.get_all()), 5) stdout_dbs = ActionExecutionOutput.query(output_type='stdout') self.assertEqual(len(stdout_dbs), 5) stderr_dbs = ActionExecutionOutput.query(output_type='stderr') self.assertEqual(len(stderr_dbs), 5) purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=True) self.assertEqual(len(ActionExecution.get_all()), 0) stdout_dbs = ActionExecutionOutput.query(output_type='stdout') self.assertEqual(len(stdout_dbs), 0) stderr_dbs = ActionExecutionOutput.query(output_type='stderr') self.assertEqual(len(stderr_dbs), 0)
def test_purge_incomplete(self): now = date_utils.get_datetime_utc_now() start_ts = now - timedelta(days=15) # Write executions before cut-off threshold exec_model = copy.deepcopy( self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_SCHEDULED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) # Insert corresponding stdout and stderr db mock models self._insert_mock_stdout_and_stderr_objects_for_execution( exec_model['id'], count=1) exec_model = copy.deepcopy( self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_RUNNING exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) # Insert corresponding stdout and stderr db mock models self._insert_mock_stdout_and_stderr_objects_for_execution( exec_model['id'], count=1) exec_model = copy.deepcopy( self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_DELAYED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) # Insert corresponding stdout and stderr db mock models self._insert_mock_stdout_and_stderr_objects_for_execution( exec_model['id'], count=1) exec_model = copy.deepcopy( self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_CANCELING exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) # Insert corresponding stdout and stderr db mock models self._insert_mock_stdout_and_stderr_objects_for_execution( exec_model['id'], count=1) exec_model = copy.deepcopy( self.models['executions']['execution1.yaml']) exec_model['start_timestamp'] = start_ts exec_model['status'] = action_constants.LIVEACTION_STATUS_REQUESTED exec_model['id'] = bson.ObjectId() ActionExecution.add_or_update(exec_model) # Insert corresponding stdout and stderr db mock models self._insert_mock_stdout_and_stderr_objects_for_execution( exec_model['id'], count=1) self.assertEqual(len(ActionExecution.get_all()), 5) stdout_dbs = ActionExecutionOutput.query(output_type='stdout') self.assertEqual(len(stdout_dbs), 5) stderr_dbs = ActionExecutionOutput.query(output_type='stderr') self.assertEqual(len(stderr_dbs), 5) # Incompleted executions shouldnt be purged purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=False) self.assertEqual(len(ActionExecution.get_all()), 5) stdout_dbs = ActionExecutionOutput.query(output_type='stdout') self.assertEqual(len(stdout_dbs), 5) stderr_dbs = ActionExecutionOutput.query(output_type='stderr') self.assertEqual(len(stderr_dbs), 5) purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=True) self.assertEqual(len(ActionExecution.get_all()), 0) stdout_dbs = ActionExecutionOutput.query(output_type='stdout') self.assertEqual(len(stdout_dbs), 0) stderr_dbs = ActionExecutionOutput.query(output_type='stderr') self.assertEqual(len(stderr_dbs), 0)