def _delay(self, liveaction_db, execution_queue_item_db):
action_execution_id = str(execution_queue_item_db.action_execution_id)
liveaction_id = str(execution_queue_item_db.liveaction_id)
queue_item_id = str(execution_queue_item_db.id)
extra = {'queue_item_id': queue_item_id}
LOG.info(
'[%s] Liveaction "%s" is delayed and scheduling queue is updated.',
action_execution_id, liveaction_id, extra=extra
)
liveaction_db = action_service.update_status(
liveaction_db, action_constants.LIVEACTION_STATUS_DELAYED, publish=False
)
execution_queue_item_db.scheduled_start_timestamp = date.append_milliseconds_to_time(
date.get_datetime_utc_now(),
POLICY_DELAYED_EXECUTION_RESCHEDULE_TIME_MS
)
try:
execution_queue_item_db.handling = False
ActionExecutionSchedulingQueue.add_or_update(execution_queue_item_db, publish=False)
except db_exc.StackStormDBObjectWriteConflictError:
LOG.warning(
'[%s] Database write conflict on updating scheduling queue.',
action_execution_id, extra=extra
)
def _handle_garbage_collection(self):
"""
Periodically look for executions which have "handling" set to "True" and haven't been
updated for a while (this likely indicates that an execution as picked up by a scheduler
process which died before finishing the processing or similar) and reset handling to
False so other scheduler can pick it up.
"""
query = {
'scheduled_start_timestamp__lte':
date.append_milliseconds_to_time(
date.get_datetime_utc_now(),
-EXECUTION_SCHEDUELING_TIMEOUT_THRESHOLD_MS),
'handling':
True
}
execution_queue_item_dbs = ActionExecutionSchedulingQueue.query(
**query) or []
for execution_queue_item_db in execution_queue_item_dbs:
execution_queue_item_db.handling = False
try:
ActionExecutionSchedulingQueue.add_or_update(
execution_queue_item_db, publish=False)
LOG.info('Removing lock for orphaned execution queue item: %s',
execution_queue_item_db.id)
except db_exc.StackStormDBObjectWriteConflictError:
LOG.info(
'Execution queue item updated before rescheduling: %s',
execution_queue_item_db.id)
def _reset_handling_flag(self):
"""
Periodically look for executions which have "handling" set to "True" and haven't been
updated for a while (this likely indicates that an execution as picked up by a scheduler
process which died before finishing the processing or similar) and reset handling to
False so other scheduler can pick it up.
"""
query = {
'scheduled_start_timestamp__lte': date.append_milliseconds_to_time(
date.get_datetime_utc_now(),
-self._execution_scheduling_timeout_threshold_min
),
'handling': True
}
execution_queue_item_dbs = ActionExecutionSchedulingQueue.query(**query) or []
for execution_queue_item_db in execution_queue_item_dbs:
execution_queue_item_db.handling = False
try:
ActionExecutionSchedulingQueue.add_or_update(execution_queue_item_db, publish=False)
LOG.info(
'[%s] Removing lock for orphaned execution queue item "%s".',
execution_queue_item_db.action_execution_id,
str(execution_queue_item_db.id)
)
except db_exc.StackStormDBObjectWriteConflictError:
LOG.info(
'[%s] Execution queue item "%s" updated during garbage collection.',
execution_queue_item_db.action_execution_id,
str(execution_queue_item_db.id)
)
def _get_next_execution(self):
"""
Sort execution requests by FIFO and priority and get the latest, highest priority item from
the queue and pop it off.
"""
query = {
'scheduled_start_timestamp__lte': date.get_datetime_utc_now(),
'handling': False,
'limit': 1,
'order_by': [
'+scheduled_start_timestamp',
]
}
execution_queue_item_db = ActionExecutionSchedulingQueue.query(
**query).first()
if not execution_queue_item_db:
return None
# Mark that this scheduler process is currently handling (processing) that request
# NOTE: This operation is atomic (CAS)
execution_queue_item_db.handling = True
try:
ActionExecutionSchedulingQueue.add_or_update(
execution_queue_item_db, publish=False)
return execution_queue_item_db
except db_exc.StackStormDBObjectWriteConflictError:
LOG.info('Execution queue item handled by another scheduler: %s',
execution_queue_item_db.id)
return None
def _get_next_execution(self):
"""
Sort execution requests by FIFO and priority and get the latest, highest priority item from
the queue and pop it off.
"""
query = {
'scheduled_start_timestamp__lte': date.get_datetime_utc_now(),
'handling': False,
'limit': 1,
'order_by': [
'+scheduled_start_timestamp',
]
}
execution_queue_item_db = ActionExecutionSchedulingQueue.query(**query).first()
if not execution_queue_item_db:
return None
# Mark that this scheduler process is currently handling (processing) that request
# NOTE: This operation is atomic (CAS)
execution_queue_item_db.handling = True
try:
ActionExecutionSchedulingQueue.add_or_update(execution_queue_item_db, publish=False)
return execution_queue_item_db
except db_exc.StackStormDBObjectWriteConflictError:
LOG.info('Execution queue item handled by another scheduler: %s',
execution_queue_item_db.id)
return None
def _handle_garbage_collection(self):
"""
Periodically look for executions which have "handling" set to "True" and haven't been
updated for a while (this likely indicates that an execution as picked up by a scheduler
process which died before finishing the processing or similar) and reset handling to
False so other scheduler can pick it up.
"""
query = {
'scheduled_start_timestamp__lte': date.append_milliseconds_to_time(
date.get_datetime_utc_now(),
-EXECUTION_SCHEDUELING_TIMEOUT_THRESHOLD_MS
),
'handling': True
}
execution_queue_item_dbs = ActionExecutionSchedulingQueue.query(**query) or []
for execution_queue_item_db in execution_queue_item_dbs:
execution_queue_item_db.handling = False
try:
ActionExecutionSchedulingQueue.add_or_update(execution_queue_item_db, publish=False)
LOG.info('Removing lock for orphaned execution queue item: %s',
execution_queue_item_db.id)
except db_exc.StackStormDBObjectWriteConflictError:
LOG.info(
'Execution queue item updated before rescheduling: %s',
execution_queue_item_db.id
)
def _regulate_and_schedule(self, liveaction_db, execution_queue_item_db):
action_execution_id = str(execution_queue_item_db.action_execution_id)
liveaction_id = str(execution_queue_item_db.liveaction_id)
queue_item_id = str(execution_queue_item_db.id)
extra = {'queue_item_id': queue_item_id}
LOG.info(
'[%s] Liveaction "%s" has status "%s" before applying policies.',
action_execution_id,
liveaction_id,
liveaction_db.status,
extra=extra)
# Apply policies defined for the action.
liveaction_db = policy_service.apply_pre_run_policies(liveaction_db)
LOG.info(
'[%s] Liveaction "%s" has status "%s" after applying policies.',
action_execution_id,
liveaction_id,
liveaction_db.status,
extra=extra)
if liveaction_db.status == action_constants.LIVEACTION_STATUS_DELAYED:
LOG.info(
'[%s] Liveaction "%s" is delayed and scheduling queue is updated.',
action_execution_id,
liveaction_id,
extra=extra)
liveaction_db = action_service.update_status(
liveaction_db,
action_constants.LIVEACTION_STATUS_DELAYED,
publish=False)
execution_queue_item_db.handling = False
execution_queue_item_db.scheduled_start_timestamp = date.append_milliseconds_to_time(
date.get_datetime_utc_now(),
POLICY_DELAYED_EXECUTION_RESCHEDULE_TIME_MS)
try:
ActionExecutionSchedulingQueue.add_or_update(
execution_queue_item_db, publish=False)
except db_exc.StackStormDBObjectWriteConflictError:
LOG.warning(
'[%s] Database write conflict on updating scheduling queue.',
action_execution_id,
extra=extra)
return
if (liveaction_db.status
in action_constants.LIVEACTION_COMPLETED_STATES
or liveaction_db.status
in action_constants.LIVEACTION_CANCEL_STATES):
ActionExecutionSchedulingQueue.delete(execution_queue_item_db)
return
self._schedule(liveaction_db, execution_queue_item_db)
def _fix_missing_action_execution_id(self):
"""
Auto-populate the action_execution_id in ActionExecutionSchedulingQueue if empty.
"""
for entry in ActionExecutionSchedulingQueue.query(action_execution_id__in=['', None]):
execution_db = ActionExecution.get(liveaction__id=entry.liveaction_id)
if not execution_db:
continue
msg = '[%s] Populating action_execution_id for item "%s".'
LOG.info(msg, str(execution_db.id), str(entry.id))
entry.action_execution_id = str(execution_db.id)
ActionExecutionSchedulingQueue.add_or_update(entry, publish=False)
def _regulate_and_schedule(self, liveaction_db, execution_queue_item_db):
# Apply policies defined for the action.
liveaction_db = policy_service.apply_pre_run_policies(liveaction_db)
liveaction_id = str(liveaction_db.id)
queue_item_id = str(execution_queue_item_db.id)
extra = {
'liveaction_id': liveaction_id,
'liveaction_status': liveaction_db.status,
'queue_item_id': queue_item_id
}
LOG.info('Liveaction (%s) Status Pre-Run: %s (%s)',
liveaction_id,
liveaction_db.status,
queue_item_id,
extra=extra)
if liveaction_db.status is action_constants.LIVEACTION_STATUS_POLICY_DELAYED:
liveaction_db = action_service.update_status(
liveaction_db,
action_constants.LIVEACTION_STATUS_DELAYED,
publish=False)
execution_queue_item_db.handling = False
execution_queue_item_db.scheduled_start_timestamp = date.append_milliseconds_to_time(
date.get_datetime_utc_now(),
POLICY_DELAYED_EXECUTION_RESCHEDULE_TIME_MS)
try:
ActionExecutionSchedulingQueue.add_or_update(
execution_queue_item_db, publish=False)
except db_exc.StackStormDBObjectWriteConflictError:
LOG.warning(
'Execution queue item update conflict during scheduling: %s',
execution_queue_item_db.id)
return
if (liveaction_db.status
in action_constants.LIVEACTION_COMPLETED_STATES
or liveaction_db.status
in action_constants.LIVEACTION_CANCEL_STATES):
ActionExecutionSchedulingQueue.delete(execution_queue_item_db)
return
self._schedule(liveaction_db, execution_queue_item_db)
def test_failed_next_item(self, mocked_logger):
self.reset()
liveaction_db = self._create_liveaction_db()
schedule_q_db = self.scheduler._create_execution_queue_item_db_from_liveaction(
liveaction_db,
)
schedule_q_db = ActionExecutionSchedulingQueue.add_or_update(schedule_q_db)
with mock.patch(
'st2actions.scheduler.handler.ActionExecutionSchedulingQueue.add_or_update',
side_effect=db_exc.StackStormDBObjectWriteConflictError(schedule_q_db)
):
schedule_q_db = self.scheduling_queue._get_next_execution()
self.assertIsNone(schedule_q_db)
self.assertEqual(mocked_logger.info.call_count, 2)
call_args = mocked_logger.info.call_args_list[1][0]
self.assertEqual(r'[%s] Item "%s" is already handled by another scheduler.', call_args[0])
schedule_q_db = self.scheduling_queue._get_next_execution()
self.assertIsNotNone(schedule_q_db)
ActionExecutionSchedulingQueue.delete(schedule_q_db)
def test_populate_action_execution_id(self):
self.reset()
liveaction_db = self._create_liveaction_db()
schedule_q_db = self.scheduler._create_execution_queue_item_db_from_liveaction(
liveaction_db)
# Manually unset the action_execution_id ot mock DB model of previous version.
schedule_q_db.action_execution_id = None
schedule_q_db = ActionExecutionSchedulingQueue.add_or_update(
schedule_q_db)
schedule_q_db = ActionExecutionSchedulingQueue.get_by_id(
str(schedule_q_db.id))
self.assertIsNone(schedule_q_db.action_execution_id)
# Run the clean up logic.
self.scheduling_queue._fix_missing_action_execution_id()
# Check that the action_execution_id is populated.
schedule_q_db = ActionExecutionSchedulingQueue.get_by_id(
str(schedule_q_db.id))
execution_db = execution_service.update_execution(liveaction_db)
self.assertEqual(schedule_q_db.action_execution_id,
str(execution_db.id))
def test_failed_next_item(self, mocked_logger):
self.reset()
liveaction_db = self._create_liveaction_db()
schedule_q_db = self.scheduler._create_execution_queue_item_db_from_liveaction(
liveaction_db, )
schedule_q_db = ActionExecutionSchedulingQueue.add_or_update(
schedule_q_db)
with mock.patch(
"st2actions.scheduler.handler.ActionExecutionSchedulingQueue.add_or_update",
side_effect=db_exc.StackStormDBObjectWriteConflictError(
schedule_q_db),
):
schedule_q_db = self.scheduling_queue._get_next_execution()
self.assertIsNone(schedule_q_db)
self.assertEqual(mocked_logger.info.call_count, 2)
call_args = mocked_logger.info.call_args_list[1][0]
self.assertEqual(
r'[%s] Item "%s" is already handled by another scheduler.',
call_args[0])
schedule_q_db = self.scheduling_queue._get_next_execution()
self.assertIsNotNone(schedule_q_db)
ActionExecutionSchedulingQueue.delete(schedule_q_db)
def test_failed_next_item(self, mocked_logger):
self.reset()
liveaction_db = self._create_liveaction_db()
schedule_q_db = self.scheduler._create_execution_queue_item_db_from_liveaction(
liveaction_db, )
schedule_q_db = ActionExecutionSchedulingQueue.add_or_update(
schedule_q_db)
with mock.patch(
'st2actions.scheduler.handler.ActionExecutionSchedulingQueue.add_or_update',
side_effect=db_exc.StackStormDBObjectWriteConflictError(
schedule_q_db)):
schedule_q_db = self.scheduling_queue._get_next_execution()
self.assertIsNone(schedule_q_db)
mocked_logger.info.assert_called_once()
call_args = mocked_logger.info.call_args_list[0][0]
self.assertEqual(
r'Execution queue item handled by another scheduler: %s',
call_args[0])
schedule_q_db = self.scheduling_queue._get_next_execution()
self.assertIsNotNone(schedule_q_db)
ActionExecutionSchedulingQueue.delete(schedule_q_db)
def process(self, request):
"""
Adds execution into execution_scheduling database for scheduling
:param request: Action execution request.
:type request: ``st2common.models.db.liveaction.LiveActionDB``
"""
if request.status != action_constants.LIVEACTION_STATUS_REQUESTED:
LOG.info(
'%s is ignoring %s (id=%s) with "%s" status.',
self.__class__.__name__,
type(request),
request.id,
request.status,
)
return
try:
liveaction_db = action_utils.get_liveaction_by_id(str(request.id))
except StackStormDBObjectNotFoundError:
LOG.exception("Failed to find liveaction %s in the database.",
str(request.id))
raise
query = {
"liveaction_id": str(liveaction_db.id),
}
queued_requests = ActionExecutionSchedulingQueue.query(**query)
if len(queued_requests) > 0:
# Particular execution is already being scheduled
return queued_requests[0]
if liveaction_db.delay and liveaction_db.delay > 0:
liveaction_db = action_service.update_status(
liveaction_db,
action_constants.LIVEACTION_STATUS_DELAYED,
publish=False)
execution_queue_item_db = self._create_execution_queue_item_db_from_liveaction(
liveaction_db, delay=liveaction_db.delay)
ActionExecutionSchedulingQueue.add_or_update(execution_queue_item_db,
publish=False)
return execution_queue_item_db
def test_next_execution(self):
self.reset()
schedule_q_dbs = []
delays = [2000, 5000, 4000]
expected_order = [0, 2, 1]
test_cases = []
for delay in delays:
liveaction_db = self._create_liveaction_db()
delayed_start = date.append_milliseconds_to_time(
liveaction_db.start_timestamp, delay)
test_case = {
'liveaction': liveaction_db,
'delay': delay,
'delayed_start': delayed_start
}
test_cases.append(test_case)
for test_case in test_cases:
schedule_q_dbs.append(
ActionExecutionSchedulingQueue.add_or_update(
self.scheduler.
_create_execution_queue_item_db_from_liveaction(
test_case['liveaction'],
test_case['delay'],
)))
# Wait maximum delay seconds so the query works as expected
eventlet.sleep(3.2)
for index in expected_order:
test_case = test_cases[index]
date_mock = mock.MagicMock()
date_mock.get_datetime_utc_now.return_value = test_case[
'delayed_start']
date_mock.append_milliseconds_to_time = date.append_milliseconds_to_time
with mock.patch('st2actions.scheduler.handler.date', date_mock):
schedule_q_db = self.scheduling_queue._get_next_execution()
ActionExecutionSchedulingQueue.delete(schedule_q_db)
self.assertIsInstance(schedule_q_db,
ActionExecutionSchedulingQueueItemDB)
self.assertEqual(schedule_q_db.delay, test_case['delay'])
self.assertEqual(schedule_q_db.liveaction_id,
str(test_case['liveaction'].id))
# NOTE: We can't directly assert on the timestamp due to the delays on the code and
# timing variance
scheduled_start_timestamp = schedule_q_db.scheduled_start_timestamp
test_case_start_timestamp = test_case['delayed_start']
start_timestamp_diff = (scheduled_start_timestamp -
test_case_start_timestamp)
self.assertTrue(
start_timestamp_diff <= datetime.timedelta(seconds=1))
def _delay(self, liveaction_db, execution_queue_item_db):
liveaction_db = action_service.update_status(
liveaction_db, action_constants.LIVEACTION_STATUS_DELAYED, publish=False
)
execution_queue_item_db.scheduled_start_timestamp = date.append_milliseconds_to_time(
date.get_datetime_utc_now(),
POLICY_DELAYED_EXECUTION_RESCHEDULE_TIME_MS
)
try:
execution_queue_item_db.handling = False
ActionExecutionSchedulingQueue.add_or_update(execution_queue_item_db, publish=False)
except db_exc.StackStormDBObjectWriteConflictError:
LOG.warning(
'Execution queue item update conflict during scheduling: %s',
execution_queue_item_db.id
)
def _delay(self, liveaction_db, execution_queue_item_db):
liveaction_db = action_service.update_status(
liveaction_db,
action_constants.LIVEACTION_STATUS_DELAYED,
publish=False)
execution_queue_item_db.scheduled_start_timestamp = date.append_milliseconds_to_time(
date.get_datetime_utc_now(),
POLICY_DELAYED_EXECUTION_RESCHEDULE_TIME_MS)
try:
execution_queue_item_db.handling = False
ActionExecutionSchedulingQueue.add_or_update(
execution_queue_item_db, publish=False)
except db_exc.StackStormDBObjectWriteConflictError:
LOG.warning(
'Execution queue item update conflict during scheduling: %s',
execution_queue_item_db.id)
def _get_next_execution(self):
"""
Sort execution requests by FIFO and priority and get the latest, highest priority item from
the queue and pop it off.
NOTE: FIFO order is not guaranteed anymore for executions which are re-scheduled and delayed
due to a policy.
"""
query = {
"scheduled_start_timestamp__lte":
date.get_datetime_utc_now(),
"handling":
False,
"limit":
1,
"order_by":
["+scheduled_start_timestamp", "+original_start_timestamp"],
}
execution_queue_item_db = ActionExecutionSchedulingQueue.query(
**query).first()
if not execution_queue_item_db:
return None
# Mark that this scheduler process is currently handling (processing) that request
# NOTE: This operation is atomic (CAS)
msg = '[%s] Retrieved item "%s" from scheduling queue.'
LOG.info(msg, execution_queue_item_db.action_execution_id,
execution_queue_item_db.id)
execution_queue_item_db.handling = True
try:
ActionExecutionSchedulingQueue.add_or_update(
execution_queue_item_db, publish=False)
return execution_queue_item_db
except db_exc.StackStormDBObjectWriteConflictError:
LOG.info(
'[%s] Item "%s" is already handled by another scheduler.',
execution_queue_item_db.action_execution_id,
str(execution_queue_item_db.id),
)
return None
def _cleanup_policy_delayed(self):
"""
Clean up any action execution in the deprecated policy-delayed status. Associated
entries in the scheduling queue will be removed and the action execution will be
moved back into requested status.
"""
policy_delayed_liveaction_dbs = LiveAction.query(status="policy-delayed") or []
for liveaction_db in policy_delayed_liveaction_dbs:
ex_que_qry = {"liveaction_id": str(liveaction_db.id), "handling": False}
execution_queue_item_dbs = (
ActionExecutionSchedulingQueue.query(**ex_que_qry) or []
)
for execution_queue_item_db in execution_queue_item_dbs:
# Mark the entry in the scheduling queue for handling.
try:
execution_queue_item_db.handling = True
execution_queue_item_db = (
ActionExecutionSchedulingQueue.add_or_update(
execution_queue_item_db, publish=False
)
)
except db_exc.StackStormDBObjectWriteConflictError:
msg = (
'[%s] Item "%s" is currently being processed by another scheduler.'
% (
execution_queue_item_db.action_execution_id,
str(execution_queue_item_db.id),
)
)
LOG.error(msg)
raise Exception(msg)
# Delete the entry from the scheduling queue.
LOG.info(
'[%s] Removing policy-delayed entry "%s" from the scheduling queue.',
execution_queue_item_db.action_execution_id,
str(execution_queue_item_db.id),
)
ActionExecutionSchedulingQueue.delete(execution_queue_item_db)
# Update the status of the liveaction and execution to requested.
LOG.info(
'[%s] Removing policy-delayed entry "%s" from the scheduling queue.',
execution_queue_item_db.action_execution_id,
str(execution_queue_item_db.id),
)
liveaction_db = action_service.update_status(
liveaction_db, action_constants.LIVEACTION_STATUS_REQUESTED
)
execution_service.update_execution(liveaction_db)
def _regulate_and_schedule(self, liveaction_db, execution_queue_item_db):
# Apply policies defined for the action.
liveaction_db = policy_service.apply_pre_run_policies(liveaction_db)
liveaction_id = str(liveaction_db.id)
queue_item_id = str(execution_queue_item_db.id)
extra = {
'liveaction_id': liveaction_id,
'liveaction_status': liveaction_db.status,
'queue_item_id': queue_item_id
}
LOG.info('Liveaction (%s) Status Pre-Run: %s (%s)', liveaction_id, liveaction_db.status,
queue_item_id, extra=extra)
if liveaction_db.status is action_constants.LIVEACTION_STATUS_POLICY_DELAYED:
liveaction_db = action_service.update_status(
liveaction_db, action_constants.LIVEACTION_STATUS_DELAYED, publish=False
)
execution_queue_item_db.handling = False
execution_queue_item_db.scheduled_start_timestamp = date.append_milliseconds_to_time(
date.get_datetime_utc_now(),
POLICY_DELAYED_EXECUTION_RESCHEDULE_TIME_MS
)
try:
ActionExecutionSchedulingQueue.add_or_update(execution_queue_item_db, publish=False)
except db_exc.StackStormDBObjectWriteConflictError:
LOG.warning(
'Execution queue item update conflict during scheduling: %s',
execution_queue_item_db.id
)
return
if (liveaction_db.status in action_constants.LIVEACTION_COMPLETED_STATES or
liveaction_db.status in action_constants.LIVEACTION_CANCEL_STATES):
ActionExecutionSchedulingQueue.delete(execution_queue_item_db)
return
self._schedule(liveaction_db, execution_queue_item_db)
def process(self, request):
"""
Adds execution into execution_scheduling database for scheduling
:param request: Action execution request.
:type request: ``st2common.models.db.liveaction.LiveActionDB``
"""
if request.status != action_constants.LIVEACTION_STATUS_REQUESTED:
LOG.info('%s is ignoring %s (id=%s) with "%s" status.',
self.__class__.__name__, type(request), request.id, request.status)
return
try:
liveaction_db = action_utils.get_liveaction_by_id(str(request.id))
except StackStormDBObjectNotFoundError:
LOG.exception('Failed to find liveaction %s in the database.', str(request.id))
raise
query = {
'liveaction_id': str(liveaction_db.id),
}
queued_requests = ActionExecutionSchedulingQueue.query(**query)
if len(queued_requests) > 0:
# Particular execution is already being scheduled
return queued_requests[0]
if liveaction_db.delay and liveaction_db.delay > 0:
liveaction_db = action_service.update_status(
liveaction_db,
action_constants.LIVEACTION_STATUS_DELAYED,
publish=False
)
execution_queue_item_db = self._create_execution_queue_item_db_from_liveaction(
liveaction_db,
delay=liveaction_db.delay
)
ActionExecutionSchedulingQueue.add_or_update(execution_queue_item_db, publish=False)
return execution_queue_item_db
def test_next_execution(self):
self.reset()
schedule_q_dbs = []
delays = [2000, 5000, 4000]
expected_order = [0, 2, 1]
test_cases = []
for delay in delays:
liveaction_db = self._create_liveaction_db()
delayed_start = date.append_milliseconds_to_time(liveaction_db.start_timestamp, delay)
test_case = {
'liveaction': liveaction_db,
'delay': delay,
'delayed_start': delayed_start
}
test_cases.append(test_case)
for test_case in test_cases:
schedule_q_dbs.append(
ActionExecutionSchedulingQueue.add_or_update(
self.scheduler._create_execution_queue_item_db_from_liveaction(
test_case['liveaction'],
test_case['delay'],
)
)
)
# Wait maximum delay seconds so the query works as expected
eventlet.sleep(3.2)
for index in expected_order:
test_case = test_cases[index]
date_mock = mock.MagicMock()
date_mock.get_datetime_utc_now.return_value = test_case['delayed_start']
date_mock.append_milliseconds_to_time = date.append_milliseconds_to_time
with mock.patch('st2actions.scheduler.handler.date', date_mock):
schedule_q_db = self.scheduling_queue._get_next_execution()
ActionExecutionSchedulingQueue.delete(schedule_q_db)
self.assertIsInstance(schedule_q_db, ActionExecutionSchedulingQueueItemDB)
self.assertEqual(schedule_q_db.delay, test_case['delay'])
self.assertEqual(schedule_q_db.liveaction_id, str(test_case['liveaction'].id))
# NOTE: We can't directly assert on the timestamp due to the delays on the code and
# timing variance
scheduled_start_timestamp = schedule_q_db.scheduled_start_timestamp
test_case_start_timestamp = test_case['delayed_start']
start_timestamp_diff = (scheduled_start_timestamp - test_case_start_timestamp)
self.assertTrue(start_timestamp_diff <= datetime.timedelta(seconds=1))
def test_next_execution(self):
self.reset()
schedule_q_dbs = []
delays = [100, 5000, 1000]
expected_order = [0, 2, 1]
test_cases = []
for delay in delays:
liveaction_db = self._create_liveaction_db()
delayed_start = date.append_milliseconds_to_time(
liveaction_db.start_timestamp, delay)
test_case = {
'liveaction': liveaction_db,
'delay': delay,
'delayed_start': delayed_start
}
test_cases.append(test_case)
for test_case in test_cases:
schedule_q_dbs.append(
ActionExecutionSchedulingQueue.add_or_update(
self.scheduler.
_create_execution_queue_item_db_from_liveaction(
test_case['liveaction'],
test_case['delay'],
)))
for index in expected_order:
test_case = test_cases[index]
date_mock = mock.MagicMock()
date_mock.get_datetime_utc_now.return_value = test_case[
'delayed_start']
date_mock.append_milliseconds_to_time = date.append_milliseconds_to_time
with mock.patch('st2actions.scheduler.handler.date', date_mock):
schedule_q_db = self.scheduling_queue._get_next_execution()
ActionExecutionSchedulingQueue.delete(schedule_q_db)
self.assertIsInstance(schedule_q_db,
ActionExecutionSchedulingQueueItemDB)
self.assertEqual(schedule_q_db.scheduled_start_timestamp,
test_case['delayed_start'])
self.assertEqual(schedule_q_db.delay, test_case['delay'])
self.assertEqual(schedule_q_db.liveaction_id,
str(test_case['liveaction'].id))
def test_cleanup_policy_delayed(self):
self.reset()
liveaction_db = self._create_liveaction_db()
schedule_q_db = self.scheduler._create_execution_queue_item_db_from_liveaction(
liveaction_db
)
schedule_q_db = ActionExecutionSchedulingQueue.add_or_update(schedule_q_db)
# Manually update the liveaction to policy-delayed status.
# Using action_service.update_status will throw an exception on the
# deprecated action_constants.LIVEACTION_STATUS_POLICY_DELAYED.
liveaction_db.status = "policy-delayed"
liveaction_db = LiveAction.add_or_update(liveaction_db)
execution_db = execution_service.update_execution(liveaction_db)
# Check that the execution status is set to policy-delayed.
liveaction_db = LiveAction.get_by_id(str(liveaction_db.id))
self.assertEqual(liveaction_db.status, "policy-delayed")
execution_db = ActionExecution.get_by_id(str(execution_db.id))
self.assertEqual(execution_db.status, "policy-delayed")
# Run the clean up logic.
self.scheduling_queue._cleanup_policy_delayed()
# Check that the execution status is reset to requested.
liveaction_db = LiveAction.get_by_id(str(liveaction_db.id))
self.assertEqual(
liveaction_db.status, action_constants.LIVEACTION_STATUS_REQUESTED
)
execution_db = ActionExecution.get_by_id(str(execution_db.id))
self.assertEqual(
execution_db.status, action_constants.LIVEACTION_STATUS_REQUESTED
)
# The old entry should have been deleted. Since the execution is
# reset to requested, there should be a new scheduling entry.
new_schedule_q_db = self.scheduling_queue._get_next_execution()
self.assertIsNotNone(new_schedule_q_db)
self.assertNotEqual(str(schedule_q_db.id), str(new_schedule_q_db.id))
self.assertEqual(
schedule_q_db.action_execution_id, new_schedule_q_db.action_execution_id
)
self.assertEqual(schedule_q_db.liveaction_id, new_schedule_q_db.liveaction_id)
def test_cleanup_policy_delayed(self):
self.reset()
liveaction_db = self._create_liveaction_db()
schedule_q_db = self.scheduler._create_execution_queue_item_db_from_liveaction(
liveaction_db
)
schedule_q_db = ActionExecutionSchedulingQueue.add_or_update(schedule_q_db)
# Manually update the liveaction to policy-delayed status.
# Using action_service.update_status will throw an exception on the
# deprecated action_constants.LIVEACTION_STATUS_POLICY_DELAYED.
liveaction_db.status = 'policy-delayed'
liveaction_db = LiveAction.add_or_update(liveaction_db)
execution_db = execution_service.update_execution(liveaction_db)
# Check that the execution status is set to policy-delayed.
liveaction_db = LiveAction.get_by_id(str(liveaction_db.id))
self.assertEqual(liveaction_db.status, 'policy-delayed')
execution_db = ActionExecution.get_by_id(str(execution_db.id))
self.assertEqual(execution_db.status, 'policy-delayed')
# Run the clean up logic.
self.scheduling_queue._cleanup_policy_delayed()
# Check that the execution status is reset to requested.
liveaction_db = LiveAction.get_by_id(str(liveaction_db.id))
self.assertEqual(liveaction_db.status, action_constants.LIVEACTION_STATUS_REQUESTED)
execution_db = ActionExecution.get_by_id(str(execution_db.id))
self.assertEqual(execution_db.status, action_constants.LIVEACTION_STATUS_REQUESTED)
# The old entry should have been deleted. Since the execution is
# reset to requested, there should be a new scheduling entry.
new_schedule_q_db = self.scheduling_queue._get_next_execution()
self.assertIsNotNone(new_schedule_q_db)
self.assertNotEqual(str(schedule_q_db.id), str(new_schedule_q_db.id))
self.assertEqual(schedule_q_db.action_execution_id, new_schedule_q_db.action_execution_id)
self.assertEqual(schedule_q_db.liveaction_id, new_schedule_q_db.liveaction_id)
def test_garbage_collection(self):
self.reset()
liveaction_db = self._create_liveaction_db()
schedule_q_db = self.scheduler._create_execution_queue_item_db_from_liveaction(
liveaction_db,
-70000,
)
schedule_q_db.handling = True
schedule_q_db = ActionExecutionSchedulingQueue.add_or_update(schedule_q_db)
schedule_q_db = self.scheduling_queue._get_next_execution()
self.assertIsNone(schedule_q_db)
self.scheduling_queue._handle_garbage_collection()
schedule_q_db = self.scheduling_queue._get_next_execution()
self.assertIsNotNone(schedule_q_db)
def test_garbage_collection(self):
self.reset()
liveaction_db = self._create_liveaction_db()
schedule_q_db = self.scheduler._create_execution_queue_item_db_from_liveaction(
liveaction_db,
-70000,
)
schedule_q_db.handling = True
schedule_q_db = ActionExecutionSchedulingQueue.add_or_update(schedule_q_db)
schedule_q_db = self.scheduling_queue._get_next_execution()
self.assertIsNone(schedule_q_db)
self.scheduling_queue._handle_garbage_collection()
schedule_q_db = self.scheduling_queue._get_next_execution()
self.assertIsNotNone(schedule_q_db)
def test_populate_action_execution_id(self):
self.reset()
liveaction_db = self._create_liveaction_db()
schedule_q_db = self.scheduler._create_execution_queue_item_db_from_liveaction(
liveaction_db
)
# Manually unset the action_execution_id ot mock DB model of previous version.
schedule_q_db.action_execution_id = None
schedule_q_db = ActionExecutionSchedulingQueue.add_or_update(schedule_q_db)
schedule_q_db = ActionExecutionSchedulingQueue.get_by_id(str(schedule_q_db.id))
self.assertIsNone(schedule_q_db.action_execution_id)
# Run the clean up logic.
self.scheduling_queue._fix_missing_action_execution_id()
# Check that the action_execution_id is populated.
schedule_q_db = ActionExecutionSchedulingQueue.get_by_id(str(schedule_q_db.id))
execution_db = execution_service.update_execution(liveaction_db)
self.assertEqual(schedule_q_db.action_execution_id, str(execution_db.id))