def test_actions_do_not_advance_state_on_fail(app, create_label, mock_webhook,
assert_history):
state_machine = app.config.state_machines['test_machine']
# First get the label into the action state by failing the automatic
# progression through the machine.
with mock_webhook(WebhookResult.FAIL):
create_label('foo', state_machine.name, {'should_progress': True})
with mock_webhook(WebhookResult.FAIL) as webhook:
process_cron(
process_action,
functools.partial(labels_in_state, app),
app,
state_machine,
state_machine.states[1],
)
webhook.assert_called_once_with(
'http://localhost/hook/test_machine/foo',
'application/json',
b'{"label": "foo", "metadata": {"should_progress": true}}',
mock.ANY,
mock.ANY,
)
assert_history([
(None, 'start'),
('start', 'perform_action'),
])
def process_job(
*,
# Bound at the cron thread level
is_terminating: IsTerminating,
# Bound at the state scheduling level
app: App,
state: State,
state_machine: StateMachine,
# Bound when scheduling a specific job for a state
fn: LabelStateProcessor,
label_provider: LabelProvider,
):
"""Process a single instance of a single cron job."""
def _iter_labels_until_terminating(
state_machine: StateMachine,
state: State,
) -> Iterable[str]:
return itertools.takewhile(
lambda _: not is_terminating(),
label_provider(app, state_machine, state),
)
try:
with app.logger.process_cron(state_machine, state, fn.__name__):
process_cron(
process=fn,
get_labels=_iter_labels_until_terminating,
app=app,
state=state,
state_machine=state_machine,
)
except Exception: # noqa: B902
return
def test_action_retry_trigger_continues_as_far_as_possible(
app, create_label, mock_webhook, assert_history):
state_machine = app.config.state_machines['test_machine']
# First get the label into the action state by failing the automatic
# progression through the machine.
with mock_webhook(WebhookResult.FAIL):
label = create_label(
'foo',
state_machine.name,
{'should_progress': True},
)
# Now attempt to process the retry of the action, succeeding.
with mock_webhook(WebhookResult.SUCCESS) as webhook:
with mock.patch('routemaster.state_machine.api.process_transitions',
) as mock_process_transitions:
process_cron(
process_action,
functools.partial(labels_in_state, app),
app,
state_machine,
state_machine.states[1],
)
mock_process_transitions.assert_called_once_with(
app,
label,
)
webhook.assert_called_once_with(
'http://localhost/hook/test_machine/foo',
'application/json',
b'{"label": "foo", "metadata": {"should_progress": true}}',
mock.ANY,
mock.ANY,
)
assert_history([
(None, 'start'),
('start', 'perform_action'),
('perform_action', 'end'),
])
def test_actions_do_not_advance_if_label_in_wrong_state(
app, create_label, mock_webhook, assert_history):
state_machine = app.config.state_machines['test_machine']
# Set should_progress=False so that the label will remain in start.
label = create_label('foo', state_machine.name, {'should_progress': False})
with mock_webhook(WebhookResult.SUCCESS) as webhook:
process_cron(
process_action,
# Return our target label for processing even though it shouldn't
# be here. Simulates changes to labels where we aren't expecting.
lambda x, y: [label.name],
app,
state_machine,
state_machine.states[1],
)
webhook.assert_not_called()
assert_history([
(None, 'start'),
])
def test_process_action_fails_retry_works(app, create_label, mock_webhook,
assert_history):
state_machine = app.config.state_machines['test_machine']
action = state_machine.states[1]
# First get the label into the action state by failing the automatic
# progression through the machine.
with mock_webhook(WebhookResult.FAIL):
create_label(
'foo',
state_machine.name,
{'should_progress': True},
)
# State machine should not have progressed
assert_history([
(None, 'start'),
('start', 'perform_action'),
])
# Now retry with succeeding webhook
with mock_webhook(WebhookResult.SUCCESS) as webhook:
process_cron(
process_action,
functools.partial(labels_in_state, app),
app,
state_machine,
action,
)
webhook.assert_called_once()
assert_history([
(None, 'start'),
('start', 'perform_action'),
('perform_action', 'end'),
])
def test_handles_label_state_change_race_condition(app, create_deleted_label):
test_machine = app.config.state_machines['test_machine']
state = test_machine.states[1]
# Create a label which is not in the expected state. Doing this and then
# returning the affected label from the `get_labels` call is easier and
# equivalent to having the state of the label change between the return of
# that call and when the label is used.
label = create_deleted_label('foo', 'test_machine')
mock_processor = mock.Mock()
mock_get_labels = mock.Mock(return_value=[label.name])
with mock.patch('routemaster.state_machine.api.suppress_exceptions', ):
state_machine.process_cron(
mock_processor,
mock_get_labels,
app,
test_machine,
state,
)
# Assert no attempt to process the label
mock_processor.assert_not_called()
def test_actions_retries_use_same_idempotency_token(app, create_label,
mock_webhook,
assert_history):
state_machine = app.config.state_machines['test_machine']
expected = {'token': None}
def persist_token(url, content_type, data, token, logger):
expected['token'] = token
return WebhookResult.FAIL
# First get the label into the action state by failing the automatic
# progression through the machine.
with mock.patch(
'routemaster.webhooks.RequestsWebhookRunner.__call__',
side_effect=persist_token,
) as webhook:
create_label('foo', state_machine.name, {'should_progress': True})
assert expected['token'] is not None
webhook.assert_called_once_with(
'http://localhost/hook/test_machine/foo',
'application/json',
b'{"label": "foo", "metadata": {"should_progress": true}}',
expected['token'],
mock.ANY,
)
with mock_webhook(WebhookResult.FAIL) as webhook:
process_cron(
process_action,
functools.partial(labels_in_state, app),
app,
state_machine,
state_machine.states[1],
)
webhook.assert_called_once_with(
'http://localhost/hook/test_machine/foo',
'application/json',
b'{"label": "foo", "metadata": {"should_progress": true}}',
expected['token'],
mock.ANY,
)
with mock_webhook(WebhookResult.SUCCESS) as webhook:
process_cron(
process_action,
functools.partial(labels_in_state, app),
app,
state_machine,
state_machine.states[1],
)
webhook.assert_called_once_with(
'http://localhost/hook/test_machine/foo',
'application/json',
b'{"label": "foo", "metadata": {"should_progress": true}}',
expected['token'],
mock.ANY,
)