def test_commands_SWUpdate_run_pre_stages_failed(
patch_logging, mocker, mock_swupdate
):
mock_manager, mocks, calls = mock_swupdate
update_lower_exc = TypeError('some-error')
target = 'some-target'
mocks['ensure_cluster_is_healthy'].side_effect = update_lower_exc
expected_exc_t = SWUpdateError
with pytest.raises(expected_exc_t) as excinfo:
commands.SWUpdate().run(target)
exc = excinfo.value
assert type(exc) is expected_exc_t
assert exc.reason is update_lower_exc
assert exc.rollback_error is None
expected_calls = [
calls['ensure_cluster_is_healthy'](),
]
assert mock_manager.mock_calls == expected_calls
mock_manager.reset_mock()
mocks['ensure_cluster_is_healthy'].side_effect = None
mocks['_ensure_update_repos_configuration'].side_effect = update_lower_exc
expected_exc_t = SWUpdateError
with pytest.raises(expected_exc_t) as excinfo:
commands.SWUpdate().run(target)
exc = excinfo.value
assert type(exc) is expected_exc_t
assert exc.reason is update_lower_exc
assert exc.rollback_error is None
expected_calls = [
calls['ensure_cluster_is_healthy'](),
calls['_ensure_update_repos_configuration'](target),
]
assert mock_manager.mock_calls == expected_calls
def test_commands_SWUpdate_run_happy_path(
patch_logging, mocker, mock_swupdate
):
mock_manager, mocks, calls = mock_swupdate
target = 'some-target'
# happy path
commands.SWUpdate().run(target)
expected_calls = [
calls['ensure_cluster_is_healthy'](),
calls['_ensure_update_repos_configuration'](target),
calls['YumRollbackManager'](
target,
multiple_targets_ok=True,
pre_rollback_cb=commands._pre_yum_rollback
),
calls['YumRollbackManager']().__enter__(),
calls['YumRollbackManager']()._resolve_last_txn_ids(),
calls['cluster_maintenance_enable'](),
calls['_update_component']("provisioner", target),
calls['config_salt_master'](),
calls['config_salt_minions'](),
] + [
calls['_update_component'](component, target)
for component in (
'motr', 's3server', 'hare', 'ha.cortx-ha', 'sspl', 'csm', 'uds'
)
] + [
calls['cluster_maintenance_disable'](),
calls['apply_ha_post_update'](target),
calls['ensure_cluster_is_healthy'](),
calls['YumRollbackManager']().__exit__(None, None, None)
]
assert mock_manager.mock_calls == expected_calls
def test_commands_SWUpdate_run_maintenance_enable_failed(
patch_logging, mocker, mock_swupdate):
mock_manager, mocks, calls = mock_swupdate
target = 'some-target'
update_lower_exc = TypeError('some-error')
mocks['cluster_maintenance_enable'].side_effect = update_lower_exc
with pytest.raises(SWUpdateError) as excinfo:
commands.SWUpdate().run(target)
exc = excinfo.value
assert type(exc) is SWUpdateError
assert type(exc.reason) is ClusterMaintenanceEnableError
assert exc.reason.reason is update_lower_exc
assert exc.rollback_error is None
expected_calls = [
calls['ensure_cluster_is_healthy'](),
calls['_ensure_update_repos_configuration'](target),
calls['_consul_export']('update-pre'),
calls['YumRollbackManager'](
target,
multiple_targets_ok=True,
pre_rollback_cb=commands._pre_yum_rollback),
calls['YumRollbackManager']().__enter__(),
calls['YumRollbackManager']()._resolve_last_txn_ids(),
calls['cluster_maintenance_enable'](),
calls['YumRollbackManager']().__exit__(
ClusterMaintenanceEnableError,
# XXX semes not valuable to check exact exc and trace as well
mocks['YumRollbackManager'].return_value.__exit__.call_args[0][1],
mocks['YumRollbackManager'].return_value.__exit__.call_args[0][2]),
calls['YumRollbackManager']()._yum_rollback(),
calls['cluster_maintenance_disable'](background=True),
]
assert mock_manager.mock_calls == expected_calls
def test_commands_SWUpdate_run_maintenance_enable_at_rollback_failed(
patch_logging, mocker, mock_swupdate
):
mock_manager, mocks, calls = mock_swupdate
update_lower_exc = TypeError('some-error')
target = 'some-target'
rollback_error = ValueError('some-rollback-error')
type(mocks['rollback_ctx']).rollback_error = mocker.PropertyMock(
return_value=rollback_error
)
mocks['ensure_cluster_is_healthy'].side_effect = [
mocker.DEFAULT, update_lower_exc, mocker.DEFAULT
]
mocks['cluster_maintenance_enable'].side_effect = [
mocker.DEFAULT, rollback_error, mocker.DEFAULT
]
expected_exc_t = SWUpdateFatalError
with pytest.raises(expected_exc_t) as excinfo:
commands.SWUpdate().run(target)
exc = excinfo.value
assert type(exc) is expected_exc_t
assert type(exc.reason) is ClusterNotHealthyError
assert exc.reason.reason is update_lower_exc
assert exc.rollback_error is rollback_error
expected_calls = [
calls['ensure_cluster_is_healthy'](),
calls['_ensure_update_repos_configuration'](target),
calls['YumRollbackManager'](
target,
multiple_targets_ok=True,
pre_rollback_cb=commands._pre_yum_rollback
),
calls['YumRollbackManager']().__enter__(),
calls['YumRollbackManager']()._resolve_last_txn_ids(),
calls['cluster_maintenance_enable'](),
calls['_update_component']("provisioner", target),
calls['config_salt_master'](),
calls['config_salt_minions'](),
] + [
calls['_update_component'](component, target)
for component in (
'motr', 's3server', 'hare', 'ha.cortx-ha', 'sspl', 'csm', 'uds'
)
] + [
calls['cluster_maintenance_disable'](),
calls['apply_ha_post_update'](target),
calls['ensure_cluster_is_healthy'](),
calls['YumRollbackManager']().__exit__(
ClusterNotHealthyError,
# XXX semes not valuable to check exact exc and trace as well
mocks['YumRollbackManager'].return_value.__exit__.call_args[0][1],
mocks['YumRollbackManager'].return_value.__exit__.call_args[0][2]
),
calls['cluster_maintenance_enable'](),
]
assert mock_manager.mock_calls == expected_calls
def test_commands_SWUpdate_run_sw_stack_update_failed(
patch_logging, mocker, mock_swupdate, rollback_error
):
mock_manager, mocks, calls = mock_swupdate
update_lower_exc = TypeError('some-error')
target = 'some-target'
# TODO IMPROVE parametrize to emulate errors on different stages:
# - during provisioner update
# - during salt-master config (first time and on rollback)
# - during salt-minions config (first time and on rollback)
# - ensure_salt_master_is_running on rollback
def apply_side_effect(component, *args, **kwargs):
if component == 'motr':
raise update_lower_exc
else:
return mocker.DEFAULT
type(mocks['rollback_ctx']).rollback_error = mocker.PropertyMock(
return_value=rollback_error
)
mocks['_update_component'].side_effect = apply_side_effect
expected_exc_t = SWUpdateFatalError if rollback_error else SWUpdateError
with pytest.raises(expected_exc_t) as excinfo:
commands.SWUpdate().run(target)
exc = excinfo.value
assert type(exc) is expected_exc_t
assert type(exc.reason) is SWStackUpdateError
assert exc.reason.reason is update_lower_exc
assert exc.rollback_error is rollback_error
expected_calls = [
calls['ensure_cluster_is_healthy'](),
calls['_ensure_update_repos_configuration'](target),
calls['_consul_export']('update-pre'),
calls['YumRollbackManager'](
target,
multiple_targets_ok=True,
pre_rollback_cb=commands._pre_yum_rollback
),
calls['YumRollbackManager']().__enter__(),
calls['YumRollbackManager']()._resolve_last_txn_ids(),
calls['cluster_maintenance_enable'](),
calls['_update_component']("provisioner", target),
calls['config_salt_master'](),
calls['config_salt_minions'](),
calls['_update_component']("motr", target),
calls['YumRollbackManager']().__exit__(
SWStackUpdateError,
# XXX semes not valuable to check exact exc and trace as well
mocks['YumRollbackManager'].return_value.__exit__.call_args[0][1],
mocks['YumRollbackManager'].return_value.__exit__.call_args[0][2]
),
calls['YumRollbackManager']()._yum_rollback(),
]
if rollback_error is None:
expected_calls.extend([
calls['ensure_salt_master_is_running'](),
calls['config_salt_master'](),
calls['config_salt_minions'](),
calls['_apply_provisioner_config'](target),
calls['cluster_maintenance_disable'](),
calls['_consul_export']('rollback-pre-ha-update'),
calls['apply_ha_post_update'](target),
calls['_consul_export']('rollback-post-ha-update'),
calls['ensure_cluster_is_healthy'](),
calls['_consul_export']('rollback-post'),
])
assert mock_manager.mock_calls == expected_calls
def test_commands_SWUpdate_run_post_rollback_fail(
patch_logging, mocker, mock_swupdate, post_rollback_stage_idx
):
mock_manager, mocks, calls = mock_swupdate
_idx = post_rollback_stage_idx
stage = post_rollback_stages[_idx]
stage_args = []
parts = stage.split(':')
if len(parts) > 1:
stage = parts[0]
stage_args = tuple(parts[1:])
update_lower_exc = TypeError('some-error')
post_rollback_exc = ValueError('some-post-rollback-error')
target = 'some-target'
# Note. just emulate sw update on early stage
# since it's not an object of testing here
def apply_side_effect(component, *args, **kwargs):
if component == 'provisioner':
raise update_lower_exc
else:
return mocker.DEFAULT
_curr_yum_rollback_side_effect = (
mocks['YumRollbackManager'].return_value.__enter__.side_effect
)
def yum_rollback_side_effect(*args, **kwargs):
def _side_effect(*args, **kwargs):
if not stage_args or args == stage_args:
raise post_rollback_exc
else:
return mocker.DEFAULT
mocks[stage].side_effect = _side_effect
return _curr_yum_rollback_side_effect(*args, **kwargs)
mocks['_update_component'].side_effect = apply_side_effect
mocks['YumRollbackManager'].return_value.__enter__.side_effect = (
yum_rollback_side_effect
)
with pytest.raises(SWUpdateFatalError) as excinfo:
commands.SWUpdate().run(target)
exc = excinfo.value
assert type(exc) is SWUpdateFatalError
assert type(exc.reason) is SWStackUpdateError
assert exc.reason.reason is update_lower_exc
assert exc.rollback_error is post_rollback_exc
expected_calls = [
calls['ensure_cluster_is_healthy'](),
calls['_ensure_update_repos_configuration'](target),
calls['_consul_export']('update-pre'),
calls['YumRollbackManager'](
target,
multiple_targets_ok=True,
pre_rollback_cb=commands._pre_yum_rollback
),
calls['YumRollbackManager']().__enter__(),
calls['YumRollbackManager']()._resolve_last_txn_ids(),
calls['cluster_maintenance_enable'](),
calls['_update_component']("provisioner", target),
calls['YumRollbackManager']().__exit__(
SWStackUpdateError,
# XXX semes not valuable to check exact exc and trace as well
mocks['YumRollbackManager'].return_value.__exit__.call_args[0][1],
mocks['YumRollbackManager'].return_value.__exit__.call_args[0][2]
),
calls['YumRollbackManager']()._yum_rollback()
]
for _stage in post_rollback_stages[:(_idx + 1)]:
stage_args = []
parts = _stage.split(':')
if len(parts) > 1:
_stage = parts[0]
stage_args = parts[1:]
elif _stage in (
'_apply_provisioner_config',
'apply_ha_post_update'
):
stage_args = [target]
expected_calls.append(calls[_stage](*stage_args))
assert mock_manager.mock_calls == expected_calls
