def test_commands_SWUpdate_run_pre_stages_failed( patch_logging, mocker, mock_swupdate ): mock_manager, mocks, calls = mock_swupdate update_lower_exc = TypeError('some-error') target = 'some-target' mocks['ensure_cluster_is_healthy'].side_effect = update_lower_exc expected_exc_t = SWUpdateError with pytest.raises(expected_exc_t) as excinfo: commands.SWUpdate().run(target) exc = excinfo.value assert type(exc) is expected_exc_t assert exc.reason is update_lower_exc assert exc.rollback_error is None expected_calls = [ calls['ensure_cluster_is_healthy'](), ] assert mock_manager.mock_calls == expected_calls mock_manager.reset_mock() mocks['ensure_cluster_is_healthy'].side_effect = None mocks['_ensure_update_repos_configuration'].side_effect = update_lower_exc expected_exc_t = SWUpdateError with pytest.raises(expected_exc_t) as excinfo: commands.SWUpdate().run(target) exc = excinfo.value assert type(exc) is expected_exc_t assert exc.reason is update_lower_exc assert exc.rollback_error is None expected_calls = [ calls['ensure_cluster_is_healthy'](), calls['_ensure_update_repos_configuration'](target), ] assert mock_manager.mock_calls == expected_calls
def test_commands_SWUpdate_run_happy_path( patch_logging, mocker, mock_swupdate ): mock_manager, mocks, calls = mock_swupdate target = 'some-target' # happy path commands.SWUpdate().run(target) expected_calls = [ calls['ensure_cluster_is_healthy'](), calls['_ensure_update_repos_configuration'](target), calls['YumRollbackManager']( target, multiple_targets_ok=True, pre_rollback_cb=commands._pre_yum_rollback ), calls['YumRollbackManager']().__enter__(), calls['YumRollbackManager']()._resolve_last_txn_ids(), calls['cluster_maintenance_enable'](), calls['_update_component']("provisioner", target), calls['config_salt_master'](), calls['config_salt_minions'](), ] + [ calls['_update_component'](component, target) for component in ( 'motr', 's3server', 'hare', 'ha.cortx-ha', 'sspl', 'csm', 'uds' ) ] + [ calls['cluster_maintenance_disable'](), calls['apply_ha_post_update'](target), calls['ensure_cluster_is_healthy'](), calls['YumRollbackManager']().__exit__(None, None, None) ] assert mock_manager.mock_calls == expected_calls
def test_commands_SWUpdate_run_maintenance_enable_failed( patch_logging, mocker, mock_swupdate): mock_manager, mocks, calls = mock_swupdate target = 'some-target' update_lower_exc = TypeError('some-error') mocks['cluster_maintenance_enable'].side_effect = update_lower_exc with pytest.raises(SWUpdateError) as excinfo: commands.SWUpdate().run(target) exc = excinfo.value assert type(exc) is SWUpdateError assert type(exc.reason) is ClusterMaintenanceEnableError assert exc.reason.reason is update_lower_exc assert exc.rollback_error is None expected_calls = [ calls['ensure_cluster_is_healthy'](), calls['_ensure_update_repos_configuration'](target), calls['_consul_export']('update-pre'), calls['YumRollbackManager']( target, multiple_targets_ok=True, pre_rollback_cb=commands._pre_yum_rollback), calls['YumRollbackManager']().__enter__(), calls['YumRollbackManager']()._resolve_last_txn_ids(), calls['cluster_maintenance_enable'](), calls['YumRollbackManager']().__exit__( ClusterMaintenanceEnableError, # XXX semes not valuable to check exact exc and trace as well mocks['YumRollbackManager'].return_value.__exit__.call_args[0][1], mocks['YumRollbackManager'].return_value.__exit__.call_args[0][2]), calls['YumRollbackManager']()._yum_rollback(), calls['cluster_maintenance_disable'](background=True), ] assert mock_manager.mock_calls == expected_calls
def test_commands_SWUpdate_run_maintenance_enable_at_rollback_failed( patch_logging, mocker, mock_swupdate ): mock_manager, mocks, calls = mock_swupdate update_lower_exc = TypeError('some-error') target = 'some-target' rollback_error = ValueError('some-rollback-error') type(mocks['rollback_ctx']).rollback_error = mocker.PropertyMock( return_value=rollback_error ) mocks['ensure_cluster_is_healthy'].side_effect = [ mocker.DEFAULT, update_lower_exc, mocker.DEFAULT ] mocks['cluster_maintenance_enable'].side_effect = [ mocker.DEFAULT, rollback_error, mocker.DEFAULT ] expected_exc_t = SWUpdateFatalError with pytest.raises(expected_exc_t) as excinfo: commands.SWUpdate().run(target) exc = excinfo.value assert type(exc) is expected_exc_t assert type(exc.reason) is ClusterNotHealthyError assert exc.reason.reason is update_lower_exc assert exc.rollback_error is rollback_error expected_calls = [ calls['ensure_cluster_is_healthy'](), calls['_ensure_update_repos_configuration'](target), calls['YumRollbackManager']( target, multiple_targets_ok=True, pre_rollback_cb=commands._pre_yum_rollback ), calls['YumRollbackManager']().__enter__(), calls['YumRollbackManager']()._resolve_last_txn_ids(), calls['cluster_maintenance_enable'](), calls['_update_component']("provisioner", target), calls['config_salt_master'](), calls['config_salt_minions'](), ] + [ calls['_update_component'](component, target) for component in ( 'motr', 's3server', 'hare', 'ha.cortx-ha', 'sspl', 'csm', 'uds' ) ] + [ calls['cluster_maintenance_disable'](), calls['apply_ha_post_update'](target), calls['ensure_cluster_is_healthy'](), calls['YumRollbackManager']().__exit__( ClusterNotHealthyError, # XXX semes not valuable to check exact exc and trace as well mocks['YumRollbackManager'].return_value.__exit__.call_args[0][1], mocks['YumRollbackManager'].return_value.__exit__.call_args[0][2] ), calls['cluster_maintenance_enable'](), ] assert mock_manager.mock_calls == expected_calls
def test_commands_SWUpdate_run_sw_stack_update_failed( patch_logging, mocker, mock_swupdate, rollback_error ): mock_manager, mocks, calls = mock_swupdate update_lower_exc = TypeError('some-error') target = 'some-target' # TODO IMPROVE parametrize to emulate errors on different stages: # - during provisioner update # - during salt-master config (first time and on rollback) # - during salt-minions config (first time and on rollback) # - ensure_salt_master_is_running on rollback def apply_side_effect(component, *args, **kwargs): if component == 'motr': raise update_lower_exc else: return mocker.DEFAULT type(mocks['rollback_ctx']).rollback_error = mocker.PropertyMock( return_value=rollback_error ) mocks['_update_component'].side_effect = apply_side_effect expected_exc_t = SWUpdateFatalError if rollback_error else SWUpdateError with pytest.raises(expected_exc_t) as excinfo: commands.SWUpdate().run(target) exc = excinfo.value assert type(exc) is expected_exc_t assert type(exc.reason) is SWStackUpdateError assert exc.reason.reason is update_lower_exc assert exc.rollback_error is rollback_error expected_calls = [ calls['ensure_cluster_is_healthy'](), calls['_ensure_update_repos_configuration'](target), calls['_consul_export']('update-pre'), calls['YumRollbackManager']( target, multiple_targets_ok=True, pre_rollback_cb=commands._pre_yum_rollback ), calls['YumRollbackManager']().__enter__(), calls['YumRollbackManager']()._resolve_last_txn_ids(), calls['cluster_maintenance_enable'](), calls['_update_component']("provisioner", target), calls['config_salt_master'](), calls['config_salt_minions'](), calls['_update_component']("motr", target), calls['YumRollbackManager']().__exit__( SWStackUpdateError, # XXX semes not valuable to check exact exc and trace as well mocks['YumRollbackManager'].return_value.__exit__.call_args[0][1], mocks['YumRollbackManager'].return_value.__exit__.call_args[0][2] ), calls['YumRollbackManager']()._yum_rollback(), ] if rollback_error is None: expected_calls.extend([ calls['ensure_salt_master_is_running'](), calls['config_salt_master'](), calls['config_salt_minions'](), calls['_apply_provisioner_config'](target), calls['cluster_maintenance_disable'](), calls['_consul_export']('rollback-pre-ha-update'), calls['apply_ha_post_update'](target), calls['_consul_export']('rollback-post-ha-update'), calls['ensure_cluster_is_healthy'](), calls['_consul_export']('rollback-post'), ]) assert mock_manager.mock_calls == expected_calls
def test_commands_SWUpdate_run_post_rollback_fail( patch_logging, mocker, mock_swupdate, post_rollback_stage_idx ): mock_manager, mocks, calls = mock_swupdate _idx = post_rollback_stage_idx stage = post_rollback_stages[_idx] stage_args = [] parts = stage.split(':') if len(parts) > 1: stage = parts[0] stage_args = tuple(parts[1:]) update_lower_exc = TypeError('some-error') post_rollback_exc = ValueError('some-post-rollback-error') target = 'some-target' # Note. just emulate sw update on early stage # since it's not an object of testing here def apply_side_effect(component, *args, **kwargs): if component == 'provisioner': raise update_lower_exc else: return mocker.DEFAULT _curr_yum_rollback_side_effect = ( mocks['YumRollbackManager'].return_value.__enter__.side_effect ) def yum_rollback_side_effect(*args, **kwargs): def _side_effect(*args, **kwargs): if not stage_args or args == stage_args: raise post_rollback_exc else: return mocker.DEFAULT mocks[stage].side_effect = _side_effect return _curr_yum_rollback_side_effect(*args, **kwargs) mocks['_update_component'].side_effect = apply_side_effect mocks['YumRollbackManager'].return_value.__enter__.side_effect = ( yum_rollback_side_effect ) with pytest.raises(SWUpdateFatalError) as excinfo: commands.SWUpdate().run(target) exc = excinfo.value assert type(exc) is SWUpdateFatalError assert type(exc.reason) is SWStackUpdateError assert exc.reason.reason is update_lower_exc assert exc.rollback_error is post_rollback_exc expected_calls = [ calls['ensure_cluster_is_healthy'](), calls['_ensure_update_repos_configuration'](target), calls['_consul_export']('update-pre'), calls['YumRollbackManager']( target, multiple_targets_ok=True, pre_rollback_cb=commands._pre_yum_rollback ), calls['YumRollbackManager']().__enter__(), calls['YumRollbackManager']()._resolve_last_txn_ids(), calls['cluster_maintenance_enable'](), calls['_update_component']("provisioner", target), calls['YumRollbackManager']().__exit__( SWStackUpdateError, # XXX semes not valuable to check exact exc and trace as well mocks['YumRollbackManager'].return_value.__exit__.call_args[0][1], mocks['YumRollbackManager'].return_value.__exit__.call_args[0][2] ), calls['YumRollbackManager']()._yum_rollback() ] for _stage in post_rollback_stages[:(_idx + 1)]: stage_args = [] parts = _stage.split(':') if len(parts) > 1: _stage = parts[0] stage_args = parts[1:] elif _stage in ( '_apply_provisioner_config', 'apply_ha_post_update' ): stage_args = [target] expected_calls.append(calls[_stage](*stage_args)) assert mock_manager.mock_calls == expected_calls