def test_corosync_targets_skip_offline_unfence_node_running_corosync(
self, ):
self._unfence_failure_common_calls()
self.config.http.corosync.get_corosync_online_targets(
communication_list=[
dict(
label=self.existing_nodes[0],
output=corosync_running_check_response(True),
),
dict(
label=self.existing_nodes[1],
output=corosync_running_check_response(False),
),
dict(
label=self.existing_nodes[2],
was_connected=False,
errno=7,
error_msg="an error",
),
])
self.config.http.scsi.unfence_node(
DEVICES_2,
communication_list=[
dict(
label=self.existing_nodes[0],
raw_data=json.dumps(
dict(devices=DEVICES_2, node=self.existing_nodes[0])),
),
],
)
self.config.env.push_cib(
resources=fixture_scsi(devices=DEVICES_2),
status=_fixture_status_lrm_ops(
SCSI_STONITH_ID,
lrm_start_ops=DEFAULT_LRM_START_OPS_UPDATED,
lrm_monitor_ops=DEFAULT_LRM_MONITOR_OPS_UPDATED,
),
)
stonith.update_scsi_devices(
self.env_assist.get_env(),
SCSI_STONITH_ID,
DEVICES_2,
force_flags=[reports.codes.SKIP_OFFLINE_NODES],
)
self.env_assist.assert_reports([
fixture.warn(
reports.codes.NODE_COMMUNICATION_ERROR_UNABLE_TO_CONNECT,
node=self.existing_nodes[2],
command="remote/status",
reason="an error",
),
])
def command(self, **kwargs):
return lambda: stonith.update_scsi_devices(
self.env_assist.get_env(),
self.stonith_id,
kwargs.get("devices_updated", DEVICES_2),
force_flags=kwargs.get("force_flags", ()),
)
def command(self, force_flags=()):
return lambda: stonith.update_scsi_devices(
self.env_assist.get_env(),
SCSI_STONITH_ID,
DEVICES_2,
force_flags=force_flags,
)
def test_lrm_monitor_ops_not_found(self):
self.config.runner.pcmk.is_resource_digests_supported()
self.config.runner.cib.load(
resources=fixture_scsi(resource_ops=(("monitor", "30s", None,
None), )),
status=_fixture_status_lrm_ops(SCSI_STONITH_ID),
)
self.config.runner.pcmk.load_state(
resources=FIXTURE_CRM_MON_RES_RUNNING_1,
nodes=FIXTURE_CRM_MON_NODES)
self.config.runner.pcmk.resource_digests(
SCSI_STONITH_ID,
SCSI_NODE,
name="start.op.digests",
stdout=fixture_digests_xml(SCSI_STONITH_ID,
SCSI_NODE,
devices=",".join(DEVICES_2)),
args=["devices={}".format(",".join(DEVICES_2))],
)
self.env_assist.assert_raise_library_error(
lambda: stonith.update_scsi_devices(self.env_assist.get_env(),
SCSI_STONITH_ID, DEVICES_2),
[
fixture.error(
reports.codes.STONITH_RESTARTLESS_UPDATE_UNABLE_TO_PERFORM,
reason=(
"monitor lrm_rsc_op element for resource "
f"'{SCSI_STONITH_ID}', node '{SCSI_NODE}' and interval "
"'30000' not found"),
reason_type=reports.const.
STONITH_RESTARTLESS_UPDATE_UNABLE_TO_PERFORM_REASON_OTHER,
)
],
expected_in_processor=False,
)
def test_corosync_targets_unable_to_connect(self):
self._unfence_failure_common_calls()
self.config.http.corosync.get_corosync_online_targets(
communication_list=[
dict(
label=self.existing_nodes[0],
output=corosync_running_check_response(True),
),
] + [
dict(
label=node,
was_connected=False,
errno=7,
error_msg="an error",
) for node in self.existing_nodes[1:]
])
self.env_assist.assert_raise_library_error(
lambda: stonith.update_scsi_devices(self.env_assist.get_env(),
SCSI_STONITH_ID, DEVICES_2), )
self.env_assist.assert_reports([
fixture.error(
reports.codes.NODE_COMMUNICATION_ERROR_UNABLE_TO_CONNECT,
force_code=reports.codes.SKIP_OFFLINE_NODES,
node=node,
command="remote/status",
reason="an error",
) for node in self.existing_nodes[1:]
])
def test_unfence_failure_unable_to_connect(self):
self._unfence_failure_common_calls()
self.config.http.corosync.get_corosync_online_targets(
node_labels=self.existing_nodes)
self.config.http.scsi.unfence_node(
DEVICES_2,
communication_list=[
dict(
label=self.existing_nodes[0],
raw_data=json.dumps(
dict(devices=DEVICES_2, node=self.existing_nodes[0])),
was_connected=False,
error_msg="errA",
),
dict(
label=self.existing_nodes[1],
raw_data=json.dumps(
dict(devices=DEVICES_2, node=self.existing_nodes[1])),
output=json.dumps(
dto.to_dict(
communication.dto.InternalCommunicationResultDto(
status=communication.const.COM_STATUS_ERROR,
status_msg="error",
report_list=[
reports.ReportItem.error(
reports.messages.
StonithUnfencingFailed(
"errB")).to_dto()
],
data=None,
))),
),
dict(
label=self.existing_nodes[2],
raw_data=json.dumps(
dict(devices=DEVICES_2, node=self.existing_nodes[2])),
),
],
)
self.env_assist.assert_raise_library_error(
lambda: stonith.update_scsi_devices(self.env_assist.get_env(),
SCSI_STONITH_ID, DEVICES_2), )
self.env_assist.assert_reports([
fixture.error(
reports.codes.NODE_COMMUNICATION_ERROR_UNABLE_TO_CONNECT,
node=self.existing_nodes[0],
command="api/v1/scsi-unfence-node/v1",
reason="errA",
),
fixture.error(
reports.codes.STONITH_UNFENCING_FAILED,
reason="errB",
context=reports.dto.ReportItemContextDto(
node=self.existing_nodes[1], ),
),
])
def test_pcmk_doesnt_support_digests(self):
self.config.runner.pcmk.is_resource_digests_supported(
is_supported=False)
self.env_assist.assert_raise_library_error(
lambda: stonith.update_scsi_devices(self.env_assist.get_env(),
SCSI_STONITH_ID, ()),
[
fixture.error(
reports.codes.
STONITH_RESTARTLESS_UPDATE_OF_SCSI_DEVICES_NOT_SUPPORTED, )
],
expected_in_processor=False,
)
def test_not_supported_resource_type(self):
self.config.runner.pcmk.is_resource_digests_supported()
self.config.runner.cib.load(resources=fixture_scsi())
self.env_assist.assert_raise_library_error(
lambda: stonith.update_scsi_devices(self.env_assist.get_env(),
"dummy", DEVICES_2))
self.env_assist.assert_reports([
fixture.error(
reports.codes.STONITH_RESTARTLESS_UPDATE_UNSUPPORTED_AGENT,
resource_id="dummy",
resource_type="Dummy",
supported_stonith_types=["fence_scsi"],
)
])
def test_nonexistant_id(self):
self.config.runner.pcmk.is_resource_digests_supported()
self.config.runner.cib.load(resources=fixture_scsi())
self.env_assist.assert_raise_library_error(
lambda: stonith.update_scsi_devices(self.env_assist.get_env(),
"non-existent-id", DEVICES_2))
self.env_assist.assert_reports([
fixture.error(
reports.codes.ID_NOT_FOUND,
id="non-existent-id",
expected_types=["primitive"],
context_type="cib",
context_id="",
)
])
def test_devices_option_empty(self):
self.config.runner.pcmk.is_resource_digests_supported()
self.config.runner.cib.load(resources=fixture_scsi(devices=""))
self.env_assist.assert_raise_library_error(
lambda: stonith.update_scsi_devices(self.env_assist.get_env(),
SCSI_STONITH_ID, DEVICES_2))
self.env_assist.assert_reports([
fixture.error(
reports.codes.STONITH_RESTARTLESS_UPDATE_UNABLE_TO_PERFORM,
reason=("no devices option configured for stonith device "
f"'{SCSI_STONITH_ID}'"),
reason_type=reports.const.
STONITH_RESTARTLESS_UPDATE_UNABLE_TO_PERFORM_REASON_OTHER,
)
])
def test_node_missing_name_and_missing_auth_token(self):
self.config.runner.pcmk.is_resource_digests_supported()
self.config.runner.cib.load(
resources=fixture_scsi(),
status=_fixture_status_lrm_ops(SCSI_STONITH_ID),
)
self.config.runner.pcmk.load_state(
resources=FIXTURE_CRM_MON_RES_RUNNING_1,
nodes=FIXTURE_CRM_MON_NODES)
self.config.runner.pcmk.resource_digests(
SCSI_STONITH_ID,
SCSI_NODE,
name="start.op.digests",
stdout=fixture_digests_xml(SCSI_STONITH_ID,
SCSI_NODE,
devices=",".join(DEVICES_2)),
args=["devices={}".format(",".join(DEVICES_2))],
)
self.config.runner.pcmk.resource_digests(
SCSI_STONITH_ID,
SCSI_NODE,
name="monitor.op.digests",
stdout=fixture_digests_xml(SCSI_STONITH_ID,
SCSI_NODE,
devices=",".join(DEVICES_2)),
args=[
"devices={}".format(",".join(DEVICES_2)),
"CRM_meta_interval=60000",
],
)
self.config.corosync_conf.load_content(
corosync_conf_fixture(
self.existing_corosync_nodes + [[("ring0_addr", "custom_node"),
("nodeid", "5")]], ))
self.config.env.set_known_nodes(self.existing_nodes[:-1])
self.env_assist.assert_raise_library_error(
lambda: stonith.update_scsi_devices(self.env_assist.get_env(),
SCSI_STONITH_ID, DEVICES_2), )
self.env_assist.assert_reports([
fixture.error(
reports.codes.COROSYNC_CONFIG_MISSING_NAMES_OF_NODES,
fatal=True,
),
fixture.error(
reports.codes.HOST_NOT_FOUND,
host_list=[self.existing_nodes[-1]],
),
])
def test_devices_cannot_be_empty(self):
self.config.runner.pcmk.is_resource_digests_supported()
self.config.runner.cib.load(resources=fixture_scsi())
self.env_assist.assert_raise_library_error(
lambda: stonith.update_scsi_devices(self.env_assist.get_env(),
SCSI_STONITH_ID, ()))
self.env_assist.assert_reports([
fixture.error(
reports.codes.INVALID_OPTION_VALUE,
option_name="devices",
option_value="",
allowed_values=None,
cannot_be_empty=True,
forbidden_characters=None,
)
])
def test_not_a_resource_id(self):
self.config.runner.pcmk.is_resource_digests_supported()
self.config.runner.cib.load(resources=fixture_scsi())
self.env_assist.assert_raise_library_error(
lambda: stonith.update_scsi_devices(
self.env_assist.get_env(),
f"{SCSI_STONITH_ID}-instance_attributes-devices",
DEVICES_2,
))
self.env_assist.assert_reports([
fixture.error(
reports.codes.ID_BELONGS_TO_UNEXPECTED_TYPE,
id=f"{SCSI_STONITH_ID}-instance_attributes-devices",
expected_types=["primitive"],
current_type="nvpair",
)
])
def test_stonith_resource_is_not_running(self):
self.config.runner.cib.load(resources=fixture_scsi())
self.config.runner.pcmk.is_resource_digests_supported()
self.config.runner.pcmk.load_state(
resources=FIXTURE_CRM_MON_RES_STOPPED, nodes=FIXTURE_CRM_MON_NODES
)
self.env_assist.assert_raise_library_error(
lambda: stonith.update_scsi_devices(
self.env_assist.get_env(), SCSI_STONITH_ID, DEVICES_2
),
[
fixture.error(
reports.codes.STONITH_RESTARTLESS_UPDATE_UNABLE_TO_PERFORM,
reason=f"resource '{SCSI_STONITH_ID}' is not running on any node",
reason_type=reports.const.STONITH_RESTARTLESS_UPDATE_UNABLE_TO_PERFORM_REASON_NOT_RUNNING,
)
],
expected_in_processor=False,
)
def test_crm_resource_digests_missing(self):
devices = ",".join(DEVICES_2)
self.config.runner.cib.load(
resources=fixture_scsi(),
status=_fixture_status_lrm_ops_base(
SCSI_STONITH_ID,
(
f'<lrm_rsc_op id="{SCSI_STONITH_ID}_last" '
'operation="start" op-restart-digest="somedigest" />'
),
),
)
self.config.runner.pcmk.is_resource_digests_supported()
self.config.runner.pcmk.load_state(
resources=FIXTURE_CRM_MON_RES_RUNNING_1, nodes=FIXTURE_CRM_MON_NODES
)
self.config.runner.pcmk.resource_digests(
SCSI_STONITH_ID,
SCSI_NODE,
name="start.op.digests",
stdout=fixture_digests_xml(
SCSI_STONITH_ID,
SCSI_NODE,
devices=devices,
),
args=[f"devices={devices}"],
)
self.env_assist.assert_raise_library_error(
lambda: stonith.update_scsi_devices(
self.env_assist.get_env(), SCSI_STONITH_ID, DEVICES_2
),
[
fixture.error(
reports.codes.STONITH_RESTARTLESS_UPDATE_UNABLE_TO_PERFORM,
reason=(
"necessary digest for 'op-restart-digest' attribute is "
"missing"
),
reason_type=reports.const.STONITH_RESTARTLESS_UPDATE_UNABLE_TO_PERFORM_REASON_OTHER,
)
],
expected_in_processor=False,
)
def test_corosync_targets_unable_to_perform_unfencing_operation(self, ):
self._unfence_failure_common_calls()
self.config.http.corosync.get_corosync_online_targets(
communication_list=[
dict(
label=self.existing_nodes[0],
was_connected=False,
errno=7,
error_msg="an error",
),
dict(
label=self.existing_nodes[1],
was_connected=False,
errno=7,
error_msg="an error",
),
dict(
label=self.existing_nodes[2],
output=corosync_running_check_response(False),
),
])
self.config.http.scsi.unfence_node(DEVICES_2, communication_list=[])
self.env_assist.assert_raise_library_error(
lambda: stonith.update_scsi_devices(
self.env_assist.get_env(),
SCSI_STONITH_ID,
DEVICES_2,
force_flags=[reports.codes.SKIP_OFFLINE_NODES],
), )
self.env_assist.assert_reports([
fixture.warn(
reports.codes.NODE_COMMUNICATION_ERROR_UNABLE_TO_CONNECT,
node=node,
command="remote/status",
reason="an error",
) for node in self.existing_nodes[0:2]
] + [
fixture.error(
reports.codes.UNABLE_TO_PERFORM_OPERATION_ON_ANY_NODE, ),
])
def test_no_lrm_start_op(self):
self.config.runner.cib.load(
resources=fixture_scsi(),
status=_fixture_status_lrm_ops(SCSI_STONITH_ID, lrm_start_ops=()),
)
self.config.runner.pcmk.is_resource_digests_supported()
self.config.runner.pcmk.load_state(
resources=FIXTURE_CRM_MON_RES_RUNNING_1, nodes=FIXTURE_CRM_MON_NODES
)
self.env_assist.assert_raise_library_error(
lambda: stonith.update_scsi_devices(
self.env_assist.get_env(), SCSI_STONITH_ID, DEVICES_2
),
[
fixture.error(
reports.codes.STONITH_RESTARTLESS_UPDATE_UNABLE_TO_PERFORM,
reason=(
"lrm_rsc_op element for start operation was not found"
),
reason_type=reports.const.STONITH_RESTARTLESS_UPDATE_UNABLE_TO_PERFORM_REASON_OTHER,
)
],
expected_in_processor=False,
)
def assert_command_success(
self,
devices_before=DEVICES_1,
devices_updated=DEVICES_2,
devices_add=(),
devices_remove=(),
unfence=None,
resource_ops=DEFAULT_OPS,
lrm_monitor_ops=DEFAULT_LRM_MONITOR_OPS,
lrm_start_ops=DEFAULT_LRM_START_OPS,
lrm_monitor_ops_updated=DEFAULT_LRM_MONITOR_OPS_UPDATED,
lrm_start_ops_updated=DEFAULT_LRM_START_OPS_UPDATED,
):
# pylint: disable=too-many-arguments
# pylint: disable=too-many-locals
devices_value = ",".join(sorted(devices_updated))
self.config.runner.cib.load(
resources=fixture_scsi(
devices=devices_before, resource_ops=resource_ops
),
status=_fixture_status_lrm_ops(
SCSI_STONITH_ID,
lrm_start_ops=lrm_start_ops,
lrm_monitor_ops=lrm_monitor_ops,
),
)
self.config.runner.pcmk.is_resource_digests_supported()
self.config.runner.pcmk.load_state(
resources=FIXTURE_CRM_MON_RES_RUNNING_1, nodes=FIXTURE_CRM_MON_NODES
)
devices_opt = "devices={}".format(devices_value)
self.config.runner.pcmk.resource_digests(
SCSI_STONITH_ID,
SCSI_NODE,
name="start.op.digests",
stdout=fixture_digests_xml(
SCSI_STONITH_ID, SCSI_NODE, devices=devices_value
),
args=[devices_opt],
)
for num, op in enumerate(resource_ops, 1):
name, interval, timeout, _ = op
if name != "monitor":
continue
args = [devices_opt]
args.append(
"CRM_meta_interval={}".format(
1000 * timeout_to_seconds(interval)
)
)
if timeout:
args.append(
"CRM_meta_timeout={}".format(
1000 * timeout_to_seconds(timeout)
)
)
self.config.runner.pcmk.resource_digests(
SCSI_STONITH_ID,
SCSI_NODE,
name=f"{name}-{num}.op.digests",
stdout=fixture_digests_xml(
SCSI_STONITH_ID,
SCSI_NODE,
devices=devices_value,
),
args=args,
)
if unfence:
self.config.corosync_conf.load_content(
corosync_conf_fixture(
self.existing_corosync_nodes,
get_two_node(len(self.existing_corosync_nodes)),
)
)
self.config.http.corosync.get_corosync_online_targets(
node_labels=self.existing_nodes
)
self.config.http.scsi.unfence_node(
original_devices=devices_before,
updated_devices=devices_updated,
node_labels=self.existing_nodes,
)
self.config.env.push_cib(
resources=fixture_scsi(
devices=devices_updated, resource_ops=resource_ops
),
status=_fixture_status_lrm_ops(
SCSI_STONITH_ID,
lrm_start_ops=lrm_start_ops_updated,
lrm_monitor_ops=lrm_monitor_ops_updated,
),
)
if devices_add or devices_remove:
stonith.update_scsi_devices_add_remove(
self.env_assist.get_env(),
SCSI_STONITH_ID,
devices_add,
devices_remove,
)
else:
stonith.update_scsi_devices(
self.env_assist.get_env(), SCSI_STONITH_ID, devices_updated
)
self.env_assist.assert_reports([])
