def _test_minimal(self, local_cluster_size, recovery_cluster_size):
local_nodes = generate_nodes(local_cluster_size)
remote_nodes = generate_nodes(recovery_cluster_size,
prefix="recovery-")
orig_node = remote_nodes[-1]
cfg = self.config
cfg.env.set_known_nodes(local_nodes + remote_nodes)
cfg.raw_file.exists(
file_type_codes.PCS_DR_CONFIG,
settings.pcsd_dr_config_location,
exists=False,
)
cfg.corosync_conf.load_content(corosync_conf_fixture(local_nodes))
cfg.http.corosync.get_corosync_conf(
corosync_conf_fixture(remote_nodes), node_labels=[orig_node])
cfg.http.files.put_files(
node_labels=remote_nodes,
pcs_disaster_recovery_conf=dr_cfg_fixture(DrRole.RECOVERY,
DrRole.PRIMARY,
local_nodes),
name="distribute_remote",
)
cfg.http.files.put_files(
node_labels=local_nodes,
pcs_disaster_recovery_conf=dr_cfg_fixture(DrRole.PRIMARY,
DrRole.RECOVERY,
remote_nodes),
name="distribute_local",
)
dr.set_recovery_site(self.env_assist.get_env(), orig_node)
self.env_assist.assert_reports([
fixture.info(
report_codes.FILES_DISTRIBUTION_STARTED,
file_list=[DR_CFG_DESC],
node_list=remote_nodes,
)
] + [
fixture.info(
report_codes.FILE_DISTRIBUTION_SUCCESS,
file_description=DR_CFG_DESC,
node=node,
) for node in remote_nodes
] + [
fixture.info(
report_codes.FILES_DISTRIBUTION_STARTED,
file_list=[DR_CFG_DESC],
node_list=local_nodes,
)
] + [
fixture.info(
report_codes.FILE_DISTRIBUTION_SUCCESS,
file_description=DR_CFG_DESC,
node=node,
) for node in local_nodes
])
def assert_live_required(self, forbidden_options):
self.env_assist.assert_raise_library_error(
lambda: dr.set_recovery_site(self.env_assist.get_env(), "node"), [
fixture.error(report_codes.LIVE_ENVIRONMENT_REQUIRED,
forbidden_options=forbidden_options)
],
expected_in_processor=False)
def test_write_failure(self):
self.config.http.files.put_files(
communication_list=self.success_communication + [
dict(
label=node,
output=json.dumps(
dict(files={
DR_CFG_DESC:
dict(code="unexpected", message=REASON),
}))) for node in self.failed_nodes
],
pcs_disaster_recovery_conf=dr_cfg_fixture(DrRole.PRIMARY,
DrRole.RECOVERY,
self.remote_nodes),
)
self.env_assist.assert_raise_library_error(
lambda: dr.set_recovery_site(self.env_assist.get_env(), self.node),
)
self.env_assist.assert_reports(self.expected_reports + [
fixture.error(
report_codes.FILE_DISTRIBUTION_ERROR,
file_description=DR_CFG_DESC,
reason=REASON,
node=node,
) for node in self.failed_nodes
])
def test_network_failure(self):
self.config.http.files.put_files(
communication_list=self.success_communication
+ [
dict(
label=node,
was_connected=False,
error_msg=REASON,
)
for node in self.failed_nodes
],
pcs_disaster_recovery_conf=dr_cfg_fixture(
DrRole.PRIMARY, DrRole.RECOVERY, self.remote_nodes
),
)
self.env_assist.assert_raise_library_error(
lambda: dr.set_recovery_site(self.env_assist.get_env(), self.node),
)
self.env_assist.assert_reports(
self.expected_reports
+ [
fixture.error(
report_codes.NODE_COMMUNICATION_ERROR_UNABLE_TO_CONNECT,
command="remote/put_file",
reason=REASON,
node=node,
)
for node in self.failed_nodes
]
)
def test_communication_error(self):
self.config.http.files.put_files(
communication_list=self.success_communication
+ [
dict(
label=node,
response_code=400,
output=REASON,
)
for node in self.failed_nodes
],
pcs_disaster_recovery_conf=dr_cfg_fixture(
DrRole.PRIMARY, DrRole.RECOVERY, self.remote_nodes
),
)
self.env_assist.assert_raise_library_error(
lambda: dr.set_recovery_site(self.env_assist.get_env(), self.node),
)
self.env_assist.assert_reports(
self.expected_reports
+ [
fixture.error(
report_codes.NODE_COMMUNICATION_COMMAND_UNSUCCESSFUL,
command="remote/put_file",
reason=REASON,
node=node,
)
for node in self.failed_nodes
]
)
def test_node_names_missing(self):
self.config.http.corosync.get_corosync_conf(
COROSYNC_CONF_TEMPLATE.format(
node_list="\n".join(
[
NODE_TEMPLATE_NO_NAME.format(
node=self.remote_nodes[-1],
id=len(self.remote_nodes),
)
]
+ node_list_fixture(self.remote_nodes[:-1])
)
),
node_labels=[self.node],
)
self.env_assist.assert_raise_library_error(
lambda: dr.set_recovery_site(self.env_assist.get_env(), self.node),
)
self.env_assist.assert_reports(
[
fixture.error(
report_codes.COROSYNC_CONFIG_MISSING_NAMES_OF_NODES,
fatal=True,
)
]
)
def test_file_does_not_exist(self):
self.config.http.corosync.get_corosync_conf(
communication_list=[
dict(
label=self.node,
response_code=400,
output=REASON,
)
]
)
self.env_assist.assert_raise_library_error(
lambda: dr.set_recovery_site(self.env_assist.get_env(), self.node),
)
self.env_assist.assert_reports(
[
fixture.warn(
report_codes.NODE_COMMUNICATION_COMMAND_UNSUCCESSFUL,
node=self.node,
command="remote/get_corosync_conf",
reason=REASON,
),
fixture.error(
report_codes.UNABLE_TO_PERFORM_OPERATION_ON_ANY_NODE
),
]
)
def test_network_issue(self):
self.config.http.corosync.get_corosync_conf(
communication_list=[
dict(
label=self.node,
was_connected=False,
error_msg=REASON,
)
]
)
self.env_assist.assert_raise_library_error(
lambda: dr.set_recovery_site(self.env_assist.get_env(), self.node),
)
self.env_assist.assert_reports(
[
fixture.warn(
report_codes.NODE_COMMUNICATION_ERROR_UNABLE_TO_CONNECT,
node=self.node,
command="remote/get_corosync_conf",
reason=REASON,
),
fixture.error(
report_codes.UNABLE_TO_PERFORM_OPERATION_ON_ANY_NODE
),
]
)
def test_tokens_missing_for_remote_cluster(self):
remote_nodes = generate_nodes(3, prefix="recovery-")
orig_node = remote_nodes[0]
cfg = self.config
cfg.env.set_known_nodes(self.local_nodes + remote_nodes[:-1])
cfg.raw_file.exists(
file_type_codes.PCS_DR_CONFIG,
settings.pcsd_dr_config_location,
exists=False,
)
cfg.corosync_conf.load_content(corosync_conf_fixture(self.local_nodes))
cfg.http.corosync.get_corosync_conf(
corosync_conf_fixture(remote_nodes), node_labels=[orig_node]
)
self.env_assist.assert_raise_library_error(
lambda: dr.set_recovery_site(self.env_assist.get_env(), orig_node),
)
self.env_assist.assert_reports(
[
fixture.error(
report_codes.HOST_NOT_FOUND,
host_list=remote_nodes[-1:],
)
]
)
def test_local_nodes_name_missing(self):
orig_node = "node"
cfg = self.config
cfg.env.set_known_nodes(self.local_nodes + [orig_node])
cfg.raw_file.exists(
file_type_codes.PCS_DR_CONFIG,
settings.pcsd_dr_config_location,
exists=False,
)
cfg.corosync_conf.load_content(
COROSYNC_CONF_TEMPLATE.format(
node_list="\n".join(
[
NODE_TEMPLATE_NO_NAME.format(
node=self.local_nodes[0], id=len(self.local_nodes)
)
]
+ node_list_fixture(self.local_nodes[1:])
)
)
)
self.env_assist.assert_raise_library_error(
lambda: dr.set_recovery_site(self.env_assist.get_env(), orig_node),
)
self.env_assist.assert_reports(
[
fixture.error(
report_codes.COROSYNC_CONFIG_MISSING_NAMES_OF_NODES,
fatal=True,
)
]
)
def test_dr_cfg_exist(self):
orig_node = "node"
cfg = self.config
cfg.env.set_known_nodes(self.local_nodes + [orig_node])
cfg.raw_file.exists(
file_type_codes.PCS_DR_CONFIG,
settings.pcsd_dr_config_location,
exists=True,
)
cfg.corosync_conf.load_content(corosync_conf_fixture(self.local_nodes))
self.env_assist.assert_raise_library_error(
lambda: dr.set_recovery_site(self.env_assist.get_env(), orig_node),
)
self.env_assist.assert_reports(
[fixture.error(report_codes.DR_CONFIG_ALREADY_EXIST, )])
def test_token_missing_for_node(self):
orig_node = "node"
cfg = self.config
cfg.env.set_known_nodes(self.local_nodes)
cfg.raw_file.exists(
file_type_codes.PCS_DR_CONFIG,
settings.pcsd_dr_config_location,
exists=False,
)
cfg.corosync_conf.load_content(corosync_conf_fixture(self.local_nodes))
self.env_assist.assert_raise_library_error(
lambda: dr.set_recovery_site(self.env_assist.get_env(), orig_node),
)
self.env_assist.assert_reports([
fixture.error(
report_codes.HOST_NOT_FOUND,
host_list=[orig_node],
)
])
def test_node_part_of_local_cluster(self):
orig_node = self.local_nodes[-1]
cfg = self.config
cfg.env.set_known_nodes(self.local_nodes + [orig_node])
cfg.raw_file.exists(
file_type_codes.PCS_DR_CONFIG,
settings.pcsd_dr_config_location,
exists=False,
)
cfg.corosync_conf.load_content(corosync_conf_fixture(self.local_nodes))
self.env_assist.assert_raise_library_error(
lambda: dr.set_recovery_site(self.env_assist.get_env(), orig_node),
)
self.env_assist.assert_reports([
fixture.error(
report_codes.NODE_IN_LOCAL_CLUSTER,
node=orig_node,
)
])
