def get_local_node_status(runner):
try:
cluster_status = ClusterState(get_cluster_status_dom(runner))
node_name = get_local_node_name(runner)
except PacemakerNotConnectedException:
return {"offline": True}
for node_status in cluster_status.node_section.nodes:
if node_status.attrs.name == node_name:
result = {
"offline": False,
}
for attr in (
"id",
"name",
"type",
"online",
"standby",
"standby_onfail",
"maintenance",
"pending",
"unclean",
"shutdown",
"expected_up",
"is_dc",
"resources_running",
):
result[attr] = getattr(node_status.attrs, attr)
return result
raise LibraryError(
ReportItem.error(reports.messages.NodeNotFound(node_name)))
def verify(env, verbose=False):
runner = env.cmd_runner()
dummy_stdout, verify_stderr, verify_returncode = verify_cmd(
runner,
verbose=verbose,
)
#1) Do not even try to think about upgrading!
#2) We do not need cib management in env (no need for push...).
#So env.get_cib is not best choice here (there were considerations to
#upgrade cib at all times inside env.get_cib). Go to a lower level here.
if verify_returncode != 0:
env.report_processor.append(reports.invalid_cib_content(verify_stderr))
#Cib is sometimes loadable even if `crm_verify` fails (e.g. when
#fencing topology is invalid). On the other hand cib with id duplication
#is not loadable.
#We try extra checks when cib is possible to load.
cib_xml, dummy_stderr, returncode = get_cib_xml_cmd_results(runner)
if returncode != 0:
#can raise; raise LibraryError is better but in this case we prefer
#be consistent with raising below
env.report_processor.send()
else:
cib_xml = get_cib_xml(runner)
cib = get_cib(cib_xml)
fencing_topology.verify(
env.report_processor, get_fencing_topology(cib), get_resources(cib),
ClusterState(get_cluster_status_xml(runner)).node_section.nodes)
#can raise
env.report_processor.send()
def get_status(self):
return ClusterState("""
<crm_mon version="2.0.3">
<summary>
<stack type="corosync" />
<current_dc present="true" />
<last_update time="Wed Nov 6 13:45:41 2019" />
<last_change time="Wed Nov 6 10:42:54 2019"
user="******" client="crmd" origin="node1"
/>
<nodes_configured number="2" />
<resources_configured number="0" disabled="0" blocked="0" />
<cluster_options stonith-enabled="true"
symmetric-cluster="true" no-quorum-policy="stop"
maintenance-mode="false"
/>
</summary>
<nodes>
<node name="nodeA" id="1" online="true" standby="false"
standby_onfail="false" maintenance="false"
pending="false" unclean="false" shutdown="false"
expected_up="true" is_dc="true" resources_running="0"
type="member"
/>
<node name="nodeB" id="2" online="true" standby="false"
standby_onfail="false" maintenance="false"
pending="false" unclean="false" shutdown="false"
expected_up="true" is_dc="false" resources_running="0"
type="member"
/>
</nodes>
</crm_mon>
""").node_section.nodes
def get_local_node_status(runner):
try:
cluster_status = ClusterState(get_cluster_status_xml(runner))
except CrmMonErrorException:
return {"offline": True}
node_name = get_local_node_name(runner)
for node_status in cluster_status.node_section.nodes:
if node_status.attrs.name == node_name:
result = {
"offline": False,
}
for attr in (
'id',
'name',
'type',
'online',
'standby',
'standby_onfail',
'maintenance',
'pending',
'unclean',
'shutdown',
'expected_up',
'is_dc',
'resources_running',
):
result[attr] = getattr(node_status.attrs, attr)
return result
raise LibraryError(reports.node_not_found(node_name))
def resource_refresh(
runner: CommandRunner,
resource: Optional[str] = None,
node: Optional[str] = None,
strict: bool = False,
force: bool = False,
):
if not force and not node and not resource:
summary = ClusterState(get_cluster_status_xml(runner)).summary
operations = summary.nodes.attrs.count * summary.resources.attrs.count
if operations > __RESOURCE_REFRESH_OPERATION_COUNT_THRESHOLD:
raise LibraryError(
reports.resource_refresh_too_time_consuming(
__RESOURCE_REFRESH_OPERATION_COUNT_THRESHOLD))
cmd = [__exec("crm_resource"), "--refresh"]
if resource:
cmd.extend(["--resource", resource])
if node:
cmd.extend(["--node", node])
if strict:
cmd.extend(["--force"])
stdout, stderr, retval = runner.run(cmd)
if retval != 0:
raise LibraryError(
reports.resource_refresh_error(join_multilines([stderr, stdout]),
resource, node))
# usefull output (what has been done) goes to stderr
return join_multilines([stdout, stderr])
def add_level(lib_env,
level,
target_type,
target_value,
devices,
force_device=False,
force_node=False):
"""
Validate and add a new fencing level
LibraryError lib_env -- environment
int|string level -- level (index) of the new fencing level
constant target_type -- the new fencing level target value type
mixed target_value -- the new fencing level target value
Iterable devices -- list of stonith devices for the new fencing level
bool force_device -- continue even if a stonith device does not exist
bool force_node -- continue even if a node (target) does not exist
"""
version_check = None
if target_type == TARGET_TYPE_REGEXP:
version_check = (2, 3, 0)
elif target_type == TARGET_TYPE_ATTRIBUTE:
version_check = (2, 4, 0)
cib = lib_env.get_cib(version_check)
cib_fencing_topology.add_level(
lib_env.report_processor, get_fencing_topology(cib),
get_resources(cib), level, target_type, target_value, devices,
ClusterState(get_cluster_status_xml(
lib_env.cmd_runner())).node_section.nodes, force_device,
force_node)
lib_env.report_processor.send()
lib_env.push_cib()
def test_refuse_invalid_document(self):
self.covered_status.append_to_first_tag_name(
'nodes', '<node without="required attributes" />')
assert_raise_library_error(
lambda: ClusterState(str(self.covered_status)),
(severities.ERROR, report_codes.BAD_CLUSTER_STATE_FORMAT, {}))
def setUp(self):
self.node1 = etree.fromstring("""
<node id="1" uname="name-test1" type="member" />
""")
self.node2 = etree.fromstring("""
<node id="2" uname="name-test2" type="member" />
""")
self.nodes = etree.Element("nodes")
self.nodes.append(self.node1)
self.state = ClusterState("""
<crm_mon version="1.1.15">
<summary>
<current_dc present="true" />
<nodes_configured number="2" expected_votes="unknown" />
<resources_configured number="0" />
</summary>
<nodes>
<node name="name-test1" id="1" online="true" standby="false"
standby_onfail="false" maintenance="false"
pending="false" unclean="false" shutdown="false"
expected_up="true" is_dc="true" resources_running="0"
type="member"
/>
<node name="name-test2" id="2" online="true" standby="false"
standby_onfail="false" maintenance="false"
pending="false" unclean="false" shutdown="false"
expected_up="true" is_dc="false" resources_running="0"
type="member"
/>
</nodes>
</crm_mon>
""").node_section.nodes
def test_resources_count(self):
with open(rc("crm_mon.minimal.xml")) as crm_mon_file:
crm_mon_xml = crm_mon_file.read()
self.assertEqual(
0,
ClusterState(
complete_state(crm_mon_xml)).summary.resources.attrs.count,
)
def cib_runner_nodes(lib_env: LibraryEnvironment, wait: WaitType):
wait_timeout = lib_env.ensure_wait_satisfiable(wait)
yield (
lib_env.get_cib(),
lib_env.cmd_runner(),
ClusterState(lib_env.get_cluster_state()).node_section.nodes,
)
lib_env.push_cib(wait_timeout=wait_timeout)
def test_refuse_invalid_document(self, mock_validate):
mock_validate.side_effect = etree.DocumentInvalid("some error")
self.covered_status.append_to_first_tag_name(
"nodes", '<node without="required attributes" />')
assert_raise_library_error(
lambda: ClusterState(str(self.covered_status)),
(severities.ERROR, report_codes.BAD_CLUSTER_STATE_FORMAT, {}),
)
def cib_runner_nodes(lib_env, wait):
lib_env.ensure_wait_satisfiable(wait)
runner = lib_env.cmd_runner()
state_nodes = ClusterState(
get_cluster_status_xml(runner)).node_section.nodes
yield (lib_env.get_cib(), runner, state_nodes)
lib_env.push_cib(wait=wait)
def test_can_get_node_names(self):
self.covered_status.append_to_first_tag_name(
'nodes',
self.fixture_node_string(name='node1', id='1'),
self.fixture_node_string(name='node2', id='2'),
)
xml = str(self.covered_status)
self.assertEqual(
['node1', 'node2'],
[node.attrs.name for node in ClusterState(xml).node_section.nodes])
def test_can_get_node_names(self):
self.covered_status.append_to_first_tag_name(
"nodes",
self.fixture_node_string(name="node1", id="1"),
self.fixture_node_string(name="node2", id="2"),
)
xml = str(self.covered_status)
self.assertEqual(
["node1", "node2"],
[node.attrs.name for node in ClusterState(xml).node_section.nodes],
)
def test_can_filter_out_remote_nodes(self):
self.covered_status.append_to_first_tag_name(
'nodes',
self.fixture_node_string(name='node1', id='1'),
self.fixture_node_string(name='node2', type='remote', id='2'),
)
xml = str(self.covered_status)
self.assertEqual(['node1'], [
node.attrs.name for node in ClusterState(xml).node_section.nodes
if node.attrs.type != 'remote'
])
def get_status(self):
with open(rc("crm_mon.minimal.xml")) as crm_mon_file:
crm_mon_xml = crm_mon_file.read()
return ClusterState(
fixture_crm_mon.complete_state(
crm_mon_xml,
nodes_xml="""
<nodes>
<node name="nodeA" id="1" is_dc="true" />
<node name="nodeB" id="2" />
</nodes>
""",
)).node_section.nodes
def verify(lib_env):
"""
Check if all cluster nodes and stonith devices used in fencing levels exist
LibraryError lib_env -- environment
"""
cib = lib_env.get_cib()
cib_fencing_topology.verify(
lib_env.report_processor, get_fencing_topology(cib),
get_resources(cib),
ClusterState(get_cluster_status_xml(
lib_env.cmd_runner())).node_section.nodes)
lib_env.report_processor.send()
def verify(lib_env: LibraryEnvironment):
"""
Check if all cluster nodes and stonith devices used in fencing levels exist
LibraryEnvironment lib_env -- environment
"""
cib = lib_env.get_cib()
lib_env.report_processor.report_list(
cib_fencing_topology.verify(
get_fencing_topology(cib),
get_resources(cib),
ClusterState(lib_env.get_cluster_state()).node_section.nodes,
))
if lib_env.report_processor.has_errors:
raise LibraryError()
def test_can_filter_out_remote_nodes(self):
self.covered_status.append_to_first_tag_name(
"nodes",
self.fixture_node_string(name="node1", id="1"),
self.fixture_node_string(name="node2", type="remote", id="2"),
)
xml = str(self.covered_status)
self.assertEqual(
["node1"],
[
node.attrs.name
for node in ClusterState(xml).node_section.nodes
if node.attrs.type != "remote"
],
)
def test_can_get_node_names(self):
with open(rc("crm_mon.minimal.xml")) as crm_mon_file:
crm_mon_xml = crm_mon_file.read()
state_dom = complete_state(
crm_mon_xml,
nodes_xml="""
<nodes>
<node name="node1" id="1" />
<node name="node2" id="2" />
</nodes>
""",
)
self.assertEqual(
["node1", "node2"],
[
node.attrs.name
for node in ClusterState(state_dom).node_section.nodes
],
)
def add_level(
lib_env: LibraryEnvironment,
level,
target_type,
target_value,
devices,
force_device=False,
force_node=False,
):
"""
Validate and add a new fencing level
LibraryEnvironment lib_env -- environment
int|string level -- level (index) of the new fencing level
constant target_type -- the new fencing level target value type
mixed target_value -- the new fencing level target value
Iterable devices -- list of stonith devices for the new fencing level
bool force_device -- continue even if a stonith device does not exist
bool force_node -- continue even if a node (target) does not exist
"""
cib = lib_env.get_cib()
cib_fencing_topology.add_level(
lib_env.report_processor,
get_fencing_topology(cib),
get_resources(cib),
level,
target_type,
target_value,
devices,
ClusterState(lib_env.get_cluster_state()).node_section.nodes,
force_device,
force_node,
)
if lib_env.report_processor.has_errors:
raise LibraryError()
lib_env.push_cib()
def get_status(self):
return ClusterState("""
<crm_mon version="1.1.15">
<summary>
<current_dc present="true" />
<nodes_configured number="2" expected_votes="unknown" />
<resources_configured number="0" />
</summary>
<nodes>
<node name="nodeA" id="1" online="true" standby="false"
standby_onfail="false" maintenance="false"
pending="false" unclean="false" shutdown="false"
expected_up="true" is_dc="true" resources_running="0"
type="member"
/>
<node name="nodeB" id="2" online="true" standby="false"
standby_onfail="false" maintenance="false"
pending="false" unclean="false" shutdown="false"
expected_up="true" is_dc="false" resources_running="0"
type="member"
/>
</nodes>
</crm_mon>
""").node_section.nodes
def nodes_status(lib, argv, modifiers):
"""
Options:
* -f - CIB file - for config subcommand and not for both or corosync
* --corosync_conf - only for config subcommand
NOTE: modifiers check is in subcommand
"""
del lib
if len(argv) == 1 and (argv[0] == "config"):
modifiers.ensure_only_supported("-f", "--corosync_conf")
if utils.hasCorosyncConf():
corosync_nodes, report_list = get_existing_nodes_names(
utils.get_corosync_conf_facade())
if report_list:
process_library_reports(report_list)
else:
corosync_nodes = []
try:
pacemaker_nodes = sorted([
node.attrs.name for node in ClusterState(
get_cluster_status_dom(
utils.cmd_runner())).node_section.nodes
if node.attrs.type != "remote"
])
except LibraryError as e:
process_library_reports(e.args)
print("Corosync Nodes:")
if corosync_nodes:
print(" " + " ".join(corosync_nodes))
print("Pacemaker Nodes:")
if pacemaker_nodes:
print(" " + " ".join(pacemaker_nodes))
return
if len(argv) == 1 and (argv[0] == "corosync" or argv[0] == "both"):
modifiers.ensure_only_supported()
all_nodes, report_list = get_existing_nodes_names(
utils.get_corosync_conf_facade())
if report_list:
process_library_reports(report_list)
online_nodes = utils.getCorosyncActiveNodes()
offline_nodes = []
for node in all_nodes:
if node not in online_nodes:
offline_nodes.append(node)
online_nodes.sort()
offline_nodes.sort()
print("Corosync Nodes:")
print(" ".join([" Online:"] + online_nodes))
print(" ".join([" Offline:"] + offline_nodes))
if argv[0] != "both":
sys.exit(0)
modifiers.ensure_only_supported("-f")
info_dom = utils.getClusterState()
nodes = info_dom.getElementsByTagName("nodes")
if nodes.length == 0:
utils.err("No nodes section found")
onlinenodes = []
offlinenodes = []
standbynodes = []
standbynodes_with_resources = []
maintenancenodes = []
remote_onlinenodes = []
remote_offlinenodes = []
remote_standbynodes = []
remote_standbynodes_with_resources = []
remote_maintenancenodes = []
for node in nodes[0].getElementsByTagName("node"):
node_name = node.getAttribute("name")
node_remote = node.getAttribute("type") == "remote"
if node.getAttribute("online") == "true":
if node.getAttribute("standby") == "true":
is_running_resources = (node.getAttribute("resources_running")
!= "0")
if node_remote:
if is_running_resources:
remote_standbynodes_with_resources.append(node_name)
else:
remote_standbynodes.append(node_name)
else:
if is_running_resources:
standbynodes_with_resources.append(node_name)
else:
standbynodes.append(node_name)
if node.getAttribute("maintenance") == "true":
if node_remote:
remote_maintenancenodes.append(node_name)
else:
maintenancenodes.append(node_name)
if (node.getAttribute("standby") == "false"
and node.getAttribute("maintenance") == "false"):
if node_remote:
remote_onlinenodes.append(node_name)
else:
onlinenodes.append(node_name)
else:
if node_remote:
remote_offlinenodes.append(node_name)
else:
offlinenodes.append(node_name)
print("Pacemaker Nodes:")
print(" ".join([" Online:"] + onlinenodes))
print(" ".join([" Standby:"] + standbynodes))
print(" ".join([" Standby with resource(s) running:"] +
standbynodes_with_resources))
print(" ".join([" Maintenance:"] + maintenancenodes))
print(" ".join([" Offline:"] + offlinenodes))
print("Pacemaker Remote Nodes:")
print(" ".join([" Online:"] + remote_onlinenodes))
print(" ".join([" Standby:"] + remote_standbynodes))
print(" ".join([" Standby with resource(s) running:"] +
remote_standbynodes_with_resources))
print(" ".join([" Maintenance:"] + remote_maintenancenodes))
print(" ".join([" Offline:"] + remote_offlinenodes))
def nodes_status(argv):
if len(argv) == 1 and (argv[0] == "config"):
if utils.hasCorosyncConf():
corosync_nodes = utils.get_corosync_conf_facade().get_nodes_names()
else:
corosync_nodes = []
try:
pacemaker_nodes = sorted([
node.attrs.name for node in ClusterState(
utils.getClusterStateXml()).node_section.nodes
if node.attrs.type != 'remote'
])
except LibraryError as e:
utils.process_library_reports(e.args)
print("Corosync Nodes:")
if corosync_nodes:
print(" " + " ".join(corosync_nodes))
print("Pacemaker Nodes:")
if pacemaker_nodes:
print(" " + " ".join(pacemaker_nodes))
return
if len(argv) == 1 and (argv[0] == "corosync" or argv[0] == "both"):
all_nodes = utils.get_corosync_conf_facade().get_nodes_names()
online_nodes = utils.getCorosyncActiveNodes()
offline_nodes = []
for node in all_nodes:
if node not in online_nodes:
offline_nodes.append(node)
online_nodes.sort()
offline_nodes.sort()
print("Corosync Nodes:")
print(" ".join([" Online:"] + online_nodes))
print(" ".join([" Offline:"] + offline_nodes))
if argv[0] != "both":
sys.exit(0)
info_dom = utils.getClusterState()
nodes = info_dom.getElementsByTagName("nodes")
if nodes.length == 0:
utils.err("No nodes section found")
onlinenodes = []
offlinenodes = []
standbynodes = []
maintenancenodes = []
remote_onlinenodes = []
remote_offlinenodes = []
remote_standbynodes = []
remote_maintenancenodes = []
for node in nodes[0].getElementsByTagName("node"):
node_name = node.getAttribute("name")
node_remote = node.getAttribute("type") == "remote"
if node.getAttribute("online") == "true":
if node.getAttribute("standby") == "true":
if node_remote:
remote_standbynodes.append(node_name)
else:
standbynodes.append(node_name)
elif node.getAttribute("maintenance") == "true":
if node_remote:
remote_maintenancenodes.append(node_name)
else:
maintenancenodes.append(node_name)
else:
if node_remote:
remote_onlinenodes.append(node_name)
else:
onlinenodes.append(node_name)
else:
if node_remote:
remote_offlinenodes.append(node_name)
else:
offlinenodes.append(node_name)
print("Pacemaker Nodes:")
print(" ".join([" Online:"] + onlinenodes))
print(" ".join([" Standby:"] + standbynodes))
print(" ".join([" Maintenance:"] + maintenancenodes))
print(" ".join([" Offline:"] + offlinenodes))
print("Pacemaker Remote Nodes:")
print(" ".join([" Online:"] + remote_onlinenodes))
print(" ".join([" Standby:"] + remote_standbynodes))
print(" ".join([" Maintenance:"] + remote_maintenancenodes))
print(" ".join([" Offline:"] + remote_offlinenodes))
def test_refuse_invalid_xml(self):
assert_raise_library_error(
lambda: ClusterState('invalid xml'),
(severities.ERROR, report_codes.BAD_CLUSTER_STATE_FORMAT, {}))
def test_minimal_crm_mon_is_valid(self):
ClusterState(str(self.covered_status))
def test_resources_count(self):
xml = str(self.covered_status)
self.assertEqual(0, ClusterState(xml).summary.resources.attrs.count)
