def transition_to_state(ironic_client, nodes, transition,
target_provision_state):
# Grab a new copy of the nodes to get the current provision_state
nodes = refresh_nodes(ironic_client, nodes)
for node in nodes:
if node.provision_state == target_provision_state:
logger.debug("Node {} ({}) is already in the {} state".format(
CredentialHelper.get_drac_ip(node), node.uuid,
target_provision_state))
else:
logger.debug("Transitioning node {} ({}) into the {} "
"state".format(CredentialHelper.get_drac_ip(node),
node.uuid, target_provision_state))
ironic_client.node.set_provision_state(node.uuid, transition)
while True:
all_nodes_transitioned = True
nodes = refresh_nodes(ironic_client, nodes)
for node in nodes:
if node.provision_state != target_provision_state:
logger.debug("Node {} ({}) is still in the {} "
"state".format(CredentialHelper.get_drac_ip(node),
node.uuid, node.provision_state))
all_nodes_transitioned = False
if all_nodes_transitioned:
break
else:
sleep(1)
return nodes
def ib_introspect(node):
command = "openstack overcloud node introspect " + node.uuid
if os.system(command) == 0:
logger.info("In-Band introspection succeeded on node {} ({})".format(
CredentialHelper.get_drac_ip(node), node.uuid))
else:
raise RuntimeError("In-Band introspection failed on node "
"{} ({})".format(
CredentialHelper.get_drac_ip(node), node.uuid))
def ib_introspect(node):
# Note that the in-band introspection CLI command is synchronous. The node
# will be out of the "inspecting" state when this command returns.
command = "openstack overcloud node introspect " + node.uuid
if os.system(command) == 0:
logger.info("In-Band introspection succeeded on node {} ({})".format(
CredentialHelper.get_drac_ip(node), node.uuid))
else:
raise RuntimeError("In-Band introspection failed on node "
"{} ({})".format(CredentialHelper.get_drac_ip(node),
node.uuid))
def main():
os_auth_url, os_tenant_name, os_username, os_password, \
os_user_domain_name, os_project_domain_name = \
CredentialHelper.get_undercloud_creds()
auth_url = os_auth_url + "v3"
kwargs = {'os_username': os_username,
'os_password': os_password,
'os_auth_url': os_auth_url,
'os_tenant_name': os_tenant_name,
'os_user_domain_name': os_user_domain_name,
'os_project_domain_name': os_project_domain_name}
ironic = ironicclient.client.get_client(1, **kwargs)
nodes = ironic.node.list(detail=True)
auth = v3.Password(
auth_url=auth_url,
username=os_username,
password=os_password,
project_name=os_tenant_name,
user_domain_name=os_user_domain_name,
project_domain_name=os_project_domain_name
)
sess = session.Session(auth=auth)
nova = novaclient.Client('2', session=sess)
# Slightly odd syntax for declaring 'banner' reduces the line length
banner = (
"+-----------------+---------------------------+-----------------+"
)
nodeinfo = "| {:<15} | {:<25} | {:<15} |"
print banner
print nodeinfo.format('iDRAC Addr', 'Node Name', 'Provision Addr')
print banner
# Display the list ordered by the iDRAC address
for n in sorted(nodes, key=lambda x: CredentialHelper.get_drac_ip(x)):
idrac_addr = CredentialHelper.get_drac_ip(n)
if 'display_name' in n.instance_info:
node_name = n.instance_info['display_name']
else:
node_name = 'None'
prov_addr = 'None'
if n.instance_uuid:
nova_ips = nova.servers.ips(n.instance_uuid)
if nova_ips and 'ctlplane' in nova_ips:
prov_addr = nova_ips['ctlplane'][0]['addr']
print nodeinfo.format(idrac_addr, node_name, prov_addr)
print banner
def delete_non_pxe_ports(ironic_client, node):
ip = CredentialHelper.get_drac_ip(node)
logger.info("Deleting all non-PXE ports from node {} ({})...".format(
ip, node.uuid))
for port in ironic_client.node.list_ports(node.uuid):
if port.address.lower() != \
node.properties["provisioning_mac"].lower():
logger.info("Deleting port {} ({}) {}".format(
ip, node.uuid, port.address.lower()))
ironic_client.port.delete(port.uuid)
def introspect_nodes(in_band, ironic_client, nodes, transition_nodes=True):
# Check to see if provisioning_mac has been set on all the nodes
bad_nodes = []
for node in nodes:
if "provisioning_mac" not in node.properties:
bad_nodes.append(node)
if bad_nodes:
ips = [CredentialHelper.get_drac_ip(node) for node in bad_nodes]
fail_msg = "\n".join(ips)
logger.error("Run config_idrac.py on {} before running "
"introspection".format(fail_msg))
if transition_nodes:
nodes = transition_to_state(ironic_client, nodes, 'manage',
'manageable')
threads = []
bad_nodes = []
inspecting = []
for node in nodes:
use_oob_introspection = is_introspection_oob(in_band, node, logger)
introspection_type = "out-of-band"
if not use_oob_introspection:
introspection_type = "in-band"
logger.info("Starting {} introspection on node "
"{} ({})".format(introspection_type,
CredentialHelper.get_drac_ip(node),
node.uuid))
if not use_oob_introspection:
thread = ThreadWithExHandling(logger,
object_identity=node,
target=ib_introspect,
args=(node, ))
threads.append(thread)
thread.start()
else:
# Note that this CLI command is asynchronous, so it will return
# immediately and Ironic conductor will begin OOB introspection in
# the background
if os.system("openstack baremetal node inspect " + node.uuid) != 0:
bad_nodes.append(node)
else:
# Wait until the node goes into the inspecting state before
# continuing. This is necessary because OOB introspection
# completes very quickly on some nodes. The busy wait is
# intentional.
node = ironic_client.node.get(node.uuid)
logger.debug(
CredentialHelper.get_drac_ip(node) + " provision_state=" +
node.provision_state)
while node.provision_state != INSPECTING:
node = ironic_client.node.get(node.uuid)
logger.debug(
CredentialHelper.get_drac_ip(node) +
" provision_state=" + node.provision_state)
logger.debug(
CredentialHelper.get_drac_ip(node) +
" adding to inspecting")
inspecting.append(node)
# Note that the in-band introspection CLI command is synchronous. By the
# time all of the threads have completed, the nodes are all out of the
# "inspecting" state.
for thread in threads:
thread.join()
for thread in threads:
if thread.ex is not None:
bad_nodes.append(thread.object_identity)
if bad_nodes:
ips = [
"{} ({})".format(CredentialHelper.get_drac_ip(node), node.uuid)
for node in bad_nodes
]
raise RuntimeError("Failed to introspect {}".format(", ".join(ips)))
if use_oob_introspection:
# Wait for the nodes to transition out of "inspecting"
# Allow 10 minutes to complete OOB introspection
logger.info("Waiting for introspection to complete...")
introspection_timeout = 600
while introspection_timeout > 0:
inspecting = refresh_nodes(ironic_client, inspecting)
for node in inspecting:
if node.provision_state != INSPECTING:
inspecting.remove(node)
if len(inspecting) == 0:
logger.info("Introspection finished")
break
else:
logger.debug("Still inspecting=" + ", ".join([
"{} ({})".format(CredentialHelper.get_drac_ip(node),
node.uuid) for node in inspecting
]))
introspection_timeout -= 1
if introspection_timeout > 0:
sleep(1)
if introspection_timeout == 0:
error_msg = "Introspection failed."
if len(inspecting) > 0:
inspecting_ips = [
"{} ({})".format(CredentialHelper.get_drac_ip(node),
node.uuid) for node in inspecting
]
error_msg += " The following nodes never exited the " \
"{} state: {}.".format(INSPECTING,
", ".join(inspecting_ips))
raise RuntimeError(error_msg)
if not use_oob_introspection:
# The PERC H740P RAID controller only makes virtual disks visible to
# the host OS. Physical disks are not visible with this controller
# because it does not support pass-through mode. This results in
# local_gb not being set during IB introspection, which causes
# problems further along in the flow.
# Check to see if all nodes have local_gb defined, and if not run OOB
# introspection to discover local_gb.
nodes = refresh_nodes(ironic_client, nodes)
bad_nodes = []
for node in nodes:
if 'local_gb' not in node.properties:
bad_nodes.append(node)
if bad_nodes:
ips = [CredentialHelper.get_drac_ip(node) for node in bad_nodes]
fail_msg = "\n".join(ips)
logger.info("local_gb was not discovered on: {}".format(fail_msg))
logger.info("Running OOB introspection to populate it.")
introspect_nodes(False,
ironic_client,
bad_nodes,
transition_nodes=False)
if transition_nodes:
nodes = transition_to_state(ironic_client, nodes, 'provide',
'available')
if use_oob_introspection:
# FIXME: Remove this hack when OOB introspection is fixed
for node in nodes:
delete_non_pxe_ports(ironic_client, node)
def introspect_nodes(in_band, ironic_client, nodes,
transition_nodes=True):
# Check to see if provisioning_mac has been set on all the nodes
bad_nodes = []
for node in nodes:
if "provisioning_mac" not in node.properties:
bad_nodes.append(node)
if bad_nodes:
ips = [CredentialHelper.get_drac_ip(node) for node in bad_nodes]
fail_msg = "\n".join(ips)
logger.error("Run config_idrac.py on {} before running "
"introspection".format(fail_msg))
if transition_nodes:
nodes = transition_to_state(ironic_client, nodes,
'manage', 'manageable')
threads = []
bad_nodes = []
for node in nodes:
use_oob_introspection = is_introspection_oob(in_band, node, logger)
introspection_type = "out-of-band"
if not use_oob_introspection:
introspection_type = "in-band"
logger.info("Starting {} introspection on node "
"{} ({})".format(introspection_type,
CredentialHelper.get_drac_ip(node),
node.uuid))
if not use_oob_introspection:
thread = ThreadWithExHandling(logger,
object_identity=node,
target=ib_introspect,
args=(node,))
threads.append(thread)
thread.start()
else:
if os.system("openstack baremetal node inspect " + node.uuid) != 0:
bad_nodes.append(node)
for thread in threads:
thread.join()
for thread in threads:
if thread.ex is not None:
bad_nodes.append(thread.object_identity)
if bad_nodes:
ips = ["{} ({})".format(CredentialHelper.get_drac_ip(node),
node.uuid) for node in bad_nodes]
raise RuntimeError("Failed to introspect {}".format(", ".join(ips)))
if not use_oob_introspection:
# The PERC H740P RAID controller only makes virtual disks visible to
# the host OS. Physical disks are not visible with this controller
# because it does not support pass-through mode. This results in
# local_gb not being set during IB introspection, which causes
# problems further along in the flow.
# Check to see if all nodes have local_gb defined, and if not run OOB
# introspection to discover local_gb.
nodes = refresh_nodes(ironic_client, nodes)
bad_nodes = []
for node in nodes:
if 'local_gb' not in node.properties:
bad_nodes.append(node)
if bad_nodes:
ips = [CredentialHelper.get_drac_ip(node) for node in bad_nodes]
fail_msg = "\n".join(ips)
logger.info("local_gb was not discovered on: {}".format(fail_msg))
logger.info("Running OOB introspection to populate it.")
introspect_nodes(False, ironic_client, bad_nodes,
transition_nodes=False)
if transition_nodes:
nodes = transition_to_state(ironic_client, nodes,
'provide', 'available')
if use_oob_introspection:
# FIXME: Remove this hack when OOB introspection is fixed
for node in nodes:
delete_non_pxe_ports(ironic_client, node)
