def introspect_nodes(in_band, ironic_client, nodes, transition_nodes=True):
# Check to see if provisioning_mac has been set on all the nodes
bad_nodes = []
for node in nodes:
if "provisioning_mac" not in node.properties:
bad_nodes.append(node)
if bad_nodes:
ips = [CredentialHelper.get_drac_ip(node) for node in bad_nodes]
fail_msg = "\n".join(ips)
logger.error("Run config_idrac.py on {} before running "
"introspection".format(fail_msg))
if transition_nodes:
nodes = transition_to_state(ironic_client, nodes, 'manage',
'manageable')
threads = []
bad_nodes = []
inspecting = []
for node in nodes:
use_oob_introspection = is_introspection_oob(in_band, node, logger)
introspection_type = "out-of-band"
if not use_oob_introspection:
introspection_type = "in-band"
logger.info("Starting {} introspection on node "
"{} ({})".format(introspection_type,
CredentialHelper.get_drac_ip(node),
node.uuid))
if not use_oob_introspection:
thread = ThreadWithExHandling(logger,
object_identity=node,
target=ib_introspect,
args=(node, ))
threads.append(thread)
thread.start()
else:
# Note that this CLI command is asynchronous, so it will return
# immediately and Ironic conductor will begin OOB introspection in
# the background
if os.system("openstack baremetal node inspect " + node.uuid) != 0:
bad_nodes.append(node)
else:
# Wait until the node goes into the inspecting state before
# continuing. This is necessary because OOB introspection
# completes very quickly on some nodes. The busy wait is
# intentional.
node = ironic_client.node.get(node.uuid)
logger.debug(
CredentialHelper.get_drac_ip(node) + " provision_state=" +
node.provision_state)
while node.provision_state != INSPECTING:
node = ironic_client.node.get(node.uuid)
logger.debug(
CredentialHelper.get_drac_ip(node) +
" provision_state=" + node.provision_state)
logger.debug(
CredentialHelper.get_drac_ip(node) +
" adding to inspecting")
inspecting.append(node)
# Note that the in-band introspection CLI command is synchronous. By the
# time all of the threads have completed, the nodes are all out of the
# "inspecting" state.
for thread in threads:
thread.join()
for thread in threads:
if thread.ex is not None:
bad_nodes.append(thread.object_identity)
if bad_nodes:
ips = [
"{} ({})".format(CredentialHelper.get_drac_ip(node), node.uuid)
for node in bad_nodes
]
raise RuntimeError("Failed to introspect {}".format(", ".join(ips)))
if use_oob_introspection:
# Wait for the nodes to transition out of "inspecting"
# Allow 10 minutes to complete OOB introspection
logger.info("Waiting for introspection to complete...")
introspection_timeout = 600
while introspection_timeout > 0:
inspecting = refresh_nodes(ironic_client, inspecting)
for node in inspecting:
if node.provision_state != INSPECTING:
inspecting.remove(node)
if len(inspecting) == 0:
logger.info("Introspection finished")
break
else:
logger.debug("Still inspecting=" + ", ".join([
"{} ({})".format(CredentialHelper.get_drac_ip(node),
node.uuid) for node in inspecting
]))
introspection_timeout -= 1
if introspection_timeout > 0:
sleep(1)
if introspection_timeout == 0:
error_msg = "Introspection failed."
if len(inspecting) > 0:
inspecting_ips = [
"{} ({})".format(CredentialHelper.get_drac_ip(node),
node.uuid) for node in inspecting
]
error_msg += " The following nodes never exited the " \
"{} state: {}.".format(INSPECTING,
", ".join(inspecting_ips))
raise RuntimeError(error_msg)
if not use_oob_introspection:
# The PERC H740P RAID controller only makes virtual disks visible to
# the host OS. Physical disks are not visible with this controller
# because it does not support pass-through mode. This results in
# local_gb not being set during IB introspection, which causes
# problems further along in the flow.
# Check to see if all nodes have local_gb defined, and if not run OOB
# introspection to discover local_gb.
nodes = refresh_nodes(ironic_client, nodes)
bad_nodes = []
for node in nodes:
if 'local_gb' not in node.properties:
bad_nodes.append(node)
if bad_nodes:
ips = [CredentialHelper.get_drac_ip(node) for node in bad_nodes]
fail_msg = "\n".join(ips)
logger.info("local_gb was not discovered on: {}".format(fail_msg))
logger.info("Running OOB introspection to populate it.")
introspect_nodes(False,
ironic_client,
bad_nodes,
transition_nodes=False)
if transition_nodes:
nodes = transition_to_state(ironic_client, nodes, 'provide',
'available')
if use_oob_introspection:
# FIXME: Remove this hack when OOB introspection is fixed
for node in nodes:
delete_non_pxe_ports(ironic_client, node)
def main():
args = parse_arguments()
LoggingHelper.configure_logging(args.logging_level)
try:
json_file = os.path.expanduser(args.node_definition)
with open(json_file, 'r') as instackenv_json:
instackenv = json.load(instackenv_json)
except (IOError, ValueError):
LOG.exception("Failed to load node definition file {}".format(
args.node_definition))
sys.exit(1)
json_config = None
if args.json_config is not None:
try:
json_config = json.loads(args.json_config)
except:
LOG.exception("Failed to parse json_config data")
sys.exit(1)
try:
model_properties = Utils.get_model_properties(args.model_properties)
# Configure all the nodes
if "nodes" not in instackenv:
raise ValueError("{} must contain an array of "
"\"nodes\"".format(args.node_definition))
instack_lock = threading.Lock()
threads = []
for node in instackenv["nodes"]:
pxe_nic = None
password = None
if json_config is not None:
node_config = None
if node["pm_addr"] in json_config.keys():
node_config = json_config[node["pm_addr"]]
elif node["service_tag"] in json_config.keys():
node_config = json_config[node["service_tag"]]
if node_config is not None:
if "pxe_nic" in node_config.keys():
pxe_nic = node_config["pxe_nic"]
if "password" in node_config.keys():
password = node_config["password"]
thread = ThreadWithExHandling(LOG,
target=config_idrac.config_idrac,
args=(instack_lock,
node["pm_addr"],
args.node_definition,
model_properties,
pxe_nic,
password))
threads.append(thread)
thread.start()
for thread in threads:
thread.join()
failed_threads = 0
for thread in threads:
if thread.ex is not None:
failed_threads += 1
if failed_threads == 0:
LOG.info("Successfully configured all iDRACs")
else:
LOG.info("Failed to configure {} out of {} iDRACs".format(
failed_threads, len(threads)))
sys.exit(1)
except ValueError as ex:
LOG.error(ex)
sys.exit(1)
except Exception as ex:
LOG.exception(ex.message)
sys.exit(1)
def introspect_nodes(in_band, ironic_client, nodes,
transition_nodes=True):
# Check to see if provisioning_mac has been set on all the nodes
bad_nodes = []
for node in nodes:
if "provisioning_mac" not in node.properties:
bad_nodes.append(node)
if bad_nodes:
ips = [CredentialHelper.get_drac_ip(node) for node in bad_nodes]
fail_msg = "\n".join(ips)
logger.error("Run config_idrac.py on {} before running "
"introspection".format(fail_msg))
if transition_nodes:
nodes = transition_to_state(ironic_client, nodes,
'manage', 'manageable')
threads = []
bad_nodes = []
for node in nodes:
use_oob_introspection = is_introspection_oob(in_band, node, logger)
introspection_type = "out-of-band"
if not use_oob_introspection:
introspection_type = "in-band"
logger.info("Starting {} introspection on node "
"{} ({})".format(introspection_type,
CredentialHelper.get_drac_ip(node),
node.uuid))
if not use_oob_introspection:
thread = ThreadWithExHandling(logger,
object_identity=node,
target=ib_introspect,
args=(node,))
threads.append(thread)
thread.start()
else:
if os.system("openstack baremetal node inspect " + node.uuid) != 0:
bad_nodes.append(node)
for thread in threads:
thread.join()
for thread in threads:
if thread.ex is not None:
bad_nodes.append(thread.object_identity)
if bad_nodes:
ips = ["{} ({})".format(CredentialHelper.get_drac_ip(node),
node.uuid) for node in bad_nodes]
raise RuntimeError("Failed to introspect {}".format(", ".join(ips)))
if not use_oob_introspection:
# The PERC H740P RAID controller only makes virtual disks visible to
# the host OS. Physical disks are not visible with this controller
# because it does not support pass-through mode. This results in
# local_gb not being set during IB introspection, which causes
# problems further along in the flow.
# Check to see if all nodes have local_gb defined, and if not run OOB
# introspection to discover local_gb.
nodes = refresh_nodes(ironic_client, nodes)
bad_nodes = []
for node in nodes:
if 'local_gb' not in node.properties:
bad_nodes.append(node)
if bad_nodes:
ips = [CredentialHelper.get_drac_ip(node) for node in bad_nodes]
fail_msg = "\n".join(ips)
logger.info("local_gb was not discovered on: {}".format(fail_msg))
logger.info("Running OOB introspection to populate it.")
introspect_nodes(False, ironic_client, bad_nodes,
transition_nodes=False)
if transition_nodes:
nodes = transition_to_state(ironic_client, nodes,
'provide', 'available')
if use_oob_introspection:
# FIXME: Remove this hack when OOB introspection is fixed
for node in nodes:
delete_non_pxe_ports(ironic_client, node)