def reconfigure(_host, _idrac_host):
logger.debug("Reconfiguring host: %s" % _host)
interfaces = os.path.dirname(__file__) + "/../../conf/idrac_interfaces.yml"
if is_supported(_host.name):
badfish = Badfish(_idrac_host, conf["ipmi_username"], conf["ipmi_password"])
host_type = "foreman"
if _host.nullos:
host_type = "director"
try:
badfish.change_boot(host_type, interfaces, True)
except SystemExit as ex:
logger.debug(ex)
logger.error("Something went wrong.")
return
_host.update(build=False, realeased=True, last_build=datetime.now())
return
else:
_host.update(build=False, realeased=True, last_build=datetime.now())
return
async def move_and_rebuild(host,
new_cloud,
semaphore,
rebuild=False,
loop=None):
build_start = datetime.now()
logger.debug("Moving and rebuilding host: %s" % host)
untouchable_hosts = Config["untouchable_hosts"]
logger.debug("Untouchable hosts: %s" % untouchable_hosts)
_host_obj = Host.objects(name=host).first()
if host in untouchable_hosts:
logger.error("No way...")
return False
_target_cloud = Cloud.objects(name=new_cloud).first()
ipmi_new_pass = (f"{Config['infra_location']}@{_target_cloud.ticket}"
if _target_cloud.ticket else Config["ipmi_password"])
ipmi_set_pass = [
"user",
"set",
"password",
str(Config["ipmi_cloud_username_id"]),
ipmi_new_pass,
]
new_semaphore = asyncio.Semaphore(20)
await execute_ipmi(host, arguments=ipmi_set_pass, semaphore=new_semaphore)
ipmi_set_operator = [
"user", "priv",
str(Config["ipmi_cloud_username_id"]), "0x4"
]
await execute_ipmi(host,
arguments=ipmi_set_operator,
semaphore=new_semaphore)
badfish = None
if rebuild and _target_cloud.name != _host_obj.default_cloud.name:
if Config.pdu_management:
# TODO: pdu management
pass
try:
badfish = await badfish_factory(
"mgmt-%s" % host,
Config["ipmi_username"],
Config["ipmi_password"],
propagate=True,
)
except BadfishException:
logger.error(
f"Could not initialize Badfish. Verify ipmi credentials for mgmt-{host}."
)
return False
if is_supported(host):
try:
interfaces_path = os.path.join(
os.path.dirname(__file__),
"../../conf/idrac_interfaces.yml")
await badfish.change_boot("director", interfaces_path)
# wait 10 minutes for the boot order job to complete
await asyncio.sleep(600)
except BadfishException:
logger.error(
f"Could not set boot order via Badfish for mgmt-{host}.")
return False
try:
await badfish.set_power_state("on")
except BadfishException:
logger.error(f"Failed to power on {host}")
return False
foreman_results = []
params = [
{
"name": "operatingsystems",
"value": Config["foreman_default_os"],
"identifier": "title",
},
{
"name": "ptables",
"value": Config["foreman_default_ptable"]
},
{
"name": "media",
"value": Config["foreman_default_medium"]
},
]
foreman = Foreman(
Config["foreman_api_url"],
Config["foreman_username"],
Config["foreman_password"],
semaphore=semaphore,
loop=loop,
)
set_result = await foreman.set_host_parameter(host, "overcloud",
"true")
foreman_results.append(set_result)
put_result = await foreman.put_parameter(host, "build", 1)
foreman_results.append(put_result)
put_param_result = await foreman.put_parameters_by_name(host, params)
foreman_results.append(put_param_result)
owner_id = await foreman.get_user_id(new_cloud)
host_id = await foreman.get_host_id(host)
put_result = await foreman.put_element("hosts", host_id, "owner_id",
owner_id)
foreman_results.append(put_result)
for result in foreman_results:
if isinstance(result, Exception) or not result:
logger.error(
"There was something wrong setting Foreman host parameters."
)
return False
if is_supported(host):
try:
await badfish.boot_to_type(
"foreman",
os.path.join(os.path.dirname(__file__),
"../../conf/idrac_interfaces.yml"),
)
await badfish.reboot_server(graceful=False)
except BadfishException:
logger.error(f"Error setting PXE boot via Badfish on {host}.")
await badfish.reboot_server(graceful=False)
return False
else:
try:
asyncio.run_coroutine_threadsafe(
badfish.unmount_virtual_media(),
loop,
)
except BadfishException:
logger.warning(
f"Could not unmount virtual media for mgmt-{host}.")
try:
ipmi_pxe_persistent = [
"chassis",
"bootdev",
"pxe",
"options=persistent",
]
await execute_ipmi(host,
arguments=ipmi_pxe_persistent,
semaphore=new_semaphore)
await ipmi_reset(host, new_semaphore)
except Exception as ex:
logger.debug(ex)
logger.error(
f"There was something wrong setting PXE flag or resetting IPMI on {host}."
)
if _target_cloud.name == _host_obj.default_cloud.name:
if not badfish:
try:
badfish = await badfish_factory(
"mgmt-%s" % host,
Config["ipmi_username"],
Config["ipmi_password"],
propagate=True,
)
except BadfishException:
logger.error(
f"Could not initialize Badfish. Verify ipmi credentials for mgmt-{host}."
)
return False
await badfish.set_power_state("off")
source_cloud_schedule = Schedule.current_schedule(
cloud=_host_obj.cloud.name)
if not source_cloud_schedule:
_old_cloud_obj = Cloud.objects(name=_host_obj.cloud.name).first()
_old_cloud_obj.update(vlan=None)
schedule = Schedule.current_schedule(cloud=_target_cloud,
host=_host_obj).first()
if schedule:
schedule.update(build_start=build_start, build_end=datetime.now())
schedule.save()
logger.debug("Updating host: %s")
_host_obj.update(cloud=_target_cloud,
build=False,
last_build=datetime.now(),
validated=False)
return True
async def post_system_test(self):
password = f"{Config['infra_location']}@{self.cloud.ticket}"
foreman = Foreman(
Config["foreman_api_url"],
self.cloud.name,
password,
loop=self.loop,
)
valid_creds = await foreman.verify_credentials()
if not valid_creds:
logger.error("Unable to query Foreman for cloud: %s" % self.cloud.name)
logger.error("Verify Foreman password is correct: %s" % password)
self.report = (
self.report
+ "Unable to query Foreman for cloud: %s\n" % self.cloud.name
)
self.report = (
self.report + "Verify Foreman password is correct: %s\n" % password
)
return False
build_hosts = await foreman.get_build_hosts()
pending = []
schedules = Schedule.current_schedule(cloud=self.cloud)
if schedules:
for schedule in schedules:
if schedule.host and schedule.host.name in build_hosts:
pending.append(schedule.host.name)
if pending:
logger.info(
"The following hosts are marked for build and will now be rebooted:"
)
self.report = (
self.report + "The following hosts are marked for build:\n"
)
for host in pending:
logger.info(host)
try:
nc = Netcat(host)
healthy = await nc.health_check()
except OSError:
healthy = False
if not healthy:
logger.warning(
"Host %s didn't pass the health check. "
"Potential provisioning in process. SKIPPING." % host
)
continue
badfish = None
try:
badfish = await badfish_factory(
"mgmt-" + host,
str(Config["ipmi_username"]),
str(Config["ipmi_password"]),
)
if is_supported(host):
await badfish.boot_to_type(
"foreman",
os.path.join(
os.path.dirname(__file__),
"../../conf/idrac_interfaces.yml",
),
)
else:
await badfish.set_next_boot_pxe()
await badfish.reboot_server()
except BadfishException as ಥ﹏ಥ:
logger.debug(ಥ﹏ಥ)
if badfish:
logger.warning(
f"There was something wrong trying to boot from Foreman interface for: {host}"
)
await badfish.reboot_server()
else:
logger.error(
f"Could not initiate Badfish instance for: {host}"
)
self.report = self.report + "%s\n" % host
return False
failed = False
for host in self.hosts:
try:
badfish = await badfish_factory(
"mgmt-" + host.name,
str(Config["ipmi_cloud_username"]),
password,
)
await badfish.validate_credentials()
except BadfishException:
logger.info(f"Could not verify badfish credentials for: {host.name}")
failed = True
return not failed
def move_and_rebuild(host, old_cloud, new_cloud, rebuild=False):
logger.debug("Moving and rebuilding host: %s" % host)
untouchable_hosts = conf["untouchable_hosts"]
logger.debug("Untouchable hosts: %s" % untouchable_hosts)
_host_obj = Host.objects(name=host).first()
if not _host_obj.interfaces:
logger.error("Host has no interfaces defined.")
return False
if host in untouchable_hosts:
logger.error("No way...")
return False
_old_cloud_obj = Cloud.objects(name=old_cloud).first()
_new_cloud_obj = Cloud.objects(name=new_cloud).first()
logger.debug("Connecting to switch on: %s" %
_host_obj.interfaces[0].ip_address)
_public_vlan_obj = Vlan.objects(cloud=_new_cloud_obj).first()
for i, interface in enumerate(_host_obj.interfaces):
ssh_helper = SSHHelper(interface.ip_address, conf["junos_username"])
old_vlan_out = ssh_helper.run_cmd("show configuration interfaces %s" %
interface.switch_port)
old_vlan = old_vlan_out[0].split(";")[0].split()[1][7:]
if not old_vlan:
logger.warning(
"Warning: Could not determine the previous VLAN for %s on %s, switch %s, switchport %s"
% (host, interface.name, interface.ip_address,
interface.switch_port))
old_vlan = get_vlan(_old_cloud_obj, i)
new_vlan = get_vlan(_new_cloud_obj, i)
if _public_vlan_obj and i == len(_host_obj.interfaces) - 1:
logger.info("Setting last interface to public vlan %s." % new_vlan)
if int(old_vlan) != int(_public_vlan_obj.vlan_id):
success = juniper_convert_port_public(
interface.ip_address, interface.switch_port, str(old_vlan),
str(_public_vlan_obj.vlan_id))
if success:
logger.info("Successfully updated switch settings.")
else:
logger.error(
"There was something wrong updating switch for %s:%s" %
(host, interface.name))
return False
else:
if int(old_vlan) != int(new_vlan):
success = juniper_set_port(interface.ip_address,
interface.switch_port,
str(old_vlan), str(new_vlan))
if success:
logger.info("Successfully update switch settings.")
else:
logger.error(
"There was something wrong updating switch for %s:%s" %
(host, interface.name))
return False
if i == len(_host_obj.interfaces) - 1:
_old_vlan_obj = Vlan.objects(cloud=_old_cloud_obj).first()
if _old_vlan_obj:
_old_vlan_obj.update(cloud=None)
ssh_helper.disconnect()
ipmi_new_pass = _new_cloud_obj.ticket if _new_cloud_obj.ticket else conf[
"$ipmi_password"]
foreman = Foreman(
conf["foreman_api_url"],
conf["foreman_username"],
conf["foreman_password"],
)
foreman.remove_role(_old_cloud_obj.name, _host_obj.name)
foreman.add_role(_new_cloud_obj.name, _host_obj.name)
foreman.update_user_password(_new_cloud_obj.name, ipmi_new_pass)
ipmi_set_pass = [
"/usr/bin/ipmitool", "-I", "lanplus", "-H",
"mgmt-%s" % host, "-U", conf["ipmi_username"], "-P",
conf["ipmi_password"], "user", "set", "password",
str(conf["ipmi_cloud_username_id"]), ipmi_new_pass
]
logger.debug("ipmi_set_pass: %s" % ipmi_set_pass)
subprocess.call(ipmi_set_pass)
ipmi_set_operator = [
"/usr/bin/ipmitool", "-I", "lanplus", "-H",
"mgmt-%s" % host, "-U", conf["ipmi_username"], "-P",
conf["ipmi_password"], "user", "priv",
str(conf["ipmi_cloud_username_id"]), "0x4"
]
logger.debug("ipmi_set_operator: %s" % ipmi_set_operator)
subprocess.call(ipmi_set_operator)
if rebuild and _new_cloud_obj.name != "cloud01":
badfish = Badfish("mgmt-%s" % host, conf["ipmi_username"],
conf["ipmi_password"])
if "pdu_management" in conf and conf["pdu_management"]:
# TODO: pdu management
pass
if is_supermicro(host):
ipmi_pxe_persistent = [
"/usr/bin/ipmitool", "-I", "lanplus", "-H",
"mgmt-%s" % host, "-U", conf["ipmi_username"], "-P",
conf["ipmi_password"], "chassis", "bootdev", "pxe", "options",
"=", "persistent"
]
logger.debug("ipmi_pxe_persistent: %s" % ipmi_pxe_persistent)
subprocess.call(ipmi_pxe_persistent)
if is_supported(host):
try:
badfish.change_boot(
"director",
os.path.join(os.path.dirname(__file__),
"../../conf/idrac_interfaces.yml"))
except SystemExit as ex:
logger.debug(ex)
logger.error("Could not set boot order via Badfish.")
return False
foreman_success = foreman.remove_extraneous_interfaces(host)
foreman_success = foreman.set_host_parameter(
host, "rhel73", "false") and foreman_success
foreman_success = foreman.set_host_parameter(
host, "rhel75", "false") and foreman_success
foreman_success = foreman.set_host_parameter(
host, "overcloud", "true") and foreman_success
foreman_success = foreman.put_parameter(host, "build",
1) and foreman_success
foreman_success = foreman.put_parameter_by_name(
host, "operatingsystems", conf["foreman_default_os"],
"title") and foreman_success
foreman_success = foreman.put_parameter_by_name(
host, "ptables",
conf["foreman_default_ptable"]) and foreman_success
foreman_success = foreman.put_parameter_by_name(
host, "media", conf["foreman_default_medium"]) and foreman_success
if not foreman_success:
logger.error(
"There was something wrong setting Foreman host parameters.")
try:
badfish.set_next_boot_pxe()
badfish.reboot_server()
except SystemExit:
if is_supermicro(host):
logger.warning("Badfish not yet supported on Supermicro: %s." %
host)
else:
logger.exception(
"There was something wrong setting next PXE boot via Badfish."
)
return False
logger.debug("Updating host: %s")
_host_obj.update(cloud=_new_cloud_obj,
build=False,
last_build=datetime.now())
else:
logger.debug("Updating host: %s")
_host_obj.update(cloud=_new_cloud_obj,
build=False,
last_build=datetime.now())
async def move_and_rebuild(host,
new_cloud,
semaphore,
rebuild=False,
loop=None):
build_start = datetime.now()
logger.debug("Moving and rebuilding host: %s" % host)
untouchable_hosts = conf["untouchable_hosts"]
logger.debug("Untouchable hosts: %s" % untouchable_hosts)
_host_obj = Host.objects(name=host).first()
if host in untouchable_hosts:
logger.error("No way...")
return False
_new_cloud_obj = Cloud.objects(name=new_cloud).first()
ipmi_new_pass = (f"{conf['infra_location']}@{_new_cloud_obj.ticket}"
if _new_cloud_obj.ticket else conf["ipmi_password"])
ipmi_set_pass = [
"user",
"set",
"password",
str(conf["ipmi_cloud_username_id"]),
ipmi_new_pass,
]
new_semaphore = asyncio.Semaphore(20)
await execute_ipmi(host, arguments=ipmi_set_pass, semaphore=new_semaphore)
ipmi_set_operator = [
"user", "priv",
str(conf["ipmi_cloud_username_id"]), "0x4"
]
await execute_ipmi(host,
arguments=ipmi_set_operator,
semaphore=new_semaphore)
if rebuild and _new_cloud_obj.name != _host_obj.default_cloud.name:
if "pdu_management" in conf and conf["pdu_management"]:
# TODO: pdu management
pass
if is_supported(host):
try:
badfish = await badfish_factory(
"mgmt-%s" % host,
conf["ipmi_username"],
conf["ipmi_password"],
propagate=True,
)
except BadfishException:
logger.error(
f"Could not initialize Badfish. Verify ipmi credentials for mgmt-{host}."
)
return False
try:
changed_boot_order = asyncio.run_coroutine_threadsafe(
badfish.change_boot(
"director",
os.path.join(os.path.dirname(__file__),
"../../conf/idrac_interfaces.yml"),
),
loop,
)
if changed_boot_order:
await badfish.reboot_server(graceful=False)
except BadfishException:
logger.error(
f"Could not set boot order via Badfish for mgmt-{host}.")
return False
foreman_results = []
params = [
{
"name": "operatingsystems",
"value": conf["foreman_default_os"],
"identifier": "title",
},
{
"name": "ptables",
"value": conf["foreman_default_ptable"]
},
{
"name": "media",
"value": conf["foreman_default_medium"]
},
]
foreman = Foreman(
conf["foreman_api_url"],
conf["foreman_username"],
conf["foreman_password"],
semaphore=semaphore,
loop=loop,
)
set_result = await foreman.set_host_parameter(host, "overcloud",
"true")
foreman_results.append(set_result)
put_result = await foreman.put_parameter(host, "build", 1)
foreman_results.append(put_result)
put_param_result = await foreman.put_parameters_by_name(host, params)
foreman_results.append(put_param_result)
owner_id = await foreman.get_user_id(new_cloud)
host_id = await foreman.get_host_id(host)
put_result = await foreman.put_element("hosts", host_id, "owner_id",
owner_id)
foreman_results.append(put_result)
for result in foreman_results:
if isinstance(result, Exception) or not result:
logger.error(
"There was something wrong setting Foreman host parameters."
)
return False
healthy = False
for i in range(RETRIES):
nc = Netcat(_host_obj.name)
if nc.health_check():
healthy = True
nc.close()
break
nc.close()
if not healthy:
logger.error("Failed to recover host after changing boot order.")
return False
if is_supported(host):
try:
await badfish.boot_to_type(
"foreman",
os.path.join(os.path.dirname(__file__),
"../../conf/idrac_interfaces.yml"),
)
await badfish.reboot_server(graceful=False)
except BadfishException:
logger.error(f"Error setting PXE boot via Badfish on {host}.")
await badfish.reboot_server(graceful=False)
return False
else:
try:
ipmi_pxe_persistent = [
"chassis",
"bootdev",
"pxe",
"options",
"=",
"persistent",
]
await execute_ipmi(host,
arguments=ipmi_pxe_persistent,
semaphore=new_semaphore)
await ipmi_reset(host, new_semaphore)
except Exception as ex:
logger.debug(ex)
logger.error(
f"There was something wrong setting PXE flag or resetting IPMI on {host}."
)
schedule = Schedule.current_schedule(cloud=_new_cloud_obj,
host=_host_obj).first()
if schedule:
schedule.update(build_start=build_start, build_end=datetime.now())
schedule.save()
logger.debug("Updating host: %s")
_host_obj.update(cloud=_new_cloud_obj,
build=False,
last_build=datetime.now(),
validated=False)
return True