def post_system_test(self):
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
password = f"{conf['infra_location']}@{self.cloud.ticket}"
foreman = Foreman(
conf["foreman_api_url"], self.cloud.name, password, loop=loop,
)
if not loop.run_until_complete(foreman.verify_credentials()):
logger.error("Unable to query Foreman for cloud: %s" % self.cloud.name)
logger.error("Verify Foreman password is correct: %s" % password)
self.report = (
self.report
+ "Unable to query Foreman for cloud: %s\n" % self.cloud.name
)
self.report = (
self.report + "Verify Foreman password is correct: %s\n" % password
)
return False
build_hosts = loop.run_until_complete(foreman.get_build_hosts())
pending = []
schedules = Schedule.current_schedule(cloud=self.cloud)
if schedules:
for schedule in schedules:
if schedule.host and schedule.host.name in build_hosts:
pending.append(schedule.host.name)
if pending:
logger.info(
"The following hosts are marked for build and will now be rebooted:"
)
self.report = (
self.report + "The following hosts are marked for build:\n"
)
for host in pending:
logger.info(host)
nc = Netcat(host)
if not nc.health_check():
logger.warning(
"Host %s didn't pass the health check. "
"Potential provisioning in process. SKIPPING." % host
)
continue
nc.close()
badfish = None
try:
badfish = loop.run_until_complete(
badfish_factory(
"mgmt-" + host,
str(conf["ipmi_username"]),
str(conf["ipmi_password"]),
)
)
if is_supported(host):
loop.run_until_complete(
badfish.boot_to_type(
"foreman",
os.path.join(
os.path.dirname(__file__),
"../../conf/idrac_interfaces.yml",
),
)
)
else:
loop.run_until_complete(badfish.set_next_boot_pxe())
loop.run_until_complete(badfish.reboot_server())
except BadfishException as ಥ﹏ಥ:
logger.debug(ಥ﹏ಥ)
if badfish:
logger.warning(f"There was something wrong trying to boot from Foreman interface for: {host}")
loop.run_until_complete(badfish.reboot_server())
else:
logger.error(f"Could not initiate Badfish instance for: {host}")
self.report = self.report + "%s\n" % host
return False
failed = False
for host in self.hosts:
try:
badfish = loop.run_until_complete(
badfish_factory(
"mgmt-" + host.name, str(conf["ipmi_cloud_username"]), password,
)
)
loop.run_until_complete(badfish.validate_credentials())
except BadfishException:
logger.info(f"Could not verify badfish credentials for: {host.name}")
failed = True
return not failed
def post_network_test(self):
test_host = self.hosts[0]
hosts_down = []
for host in self.hosts:
nc = Netcat(host.name)
if not nc.health_check():
hosts_down.append(host.name)
nc.close()
if len(host.interfaces) > len(test_host.interfaces):
test_host = host
if hosts_down:
logger.error("The following hosts appear to be down or with no ssh connection:")
for i in hosts_down:
logger.error(i)
return False
try:
ssh_helper = SSHHelper(test_host.name)
except (SSHException, NoValidConnectionsError, socket.timeout) as ex:
logger.debug(ex)
logger.error(
"Could not establish connection with host: %s." % test_host.name
)
self.report = (
self.report
+ "Could not establish connection with host: %s.\n" % test_host.name
)
return False
host_list = " ".join([host.name for host in self.hosts])
if type(ssh_helper.run_cmd("fping -u %s" % host_list)) != list:
return False
for i, interface in enumerate(INTERFACES.keys()):
new_ips = []
host_ips = [
{"ip": socket.gethostbyname(host.name), "host": host}
for host in self.hosts
if interface in [_interface.name for _interface in host.interfaces]
]
for host in host_ips:
_host_obj = host["host"]
_interfaces = INTERFACES[interface]
last_nic = i == len(_host_obj.interfaces) - 1
if last_nic and self.cloud.vlan:
_interfaces = _interfaces[:1]
for value in _interfaces:
ip_apart = host["ip"].split(".")
octets = value.split(".")
ip_apart[0] = octets[0]
ip_apart[1] = octets[1]
new_ips.append(".".join(ip_apart))
if new_ips:
if type(ssh_helper.run_cmd("fping -u %s" % " ".join(new_ips))) != list:
return False
ssh_helper.disconnect()
return True