def post_system_test(self): loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) password = f"{conf['infra_location']}@{self.cloud.ticket}" foreman = Foreman( conf["foreman_api_url"], self.cloud.name, password, loop=loop, ) if not loop.run_until_complete(foreman.verify_credentials()): logger.error("Unable to query Foreman for cloud: %s" % self.cloud.name) logger.error("Verify Foreman password is correct: %s" % password) self.report = ( self.report + "Unable to query Foreman for cloud: %s\n" % self.cloud.name ) self.report = ( self.report + "Verify Foreman password is correct: %s\n" % password ) return False build_hosts = loop.run_until_complete(foreman.get_build_hosts()) pending = [] schedules = Schedule.current_schedule(cloud=self.cloud) if schedules: for schedule in schedules: if schedule.host and schedule.host.name in build_hosts: pending.append(schedule.host.name) if pending: logger.info( "The following hosts are marked for build and will now be rebooted:" ) self.report = ( self.report + "The following hosts are marked for build:\n" ) for host in pending: logger.info(host) nc = Netcat(host) if not nc.health_check(): logger.warning( "Host %s didn't pass the health check. " "Potential provisioning in process. SKIPPING." % host ) continue nc.close() badfish = None try: badfish = loop.run_until_complete( badfish_factory( "mgmt-" + host, str(conf["ipmi_username"]), str(conf["ipmi_password"]), ) ) if is_supported(host): loop.run_until_complete( badfish.boot_to_type( "foreman", os.path.join( os.path.dirname(__file__), "../../conf/idrac_interfaces.yml", ), ) ) else: loop.run_until_complete(badfish.set_next_boot_pxe()) loop.run_until_complete(badfish.reboot_server()) except BadfishException as ಥ﹏ಥ: logger.debug(ಥ﹏ಥ) if badfish: logger.warning(f"There was something wrong trying to boot from Foreman interface for: {host}") loop.run_until_complete(badfish.reboot_server()) else: logger.error(f"Could not initiate Badfish instance for: {host}") self.report = self.report + "%s\n" % host return False failed = False for host in self.hosts: try: badfish = loop.run_until_complete( badfish_factory( "mgmt-" + host.name, str(conf["ipmi_cloud_username"]), password, ) ) loop.run_until_complete(badfish.validate_credentials()) except BadfishException: logger.info(f"Could not verify badfish credentials for: {host.name}") failed = True return not failed
def post_network_test(self): test_host = self.hosts[0] hosts_down = [] for host in self.hosts: nc = Netcat(host.name) if not nc.health_check(): hosts_down.append(host.name) nc.close() if len(host.interfaces) > len(test_host.interfaces): test_host = host if hosts_down: logger.error("The following hosts appear to be down or with no ssh connection:") for i in hosts_down: logger.error(i) return False try: ssh_helper = SSHHelper(test_host.name) except (SSHException, NoValidConnectionsError, socket.timeout) as ex: logger.debug(ex) logger.error( "Could not establish connection with host: %s." % test_host.name ) self.report = ( self.report + "Could not establish connection with host: %s.\n" % test_host.name ) return False host_list = " ".join([host.name for host in self.hosts]) if type(ssh_helper.run_cmd("fping -u %s" % host_list)) != list: return False for i, interface in enumerate(INTERFACES.keys()): new_ips = [] host_ips = [ {"ip": socket.gethostbyname(host.name), "host": host} for host in self.hosts if interface in [_interface.name for _interface in host.interfaces] ] for host in host_ips: _host_obj = host["host"] _interfaces = INTERFACES[interface] last_nic = i == len(_host_obj.interfaces) - 1 if last_nic and self.cloud.vlan: _interfaces = _interfaces[:1] for value in _interfaces: ip_apart = host["ip"].split(".") octets = value.split(".") ip_apart[0] = octets[0] ip_apart[1] = octets[1] new_ips.append(".".join(ip_apart)) if new_ips: if type(ssh_helper.run_cmd("fping -u %s" % " ".join(new_ips))) != list: return False ssh_helper.disconnect() return True