def post_network_test(_cloud): _hosts = Host.objects(cloud=_cloud) test_host = _hosts[0] try: ssh_helper = SSHHelper(test_host.name) except Exception: logger.exception("Could not establish connection with host: %s." % test_host.name) return False host_list = " ".join([host.name for host in _hosts]) if type(ssh_helper.run_cmd("fping -u %s" % host_list)) != list: return False for interface in test_host.interfaces: new_ips = [] host_ips = [ socket.gethostbyname(host.name) for host in _hosts if interface.name in [_interface.name for _interface in host.interfaces] ] for ip in host_ips: for value in INTERFACES[interface.name]: ip_apart = ip.split(".") octets = value.split(".") ip_apart[0] = octets[0] ip_apart[1] = octets[1] new_ips.append(".".join(ip_apart)) if type(ssh_helper.run_cmd("fping -u %s" % " ".join(new_ips))) != list: return False ssh_helper.disconnect() return True
def verify(cloud): _cloud_obj = Cloud.objects(name=cloud).first() logger.info(f"Cloud qinq: {_cloud_obj.qinq}") if not _cloud_obj: logger.error(f"Cloud not found.") return if args.all: hosts = Host.objects(cloud=_cloud_obj) else: hosts = [Host.objects(cloud=_cloud_obj).first()] for host in hosts: if args.all: logger.info(f"{host.name}:") if host and host.interfaces: interfaces = sorted(host.interfaces, key=lambda k: k["name"]) for i, interface in enumerate(interfaces): ssh_helper = SSHHelper(interface.switch_ip, Config["junos_username"]) try: _, vlan_member_out = ssh_helper.run_cmd( "show configuration vlans | display set | match %s.0" % interface.switch_port) vlan_member = vlan_member_out[0].split()[2][4:].strip(",") except IndexError: logger.warning( "Could not determine the previous VLAN member for %s, switch %s, switch port %s " % ( interface.name, interface.switch_ip, interface.switch_port, )) vlan_member = 0 ssh_helper.disconnect() logger.info( f"Interface em{i+1} appears to be a member of VLAN {vlan_member}", ) else: logger.error( f"The cloud has no hosts or the host has no interfaces defined" )
def switch_config(host, old_cloud, new_cloud): _host_obj = Host.objects(name=host).first() _old_cloud_obj = Cloud.objects(name=old_cloud).first() _new_cloud_obj = Cloud.objects(name=new_cloud).first() if not _host_obj.interfaces: logger.error("Host has no interfaces defined.") return False logger.debug("Connecting to switch on: %s" % _host_obj.interfaces[0].switch_ip) switch_ip = None ssh_helper = None interfaces = sorted(_host_obj.interfaces, key=lambda k: k["name"]) for i, interface in enumerate(interfaces): last_nic = i == len(_host_obj.interfaces) - 1 if not switch_ip: switch_ip = interface.switch_ip try: ssh_helper = SSHHelper(switch_ip, Config["junos_username"]) except SSHHelperException: logger.error(f"Failed to connect to switch: {switch_ip}") return False else: if switch_ip != interface.switch_ip: ssh_helper.disconnect() switch_ip = interface.switch_ip ssh_helper = SSHHelper(switch_ip, Config["junos_username"]) result, old_vlan_out = ssh_helper.run_cmd( "show configuration interfaces %s" % interface.switch_port) old_vlan = None if result and old_vlan_out: old_vlan = old_vlan_out[0].split(";")[0].split()[1][7:] if not old_vlan: if not _new_cloud_obj.vlan and not last_nic: logger.warning( "Warning: Could not determine the previous VLAN for %s on %s, switch %s, switchport %s" % ( host, interface.name, interface.switch_ip, interface.switch_port, )) old_vlan = get_vlan(_old_cloud_obj, i) new_vlan = get_vlan(_new_cloud_obj, i) if _new_cloud_obj.vlan and last_nic: if int(old_vlan) != int(_new_cloud_obj.vlan.vlan_id): logger.info("Setting last interface to public vlan %s." % new_vlan) juniper = Juniper( interface.switch_ip, interface.switch_port, old_vlan, _new_cloud_obj.vlan.vlan_id, ) success = juniper.convert_port_public() if success: logger.info("Successfully updated switch settings.") else: logger.error( "There was something wrong updating switch for %s:%s" % (host, interface.name)) return False else: if int(old_vlan) != int(new_vlan): juniper = Juniper(interface.switch_ip, interface.switch_port, old_vlan, new_vlan) success = juniper.set_port() if success: logger.info("Successfully updated switch settings.") else: logger.error( "There was something wrong updating switch for %s:%s" % (host, interface.name)) return False if ssh_helper: ssh_helper.disconnect() return True
def verify(_host_name, change=False, nic1=None, nic2=None, nic3=None, nic4=None, nic5=None): _nics = {"em1": nic1, "em2": nic2, "em3": nic3, "em4": nic4, "em5": nic5} _host_obj = Host.objects(name=_host_name).first() if not _host_obj: logger.error(f"Hostname not found.") return logger.info(f"Host: {_host_obj.name}") if _host_obj.interfaces: interfaces = sorted(_host_obj.interfaces, key=lambda k: k["name"]) for i, interface in enumerate(interfaces): vlan = _nics.get(interface.name) if vlan: ssh_helper = SSHHelper(interface.switch_ip, Config["junos_username"]) try: _, old_vlan_out = ssh_helper.run_cmd( "show configuration interfaces %s" % interface.switch_port) old_vlan = old_vlan_out[0].split(";")[0].split()[1] if old_vlan.startswith("QinQ"): old_vlan = old_vlan[7:] except IndexError: old_vlan = 0 try: _, vlan_member_out = ssh_helper.run_cmd( "show configuration vlans | display set | match %s.0" % interface.switch_port) vlan_member = vlan_member_out[0].split()[2][4:].strip(",") except IndexError: logger.warning( "Could not determine the previous VLAN member for %s, switch %s, switch port %s " % ( interface.name, interface.switch_ip, interface.switch_port, )) vlan_member = 0 ssh_helper.disconnect() if int(old_vlan) != int(vlan): logger.warning("Interface %s not using QinQ_vl%s", interface.switch_port, vlan) if int(vlan_member) != int(vlan): logger.warning( "Interface %s appears to be a member of VLAN %s, should be %s", interface.switch_port, vlan_member, vlan, ) if change: logger.info(f"Change requested for {interface.name}") juniper = Juniper( interface.switch_ip, interface.switch_port, vlan_member, vlan, ) success = juniper.set_port() if success: logger.info( "Successfully updated switch settings.") else: logger.error( f"There was something wrong updating switch for {interface.name}" ) else: logger.info( f"Interface {interface.name} is already configured for vlan{vlan}" ) else: logger.error(f"The host has no interfaces defined")
async def post_network_test(self): test_host = self.hosts[0] hosts_down = [] for host in self.hosts: try: nc = Netcat(host.name) healthy = await nc.health_check() except OSError: healthy = False if not healthy: hosts_down.append(host.name) if len(host.interfaces) > len(test_host.interfaces): test_host = host if hosts_down: logger.error( "The following hosts appear to be down or with no ssh connection:" ) for i in hosts_down: logger.error(i) return False try: ssh_helper = SSHHelper(test_host.name) except (SSHException, NoValidConnectionsError, socket.timeout) as ex: logger.debug(ex) logger.error( "Could not establish connection with host: %s." % test_host.name ) self.report = ( self.report + "Could not establish connection with host: %s.\n" % test_host.name ) return False host_list = " ".join([host.name for host in self.hosts]) result, output = ssh_helper.run_cmd( f"fping -t {Config.FPING_TIMEOUT} -B 1 -u {host_list}" ) if not result: return False for i, interface in enumerate(Config.INTERFACES.keys()): new_ips = [] host_ips = [ {"ip": socket.gethostbyname(host.name), "host": host} for host in self.hosts if interface in [_interface.name for _interface in host.interfaces] ] for host in host_ips: _host_obj = host["host"] _interfaces = Config.INTERFACES[interface] last_nic = i == len(_host_obj.interfaces) - 1 if last_nic and self.cloud.vlan: continue for value in _interfaces: ip_apart = host["ip"].split(".") octets = value.split(".") ip_apart[0] = octets[0] ip_apart[1] = octets[1] new_ips.append(".".join(ip_apart)) if new_ips: all_ips = " ".join(new_ips) result, output = ssh_helper.run_cmd( f"fping -t {Config.FPING_TIMEOUT} -B 1 -u {all_ips}" ) if not result: pattern = re.compile(r"(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})") hosts = [] for error in output: ip = pattern.search(error.split()[-1])[0] if ip: hosts.append(ip) hosts_set = set(hosts) logger.warning("The following IPs are not responsive:") for host in hosts_set: logger.warning(host) return False ssh_helper.disconnect() return True
def verify(_cloud_name, change=False): _cloud_obj = Cloud.objects(name=_cloud_name).first() quads = Api(API_URL) hosts = quads.get_cloud_hosts(_cloud_name) for _host in hosts: _host_obj = Host.objects(name=_host["name"]).first() if _host_obj.interfaces: _public_vlan_obj = Vlan.objects(cloud=_cloud_obj).first() for i, interface in enumerate(_host_obj.interfaces): ssh_helper = SSHHelper(interface.ip_address, conf["junos_username"]) vlan = get_vlan(_cloud_obj, i) try: old_vlan_out = ssh_helper.run_cmd( "show configuration interfaces %s" % interface.switch_port) old_vlan = old_vlan_out[0].split(";")[0].split()[1][7:] except IndexError: old_vlan = None try: vlan_member_out = ssh_helper.run_cmd( "show vlans interface %s.0" % interface.switch_port) vlan_member = vlan_member_out[1].split()[2][4:].strip(",") except IndexError: vlan_member = None ssh_helper.disconnect() if not old_vlan: logger.warning( "Could not determine the previous VLAN for %s on %s, switch %s, switchport %s" % (_host["name"], interface.name, interface.ip_address, interface.switch_port)) old_vlan = get_vlan(_cloud_obj, i) if not vlan_member: logger.warning( "Could not determine the previous VLAN member for %s on %s, switch %s, switchport %s" % (_host["name"], interface.name, interface.ip_address, interface.switch_port)) vlan_member = old_vlan if int(old_vlan) != int(vlan): logger.warning("interface %s not using QinQ_vl%s", interface.switch_port, vlan) if _public_vlan_obj and i == len(_host_obj.interfaces) - 1: vlan = _public_vlan_obj.vlan_id if int(vlan_member) != int(vlan): logger.warning( "interface %s appears to be a member of VLAN %s, should be %s", interface.switch_port, vlan_member, vlan) if change: logger.info('=== INFO: change requested') if _public_vlan_obj and i == len( _host_obj.interfaces) - 1: logger.info( "Setting last interface to public vlan %s." % _public_vlan_obj.vlan_id) success = juniper_convert_port_public( interface.ip_address, interface.switch_port, str(old_vlan), str(vlan)) else: success = juniper_set_port(interface.ip_address, interface.switch_port, str(vlan_member), str(vlan)) if success: logger.info( "Successfully updated switch settings.") else: logger.error( "There was something wrong updating switch for %s:%s" % (_host.name, interface.name))
def move_and_rebuild(host, old_cloud, new_cloud, rebuild=False): logger.debug("Moving and rebuilding host: %s" % host) untouchable_hosts = conf["untouchable_hosts"] logger.debug("Untouchable hosts: %s" % untouchable_hosts) _host_obj = Host.objects(name=host).first() if not _host_obj.interfaces: logger.error("Host has no interfaces defined.") return False if host in untouchable_hosts: logger.error("No way...") return False _old_cloud_obj = Cloud.objects(name=old_cloud).first() _new_cloud_obj = Cloud.objects(name=new_cloud).first() logger.debug("Connecting to switch on: %s" % _host_obj.interfaces[0].ip_address) _public_vlan_obj = Vlan.objects(cloud=_new_cloud_obj).first() for i, interface in enumerate(_host_obj.interfaces): ssh_helper = SSHHelper(interface.ip_address, conf["junos_username"]) old_vlan_out = ssh_helper.run_cmd("show configuration interfaces %s" % interface.switch_port) old_vlan = old_vlan_out[0].split(";")[0].split()[1][7:] if not old_vlan: logger.warning( "Warning: Could not determine the previous VLAN for %s on %s, switch %s, switchport %s" % (host, interface.name, interface.ip_address, interface.switch_port)) old_vlan = get_vlan(_old_cloud_obj, i) new_vlan = get_vlan(_new_cloud_obj, i) if _public_vlan_obj and i == len(_host_obj.interfaces) - 1: logger.info("Setting last interface to public vlan %s." % new_vlan) if int(old_vlan) != int(_public_vlan_obj.vlan_id): success = juniper_convert_port_public( interface.ip_address, interface.switch_port, str(old_vlan), str(_public_vlan_obj.vlan_id)) if success: logger.info("Successfully updated switch settings.") else: logger.error( "There was something wrong updating switch for %s:%s" % (host, interface.name)) return False else: if int(old_vlan) != int(new_vlan): success = juniper_set_port(interface.ip_address, interface.switch_port, str(old_vlan), str(new_vlan)) if success: logger.info("Successfully update switch settings.") else: logger.error( "There was something wrong updating switch for %s:%s" % (host, interface.name)) return False if i == len(_host_obj.interfaces) - 1: _old_vlan_obj = Vlan.objects(cloud=_old_cloud_obj).first() if _old_vlan_obj: _old_vlan_obj.update(cloud=None) ssh_helper.disconnect() ipmi_new_pass = _new_cloud_obj.ticket if _new_cloud_obj.ticket else conf[ "$ipmi_password"] foreman = Foreman( conf["foreman_api_url"], conf["foreman_username"], conf["foreman_password"], ) foreman.remove_role(_old_cloud_obj.name, _host_obj.name) foreman.add_role(_new_cloud_obj.name, _host_obj.name) foreman.update_user_password(_new_cloud_obj.name, ipmi_new_pass) ipmi_set_pass = [ "/usr/bin/ipmitool", "-I", "lanplus", "-H", "mgmt-%s" % host, "-U", conf["ipmi_username"], "-P", conf["ipmi_password"], "user", "set", "password", str(conf["ipmi_cloud_username_id"]), ipmi_new_pass ] logger.debug("ipmi_set_pass: %s" % ipmi_set_pass) subprocess.call(ipmi_set_pass) ipmi_set_operator = [ "/usr/bin/ipmitool", "-I", "lanplus", "-H", "mgmt-%s" % host, "-U", conf["ipmi_username"], "-P", conf["ipmi_password"], "user", "priv", str(conf["ipmi_cloud_username_id"]), "0x4" ] logger.debug("ipmi_set_operator: %s" % ipmi_set_operator) subprocess.call(ipmi_set_operator) if rebuild and _new_cloud_obj.name != "cloud01": badfish = Badfish("mgmt-%s" % host, conf["ipmi_username"], conf["ipmi_password"]) if "pdu_management" in conf and conf["pdu_management"]: # TODO: pdu management pass if is_supermicro(host): ipmi_pxe_persistent = [ "/usr/bin/ipmitool", "-I", "lanplus", "-H", "mgmt-%s" % host, "-U", conf["ipmi_username"], "-P", conf["ipmi_password"], "chassis", "bootdev", "pxe", "options", "=", "persistent" ] logger.debug("ipmi_pxe_persistent: %s" % ipmi_pxe_persistent) subprocess.call(ipmi_pxe_persistent) if is_supported(host): try: badfish.change_boot( "director", os.path.join(os.path.dirname(__file__), "../../conf/idrac_interfaces.yml")) except SystemExit as ex: logger.debug(ex) logger.error("Could not set boot order via Badfish.") return False foreman_success = foreman.remove_extraneous_interfaces(host) foreman_success = foreman.set_host_parameter( host, "rhel73", "false") and foreman_success foreman_success = foreman.set_host_parameter( host, "rhel75", "false") and foreman_success foreman_success = foreman.set_host_parameter( host, "overcloud", "true") and foreman_success foreman_success = foreman.put_parameter(host, "build", 1) and foreman_success foreman_success = foreman.put_parameter_by_name( host, "operatingsystems", conf["foreman_default_os"], "title") and foreman_success foreman_success = foreman.put_parameter_by_name( host, "ptables", conf["foreman_default_ptable"]) and foreman_success foreman_success = foreman.put_parameter_by_name( host, "media", conf["foreman_default_medium"]) and foreman_success if not foreman_success: logger.error( "There was something wrong setting Foreman host parameters.") try: badfish.set_next_boot_pxe() badfish.reboot_server() except SystemExit: if is_supermicro(host): logger.warning("Badfish not yet supported on Supermicro: %s." % host) else: logger.exception( "There was something wrong setting next PXE boot via Badfish." ) return False logger.debug("Updating host: %s") _host_obj.update(cloud=_new_cloud_obj, build=False, last_build=datetime.now()) else: logger.debug("Updating host: %s") _host_obj.update(cloud=_new_cloud_obj, build=False, last_build=datetime.now())
def verify(_cloud_name, change=False): _cloud_obj = Cloud.objects(name=_cloud_name).first() hosts = Host.objects(cloud=_cloud_obj) for _host_obj in hosts: logger.info(f"Host: {_host_obj.name}") if _host_obj.interfaces: interfaces = sorted(_host_obj.interfaces, key=lambda k: k["name"]) for i, interface in enumerate(interfaces): ssh_helper = SSHHelper(interface.ip_address, conf["junos_username"]) last_nic = i == len(_host_obj.interfaces) - 1 vlan = get_vlan(_cloud_obj, i, last_nic) try: old_vlan_out = ssh_helper.run_cmd( "show configuration interfaces %s" % interface.switch_port) old_vlan = old_vlan_out[0].split(";")[0].split()[1] if old_vlan.startswith("QinQ"): old_vlan = old_vlan[7:] except IndexError: old_vlan = None try: vlan_member_out = ssh_helper.run_cmd( "show configuration vlans | display set | match %s.0" % interface.switch_port) vlan_member = vlan_member_out[0].split()[2][4:].strip(",") except IndexError: if not _cloud_obj.vlan and not last_nic: logger.warning( "Could not determine the previous VLAN member for %s, switch %s, switch port %s " % ( interface.name, interface.ip_address, interface.switch_port, )) vlan_member = None ssh_helper.disconnect() if not old_vlan: if not _cloud_obj.vlan and not last_nic: logger.warning( "Could not determine the previous VLAN for %s, switch %s, switchport %s" % ( interface.name, interface.ip_address, interface.switch_port, )) old_vlan = get_vlan(_cloud_obj, i, last_nic) if int(old_vlan) != int(vlan): logger.warning("Interface %s not using QinQ_vl%s", interface.switch_port, vlan) if _cloud_obj.vlan and i == len(_host_obj.interfaces) - 1: vlan = _cloud_obj.vlan.vlan_id if not vlan_member or int(vlan_member) != int(vlan): if not _cloud_obj.vlan and not last_nic: logger.warning( "Interface %s appears to be a member of VLAN %s, should be %s", interface.switch_port, vlan_member, vlan, ) if change: if _cloud_obj.vlan and last_nic: if int(_cloud_obj.vlan.vlan_id) != int(old_vlan): logger.info( f"Change requested for {interface.name}") logger.info( "Setting last interface to public vlan %s." % _cloud_obj.vlan.vlan_id) success = juniper_convert_port_public( interface.ip_address, interface.switch_port, old_vlan, vlan, ) else: logger.info( f"No changes required for {interface.name}" ) continue else: logger.info( f"Change requested for {interface.name}") success = juniper_set_port( interface.ip_address, interface.switch_port, vlan_member, vlan, ) if success: logger.info( "Successfully updated switch settings.") else: logger.error( f"There was something wrong updating switch for {interface.name}" )
def post_network_test(self): test_host = self.hosts[0] hosts_down = [] for host in self.hosts: nc = Netcat(host.name) if not nc.health_check(): hosts_down.append(host.name) nc.close() if len(host.interfaces) > len(test_host.interfaces): test_host = host if hosts_down: logger.error("The following hosts appear to be down or with no ssh connection:") for i in hosts_down: logger.error(i) return False try: ssh_helper = SSHHelper(test_host.name) except (SSHException, NoValidConnectionsError, socket.timeout) as ex: logger.debug(ex) logger.error( "Could not establish connection with host: %s." % test_host.name ) self.report = ( self.report + "Could not establish connection with host: %s.\n" % test_host.name ) return False host_list = " ".join([host.name for host in self.hosts]) if type(ssh_helper.run_cmd("fping -u %s" % host_list)) != list: return False for i, interface in enumerate(INTERFACES.keys()): new_ips = [] host_ips = [ {"ip": socket.gethostbyname(host.name), "host": host} for host in self.hosts if interface in [_interface.name for _interface in host.interfaces] ] for host in host_ips: _host_obj = host["host"] _interfaces = INTERFACES[interface] last_nic = i == len(_host_obj.interfaces) - 1 if last_nic and self.cloud.vlan: _interfaces = _interfaces[:1] for value in _interfaces: ip_apart = host["ip"].split(".") octets = value.split(".") ip_apart[0] = octets[0] ip_apart[1] = octets[1] new_ips.append(".".join(ip_apart)) if new_ips: if type(ssh_helper.run_cmd("fping -u %s" % " ".join(new_ips))) != list: return False ssh_helper.disconnect() return True
def verify(_cloud_name, _host_name, change=False): if not _cloud_name and not _host_name: logger.warning( f"At least one of --cloud or --host should be specified.") return _cloud_obj = None if _cloud_name: _cloud_obj = Cloud.objects(name=_cloud_name).first() if _host_name: hosts = Host.objects(name=_host_name, retired=False) else: hosts = Host.objects(cloud=_cloud_obj, retired=False) first_host = hosts.first() if not _cloud_obj: _cloud_obj = first_host.cloud if _cloud_obj != first_host.cloud: logger.warning(f"Both --cloud and --host have been specified.") logger.warning(f"Host: {first_host.name}") logger.warning(f"Cloud: {_cloud_obj.name}") logger.warning( f"However, {first_host.name} is a member of {first_host.cloud.name}" ) logger.warning(f"!!!!! Be certain this is what you want to do. !!!!!") for _host_obj in hosts: logger.info(f"Host: {_host_obj.name}") if _host_obj.interfaces: interfaces = sorted(_host_obj.interfaces, key=lambda k: k["name"]) for i, interface in enumerate(interfaces): ssh_helper = SSHHelper(interface.switch_ip, Config["junos_username"]) last_nic = i == len(_host_obj.interfaces) - 1 vlan = get_vlan(_cloud_obj, i, last_nic) try: _, old_vlan_out = ssh_helper.run_cmd( "show configuration interfaces %s" % interface.switch_port) old_vlan = old_vlan_out[0].split(";")[0].split()[1] if old_vlan.startswith("QinQ"): old_vlan = old_vlan[7:] except IndexError: old_vlan = 0 try: _, vlan_member_out = ssh_helper.run_cmd( "show configuration vlans | display set | match %s.0" % interface.switch_port) vlan_member = vlan_member_out[0].split()[2][4:].strip(",") except IndexError: if not _cloud_obj.vlan and not last_nic: logger.warning( "Could not determine the previous VLAN member for %s, switch %s, switch port %s " % ( interface.name, interface.switch_ip, interface.switch_port, )) vlan_member = 0 ssh_helper.disconnect() if int(old_vlan) != int(vlan): logger.warning("Interface %s not using QinQ_vl%s", interface.switch_port, vlan) if int(vlan_member) != int(vlan): if not last_nic: logger.warning( "Interface %s appears to be a member of VLAN %s, should be %s", interface.switch_port, vlan_member, vlan, ) if change: if _cloud_obj.vlan and last_nic: if int(_cloud_obj.vlan.vlan_id) != int(old_vlan): logger.info( f"Change requested for {interface.name}") logger.info( "Setting last interface to public vlan %s." % _cloud_obj.vlan.vlan_id) juniper = Juniper( interface.switch_ip, interface.switch_port, old_vlan, vlan, ) success = juniper.convert_port_public() else: logger.info( f"No changes required for {interface.name}" ) continue else: logger.info( f"Change requested for {interface.name}") juniper = Juniper( interface.switch_ip, interface.switch_port, vlan_member, vlan, ) success = juniper.set_port() if success: logger.info( "Successfully updated switch settings.") else: logger.error( f"There was something wrong updating switch for {interface.name}" )