def check_docker_volume(config): vprint("Checking docker volume driver") if not exe_check("docker ps >/dev/null 2>&1"): return ff("Docker is not installed", "42BAAC76") if exe_check("docker plugin ls | grep {}".format(PLUGIN)): return ff("Datera Docker plugin is not installed", "6C531C5D") plugin = exe("docker plugin ls | grev -v DESCRIPTION | " "grep {}".format(PLUGIN)) if len(plugin.strip().split('\n')) > 1: wf("More than one version of Datera docker driver installed", "B3BF691D") if 'enabled' not in plugin or 'disabled' in plugin: ff("Datera docker plugin is not enabled") test_name = "ddct-test1" if not exe_check( "docker volume create -d {} --name {} --opt replica=1 --opt " "size=1".format(PLUGIN, test_name)): return ff("Could not create a volume with the Datera Docker plugin", "621A6F51") api = config['api'] try: api.app_instances.get(test_name) except ApiNotFoundError: return ff("Docker volume {} did not create on the Datera backend" "".format(test_name), "B106D1CD") if not exe_check("docker volume rm {}".format(test_name)): ff("Could not delete Docker volume {}".format(test_name), "AF3DB8B3")
def check_kubernetes_driver_flex(config): # Is kubectl present? if not exe_check("which kubectl"): return ff("Could not detect kubectl installation", "572B0511") # Does kubectl have a supported version? kversion = exe("kubectl version").strip().split("\n") for line in kversion: m1 = KCTL_MA_RE.search(line) m2 = KCTL_MI_RE.search(line) if not m1 and m2: return ff("Could not detect kubectl version", "C1802A6E") supported = calc_version(SUPPORTED_MAJOR, SUPPORTED_MINOR) major = m1.group(1) minor = m2.group(1) found = calc_version(major, minor) if found < supported: return ff( "Kubectl has version {}, which is lower than supported " "version {}".format(found, supported), "D2DA6596") # Are dependencies installed? if not exe_check("which mkfs"): ff("mkfs is not installed", "FE13A328") if not exe_check("which iscsiadm"): ff("sg3_utils does not appear to be installed", "94BF0B77") # Is attach-detach disabled in kubelet? exstart = exe("systemctl show kubelet.service | grep ExecStart") if "--enable-controller-attach-detach=false" not in exstart: wf( "Attach-detach is enabled in kublet's systemctl entry. Run " "--enable-controller-attach-detach=false when starting kubelet " "to disable", "5B3729F2") if "Active: active" in exe("systemctl status kubelet"): kpath = KPATH_RE.search(exstart).group(1) exstart = exe("ps -ef | grep {} | grep -v grep".format(kpath)) if "--enable-controller-attach-detach=false" not in exstart: ff( "Attach-detach is enabled in kublet. Run " "--enable-controller-attach-detach=false when starting kubelet " "to disable", "86FFD7F2") else: ff("The kubelet service is not running", "0762A89B") # Agents are running? pods = exe("kubectl --namespace=datera get pods").strip() if not pods: return ff("Installer agents and provisioner agents are not running", "244C0B34") installer_agent = False provisioner_agent = False for line in pods.split("\n"): if "datera-installer-agent" in line: installer_agent = True if "datera-provisioner-agent" in line: provisioner_agent = True if not installer_agent: ff("Installer agents not found", "08193032") if not provisioner_agent: ff("Provisioner agents not found", "3AAF82CA")
def get_install_script(): vprint("Retrieving installation script") if not exe_check("wget {}".format(INSTALL_SCRIPT)): print("Failed to get iscsi-recv installation script {}".format( INSTALL_SCRIPT)) sys.exit(1) if not exe_check("chmod +x {}".format(SCRIPT_NAME)): print("Failed to set iscsi-recv install script to executable: {}" "".format(SCRIPT_NAME))
def vip1_check(config): vip1 = config["vip1_ip"] if not exe_check("ping -c 2 -W 1 {}".format(vip1), err=False): ff("Could not ping vip1 ip {}".format(vip1), "1827147B", fix=NET_FIX) timeout = 5 while not exe_check( "ip neigh show | grep {} | grep REACHABLE".format(vip1)): timeout -= 1 time.sleep(1) if timeout < 0: wf("Arp state for vip1 [{}] is not 'REACHABLE'".format(vip1), "3C33D70D") break
def check_multipath(config): vprint("Checking multipath settings") if not exe_check("which multipath", err=False): ff("Multipath binary could not be found, is it installed?", "2D18685C") if not exe_check("which systemctl"): if not exe_check("service multipathd status | grep 'Active: active'", err=False): fix = "service multipathd start" ff("multipathd not enabled", "541C10BF", fix=fix) else: if not exe_check("systemctl status multipathd | grep 'Active: active'", err=False): fix = "systemctl start multipathd" ff("multipathd not enabled", "541C10BF", fix=fix)
def check_irq(config): vprint("Checking irqbalance settings, (should be turned off)") if not exe_check("which systemctl"): if not exe_check( "service irqbalance status | " "grep 'Active: active'", err=True): fix = "service irqbalance stop" return ff("irqbalance is active", "B19D9FF1", fix=fix) else: if not exe_check( "systemctl status irqbalance | " "grep 'Active: active'", err=True): fix = "systemctl stop irqbalance && systemctl disable irqbalance" return ff("irqbalance is active", "B19D9FF1", fix=fix)
def check_iscsi(config): vprint("Checking ISCSI settings") if not exe_check("which iscsiadm"): if get_pkg_manager() == APT: fix = "apt-get install open-iscsi" elif get_pkg_manager() == YUM: fix = "yum install iscsi-initiator-utils" ff("iscsiadm is not available, has open-iscsi been installed?", "EFBB085C", fix=fix) if not exe_check("ps -ef | grep iscsid | grep -v grep"): fix = "service iscsi start || systemctl start iscsid.service" ff("iscsid is not running. Is the iscsid service running?", "EB22737E", fix=fix) ifile = "/etc/iscsi/iscsid.conf" if not os.path.exists(ifile): ff("iscsid configuration file does not exist", "C6F2B356") return with io.open(ifile, 'r') as f: iconf = f.readlines() noopt = "node.session.timeo.noop_out_timeout" noopt_found = False noopi = "node.session.timeo.noop_out_interval" noopi_found = False for index, line in enumerate(iconf): if not noopt_found and noopt in line: noopt_found = True if "2" not in line: ff("{} is not set to '2' in iscsid.conf".format(noopt), "F6A49337") elif noopt_found and noopt in line: wf( "{} duplicate found in iscsid.conf, line {}".format( noopt, index), "D3E55910") if noopi in line: noopi_found = True if "2" not in line: ff("{} is not set to '2' in iscsid.conf".format(noopi), "E48C1907") elif noopi_found and noopi in line: wf( "{} duplicate found in iscsid.conf, line {}".format( noopi, index), "CA9AA865") if not noopt_found: ff("'{} = 2' is not present in iscsid.conf".format(noopt), "E29BF18A") if not noopi_found: ff("'{} = 2' is not present in iscsid.conf".format(noopi), "A2EED511")
def vip2_check(config): vip2 = config.get("vip2_ip") if not vip2: wf("No vip2_ip found", "16EB208B") return if vip2 and not exe_check("ping -c 2 -W 1 {}".format(vip2), err=False): ff("Could not ping vip2 ip {}".format(vip2), "3D76CE5A", fix=NET_FIX) timeout = 5 while not exe_check( "ip neigh show | grep {} | grep REACHABLE".format(vip2)): timeout -= 1 time.sleep(1) if timeout < 0: wf("Arp state for vip2 [{}] is not 'REACHABLE'".format(vip2), "4F6B8D91") break
def mgmt_check(config): mgmt = config["mgmt_ip"] if not exe_check("ping -c 2 -W 1 {}".format(mgmt), err=False): ff("Could not ping management ip {}".format(mgmt), "65FC68BB", fix=NET_FIX) timeout = 5 while not exe_check( "ip neigh show | grep {} | grep REACHABLE".format(mgmt)): timeout -= 1 time.sleep(1) if timeout < 0: fix = "Check the connection to {}".format(mgmt) wf("Arp state for mgmt [{}] is not 'REACHABLE'".format(mgmt), "BF6A912A", fix=fix) break
def check_cpufreq(config): vprint("Checking cpufreq settings") if not exe_check("which cpupower"): if get_os() == UBUNTU: version = exe("uname -r").strip() fix = "apt-get install linux-tools-{}".format(version) else: # RHEL puts this stuff in kernel-tools fix = "yum install kernel-tools" return ff("cpupower is not installed", "20CEE732", fix=fix) if not exe_check( "cpupower frequency-info --governors | " "grep performance", err=False): fix = ("No-fix -- if this system is a VM governors might not be " "available and this check can be ignored") return ff("No 'performance' governor found for system", "333FBD45", fix=fix)
def check_mtu_l3(ip, config): """ We just care that the packet gets there. Fragmentation is gonna happen with L3 routing. """ vprint("Performing L3 MTU check") sif = get_interface_for_ip(ip) if not sif: return ff( "Couldn't find interface with network matching ip {}" "".format(ip), "710BFC7E") # Ping check if not exe_check("ping -s 32000 -c 2 -W 1 {}".format(ip)): if not exe_check("ping -c 2 -W 1 {}".format(ip)): return ff("Could not ping interface [{}]".format(ip), "EC2D3621") ff( "Could not ping interface [{}] with large (32k) packet size, packet" "fragmentation may not be working correctly.".format(ip), "A4CA0D72")
def check_arp(config): vprint("Checking ARP settings") if not exe_check( "sysctl --all 2>/dev/null | " "grep 'net.ipv4.conf.all.arp_announce = 2'", err=False): fix = "sysctl net.ipv4.conf.all.arp_announce=2" ff("net.ipv4.conf.all.arp_announce != 2 in sysctl", "9000C3B6", fix=fix) if not exe_check( "sysctl --all 2>/dev/null | " "grep 'net.ipv4.conf.all.arp_ignore = 1'", err=False): fix = "sysctl net.ipv4.conf.all.arp_ignore=1" ff("net.ipv4.conf.all.arp_ignore != 1 in sysctl", "BDB4D5D8", fix=fix) gcf = "/proc/sys/net/ipv4/route/gc_interval" gc = int(exe("cat {}".format(gcf))) if gc != 5: fix = "echo 5 > {}".format(gcf) ff("{} is currently set to {}".format(gcf, gc), "A06CD19F", fix=fix)
def check_mtu_normal(name, ip, config): vprint("Performing MTU check") cname = iface_dict[name] sif = get_interface_for_ip(ip) if not sif: return ff( "Couldn't find interface with network matching ip {}" "".format(ip), "710BFC7E") try: match = MTU_RE.match(exe("ip ad show {} | grep mtu".format(sif))) except subprocess.CalledProcessError: return ff("Couldn't find client {} interface MTU".format(name), "CBF8CC4C") if not match: return ff("Couldn't find client {} interface MTU".format(name), "CBF8CC4C") local_mtu = match.groups(1)[0] cluster_mtu = None api = config['api'] if name == "MGMT": cluster_mtu = api.system.network.mgmt_vip.get( )['network_paths'][0]['mtu'] elif name == "VIP1": cluster_mtu = api.system.network.get( )['access_vip']['network_paths'][0]['mtu'] elif name == "VIP2": cluster_mtu = api.system.network.get( )['access_vip']['network_paths'][0]['mtu'] if not cluster_mtu: return ff("Couldn't find cluster {} interface MTU".format(cname), "057AF23D") if str(local_mtu) != str(cluster_mtu): ff( "Local interface {} MTU does not match cluster {} interface MTU " "[{} != {}]".format(sif, cname, local_mtu, cluster_mtu), "D7F667BC") # Ping check if not exe_check("ping -s 32000 -c 2 -W 1 {}".format(ip)): ff( "Could not ping interface with large (32k) packet size, packet " "fragmentation may not be working correctly", "A4CA0D72")
def check_single_volume_performance_fio_4k(config): vprint("Checking FIO performance, single volume") if not exe_check("which fio"): ff("FIO is not installed", "0BB2848F") api = config['api']
def is_journalctl(): return exe_check("journalctl --unit {} | grep 'No entries'", err=False)
def check_requirements(): vprint("Checking Requirements") for binary in REQUIREMENTS: if not exe_check("which {}".format(binary), err=False): return "missing" + binary
def check_kubernetes_driver_csi(config): # Is kubectl present? if not exe_check("which kubectl"): return ff("Could not detect kubectl installation", "572B0511") # Does kubectl have a supported version? kversion = exe("kubectl version").strip().split("\n") for line in kversion: m1 = KCTL_MA_RE.search(line) m2 = KCTL_MI_RE.search(line) if not m1 and m2: return ff("Could not detect kubectl version", "C1802A6E") supported = calc_version(SUPPORTED_MAJOR, SUPPORTED_MINOR) major = m1.group(1) minor = m2.group(1) found = calc_version(major, minor) if found < supported: return ff( "Kubectl has version {}, which is lower than supported " "version {}".format(found, supported), "D2DA6596") # Are dependencies installed? if not exe_check("which iscsiadm"): ff("open-iscsi does not appear to be installed", "94BF0B77") # Is attach-detach disabled in kubelet? exstart = exe("systemctl show kubelet.service | grep ExecStart") if not exstart: if exe_check("microk8s.kubectl"): ff( "kubelet service not detected. microk8s is not currently " "supported", "995EA49E") return else: ff( "kubelet service not detected. 'systemctl show kubelet.service'" " returned nothing", "F3C47DDF") if "--allow-privileged" not in exstart: ff( "--allow-privileged is not enabled in kublet's systemctl entry. " "Run --allow-privileged=true when starting kubelet " "to enable", "7475B000") if "Active: active" in exe("systemctl status kubelet"): kpath = KPATH_RE.search(exstart).group(1) exstart = exe("ps -ef | grep {} | grep -v grep".format(kpath)) if "--enable-controller-attach-detach=false" in exstart: ff( "Attach-detach is disabled in kublet. Run " "--enable-controller-attach-detach=true when starting kubelet " "to enable", "86FFD7F2") else: ff("The kubelet service is not running", "0762A89B") # iscsi-recv is running? if not exe_check("ps -ef | grep iscsi-recv | grep -v grep"): fix = "Run ./setup_iscsi.sh from the datera-csi repository" ff("iscsi-recv binary is not running.", "A8B6BA35", fix=fix) # Agents are running? pods = exe("kubectl --namespace=kube-system get pods").strip() if not pods: return ff("CSI plugin pods are not running.", "49BDC893", fix="Install the CSI plugin deployment yaml. " "'kubectl create -f csi.yaml'") controller_pod = False node_pods = False for line in pods.split("\n"): if "csi-provisioner-0" in line: controller_pod = True if "csi-node-" in line: node_pods = True if not controller_pod: ff("Controller pod not found", "17FF7B78") if not node_pods: ff("At least one Node pod not found", "2FD6A7B4")
def run_install_script(): with verbose(): if not exe_check("./{}".format(SCRIPT_NAME)): print("Failed to run iscsi-recv install script successfully: {}" "".format(SCRIPT_NAME))
def check_something(config): if not exe_check("ping -C 2 -W 1 1.1.1.1"): ff("This test failed") return if not exe_check("traceroute 1.1.1.1"): wf("This is a warning")