def check_docker_volume(config):
vprint("Checking docker volume driver")
if not exe_check("docker ps >/dev/null 2>&1"):
return ff("Docker is not installed", "42BAAC76")
if exe_check("docker plugin ls | grep {}".format(PLUGIN)):
return ff("Datera Docker plugin is not installed", "6C531C5D")
plugin = exe("docker plugin ls | grev -v DESCRIPTION | "
"grep {}".format(PLUGIN))
if len(plugin.strip().split('\n')) > 1:
wf("More than one version of Datera docker driver installed",
"B3BF691D")
if 'enabled' not in plugin or 'disabled' in plugin:
ff("Datera docker plugin is not enabled")
test_name = "ddct-test1"
if not exe_check(
"docker volume create -d {} --name {} --opt replica=1 --opt "
"size=1".format(PLUGIN, test_name)):
return ff("Could not create a volume with the Datera Docker plugin",
"621A6F51")
api = config['api']
try:
api.app_instances.get(test_name)
except ApiNotFoundError:
return ff("Docker volume {} did not create on the Datera backend"
"".format(test_name), "B106D1CD")
if not exe_check("docker volume rm {}".format(test_name)):
ff("Could not delete Docker volume {}".format(test_name), "AF3DB8B3")
def check_kubernetes_driver_flex(config):
# Is kubectl present?
if not exe_check("which kubectl"):
return ff("Could not detect kubectl installation", "572B0511")
# Does kubectl have a supported version?
kversion = exe("kubectl version").strip().split("\n")
for line in kversion:
m1 = KCTL_MA_RE.search(line)
m2 = KCTL_MI_RE.search(line)
if not m1 and m2:
return ff("Could not detect kubectl version", "C1802A6E")
supported = calc_version(SUPPORTED_MAJOR, SUPPORTED_MINOR)
major = m1.group(1)
minor = m2.group(1)
found = calc_version(major, minor)
if found < supported:
return ff(
"Kubectl has version {}, which is lower than supported "
"version {}".format(found, supported), "D2DA6596")
# Are dependencies installed?
if not exe_check("which mkfs"):
ff("mkfs is not installed", "FE13A328")
if not exe_check("which iscsiadm"):
ff("sg3_utils does not appear to be installed", "94BF0B77")
# Is attach-detach disabled in kubelet?
exstart = exe("systemctl show kubelet.service | grep ExecStart")
if "--enable-controller-attach-detach=false" not in exstart:
wf(
"Attach-detach is enabled in kublet's systemctl entry. Run "
"--enable-controller-attach-detach=false when starting kubelet "
"to disable", "5B3729F2")
if "Active: active" in exe("systemctl status kubelet"):
kpath = KPATH_RE.search(exstart).group(1)
exstart = exe("ps -ef | grep {} | grep -v grep".format(kpath))
if "--enable-controller-attach-detach=false" not in exstart:
ff(
"Attach-detach is enabled in kublet. Run "
"--enable-controller-attach-detach=false when starting kubelet "
"to disable", "86FFD7F2")
else:
ff("The kubelet service is not running", "0762A89B")
# Agents are running?
pods = exe("kubectl --namespace=datera get pods").strip()
if not pods:
return ff("Installer agents and provisioner agents are not running",
"244C0B34")
installer_agent = False
provisioner_agent = False
for line in pods.split("\n"):
if "datera-installer-agent" in line:
installer_agent = True
if "datera-provisioner-agent" in line:
provisioner_agent = True
if not installer_agent:
ff("Installer agents not found", "08193032")
if not provisioner_agent:
ff("Provisioner agents not found", "3AAF82CA")
def get_install_script():
vprint("Retrieving installation script")
if not exe_check("wget {}".format(INSTALL_SCRIPT)):
print("Failed to get iscsi-recv installation script {}".format(
INSTALL_SCRIPT))
sys.exit(1)
if not exe_check("chmod +x {}".format(SCRIPT_NAME)):
print("Failed to set iscsi-recv install script to executable: {}"
"".format(SCRIPT_NAME))
def vip1_check(config):
vip1 = config["vip1_ip"]
if not exe_check("ping -c 2 -W 1 {}".format(vip1), err=False):
ff("Could not ping vip1 ip {}".format(vip1), "1827147B", fix=NET_FIX)
timeout = 5
while not exe_check(
"ip neigh show | grep {} | grep REACHABLE".format(vip1)):
timeout -= 1
time.sleep(1)
if timeout < 0:
wf("Arp state for vip1 [{}] is not 'REACHABLE'".format(vip1),
"3C33D70D")
break
def check_multipath(config):
vprint("Checking multipath settings")
if not exe_check("which multipath", err=False):
ff("Multipath binary could not be found, is it installed?", "2D18685C")
if not exe_check("which systemctl"):
if not exe_check("service multipathd status | grep 'Active: active'",
err=False):
fix = "service multipathd start"
ff("multipathd not enabled", "541C10BF", fix=fix)
else:
if not exe_check("systemctl status multipathd | grep 'Active: active'",
err=False):
fix = "systemctl start multipathd"
ff("multipathd not enabled", "541C10BF", fix=fix)
def check_irq(config):
vprint("Checking irqbalance settings, (should be turned off)")
if not exe_check("which systemctl"):
if not exe_check(
"service irqbalance status | "
"grep 'Active: active'", err=True):
fix = "service irqbalance stop"
return ff("irqbalance is active", "B19D9FF1", fix=fix)
else:
if not exe_check(
"systemctl status irqbalance | "
"grep 'Active: active'", err=True):
fix = "systemctl stop irqbalance && systemctl disable irqbalance"
return ff("irqbalance is active", "B19D9FF1", fix=fix)
def check_iscsi(config):
vprint("Checking ISCSI settings")
if not exe_check("which iscsiadm"):
if get_pkg_manager() == APT:
fix = "apt-get install open-iscsi"
elif get_pkg_manager() == YUM:
fix = "yum install iscsi-initiator-utils"
ff("iscsiadm is not available, has open-iscsi been installed?",
"EFBB085C",
fix=fix)
if not exe_check("ps -ef | grep iscsid | grep -v grep"):
fix = "service iscsi start || systemctl start iscsid.service"
ff("iscsid is not running. Is the iscsid service running?",
"EB22737E",
fix=fix)
ifile = "/etc/iscsi/iscsid.conf"
if not os.path.exists(ifile):
ff("iscsid configuration file does not exist", "C6F2B356")
return
with io.open(ifile, 'r') as f:
iconf = f.readlines()
noopt = "node.session.timeo.noop_out_timeout"
noopt_found = False
noopi = "node.session.timeo.noop_out_interval"
noopi_found = False
for index, line in enumerate(iconf):
if not noopt_found and noopt in line:
noopt_found = True
if "2" not in line:
ff("{} is not set to '2' in iscsid.conf".format(noopt),
"F6A49337")
elif noopt_found and noopt in line:
wf(
"{} duplicate found in iscsid.conf, line {}".format(
noopt, index), "D3E55910")
if noopi in line:
noopi_found = True
if "2" not in line:
ff("{} is not set to '2' in iscsid.conf".format(noopi),
"E48C1907")
elif noopi_found and noopi in line:
wf(
"{} duplicate found in iscsid.conf, line {}".format(
noopi, index), "CA9AA865")
if not noopt_found:
ff("'{} = 2' is not present in iscsid.conf".format(noopt), "E29BF18A")
if not noopi_found:
ff("'{} = 2' is not present in iscsid.conf".format(noopi), "A2EED511")
def vip2_check(config):
vip2 = config.get("vip2_ip")
if not vip2:
wf("No vip2_ip found", "16EB208B")
return
if vip2 and not exe_check("ping -c 2 -W 1 {}".format(vip2), err=False):
ff("Could not ping vip2 ip {}".format(vip2), "3D76CE5A", fix=NET_FIX)
timeout = 5
while not exe_check(
"ip neigh show | grep {} | grep REACHABLE".format(vip2)):
timeout -= 1
time.sleep(1)
if timeout < 0:
wf("Arp state for vip2 [{}] is not 'REACHABLE'".format(vip2),
"4F6B8D91")
break
def mgmt_check(config):
mgmt = config["mgmt_ip"]
if not exe_check("ping -c 2 -W 1 {}".format(mgmt), err=False):
ff("Could not ping management ip {}".format(mgmt),
"65FC68BB",
fix=NET_FIX)
timeout = 5
while not exe_check(
"ip neigh show | grep {} | grep REACHABLE".format(mgmt)):
timeout -= 1
time.sleep(1)
if timeout < 0:
fix = "Check the connection to {}".format(mgmt)
wf("Arp state for mgmt [{}] is not 'REACHABLE'".format(mgmt),
"BF6A912A",
fix=fix)
break
def check_cpufreq(config):
vprint("Checking cpufreq settings")
if not exe_check("which cpupower"):
if get_os() == UBUNTU:
version = exe("uname -r").strip()
fix = "apt-get install linux-tools-{}".format(version)
else:
# RHEL puts this stuff in kernel-tools
fix = "yum install kernel-tools"
return ff("cpupower is not installed", "20CEE732", fix=fix)
if not exe_check(
"cpupower frequency-info --governors | "
"grep performance", err=False):
fix = ("No-fix -- if this system is a VM governors might not be "
"available and this check can be ignored")
return ff("No 'performance' governor found for system",
"333FBD45",
fix=fix)
def check_mtu_l3(ip, config):
"""
We just care that the packet gets there. Fragmentation is gonna happen
with L3 routing.
"""
vprint("Performing L3 MTU check")
sif = get_interface_for_ip(ip)
if not sif:
return ff(
"Couldn't find interface with network matching ip {}"
"".format(ip), "710BFC7E")
# Ping check
if not exe_check("ping -s 32000 -c 2 -W 1 {}".format(ip)):
if not exe_check("ping -c 2 -W 1 {}".format(ip)):
return ff("Could not ping interface [{}]".format(ip), "EC2D3621")
ff(
"Could not ping interface [{}] with large (32k) packet size, packet"
"fragmentation may not be working correctly.".format(ip),
"A4CA0D72")
def check_arp(config):
vprint("Checking ARP settings")
if not exe_check(
"sysctl --all 2>/dev/null | "
"grep 'net.ipv4.conf.all.arp_announce = 2'",
err=False):
fix = "sysctl net.ipv4.conf.all.arp_announce=2"
ff("net.ipv4.conf.all.arp_announce != 2 in sysctl",
"9000C3B6",
fix=fix)
if not exe_check(
"sysctl --all 2>/dev/null | "
"grep 'net.ipv4.conf.all.arp_ignore = 1'",
err=False):
fix = "sysctl net.ipv4.conf.all.arp_ignore=1"
ff("net.ipv4.conf.all.arp_ignore != 1 in sysctl", "BDB4D5D8", fix=fix)
gcf = "/proc/sys/net/ipv4/route/gc_interval"
gc = int(exe("cat {}".format(gcf)))
if gc != 5:
fix = "echo 5 > {}".format(gcf)
ff("{} is currently set to {}".format(gcf, gc), "A06CD19F", fix=fix)
def check_mtu_normal(name, ip, config):
vprint("Performing MTU check")
cname = iface_dict[name]
sif = get_interface_for_ip(ip)
if not sif:
return ff(
"Couldn't find interface with network matching ip {}"
"".format(ip), "710BFC7E")
try:
match = MTU_RE.match(exe("ip ad show {} | grep mtu".format(sif)))
except subprocess.CalledProcessError:
return ff("Couldn't find client {} interface MTU".format(name),
"CBF8CC4C")
if not match:
return ff("Couldn't find client {} interface MTU".format(name),
"CBF8CC4C")
local_mtu = match.groups(1)[0]
cluster_mtu = None
api = config['api']
if name == "MGMT":
cluster_mtu = api.system.network.mgmt_vip.get(
)['network_paths'][0]['mtu']
elif name == "VIP1":
cluster_mtu = api.system.network.get(
)['access_vip']['network_paths'][0]['mtu']
elif name == "VIP2":
cluster_mtu = api.system.network.get(
)['access_vip']['network_paths'][0]['mtu']
if not cluster_mtu:
return ff("Couldn't find cluster {} interface MTU".format(cname),
"057AF23D")
if str(local_mtu) != str(cluster_mtu):
ff(
"Local interface {} MTU does not match cluster {} interface MTU "
"[{} != {}]".format(sif, cname, local_mtu, cluster_mtu),
"D7F667BC")
# Ping check
if not exe_check("ping -s 32000 -c 2 -W 1 {}".format(ip)):
ff(
"Could not ping interface with large (32k) packet size, packet "
"fragmentation may not be working correctly", "A4CA0D72")
def check_single_volume_performance_fio_4k(config):
vprint("Checking FIO performance, single volume")
if not exe_check("which fio"):
ff("FIO is not installed", "0BB2848F")
api = config['api']
def is_journalctl():
return exe_check("journalctl --unit {} | grep 'No entries'", err=False)
def check_requirements():
vprint("Checking Requirements")
for binary in REQUIREMENTS:
if not exe_check("which {}".format(binary), err=False):
return "missing" + binary
def check_kubernetes_driver_csi(config):
# Is kubectl present?
if not exe_check("which kubectl"):
return ff("Could not detect kubectl installation", "572B0511")
# Does kubectl have a supported version?
kversion = exe("kubectl version").strip().split("\n")
for line in kversion:
m1 = KCTL_MA_RE.search(line)
m2 = KCTL_MI_RE.search(line)
if not m1 and m2:
return ff("Could not detect kubectl version", "C1802A6E")
supported = calc_version(SUPPORTED_MAJOR, SUPPORTED_MINOR)
major = m1.group(1)
minor = m2.group(1)
found = calc_version(major, minor)
if found < supported:
return ff(
"Kubectl has version {}, which is lower than supported "
"version {}".format(found, supported), "D2DA6596")
# Are dependencies installed?
if not exe_check("which iscsiadm"):
ff("open-iscsi does not appear to be installed", "94BF0B77")
# Is attach-detach disabled in kubelet?
exstart = exe("systemctl show kubelet.service | grep ExecStart")
if not exstart:
if exe_check("microk8s.kubectl"):
ff(
"kubelet service not detected. microk8s is not currently "
"supported", "995EA49E")
return
else:
ff(
"kubelet service not detected. 'systemctl show kubelet.service'"
" returned nothing", "F3C47DDF")
if "--allow-privileged" not in exstart:
ff(
"--allow-privileged is not enabled in kublet's systemctl entry. "
"Run --allow-privileged=true when starting kubelet "
"to enable", "7475B000")
if "Active: active" in exe("systemctl status kubelet"):
kpath = KPATH_RE.search(exstart).group(1)
exstart = exe("ps -ef | grep {} | grep -v grep".format(kpath))
if "--enable-controller-attach-detach=false" in exstart:
ff(
"Attach-detach is disabled in kublet. Run "
"--enable-controller-attach-detach=true when starting kubelet "
"to enable", "86FFD7F2")
else:
ff("The kubelet service is not running", "0762A89B")
# iscsi-recv is running?
if not exe_check("ps -ef | grep iscsi-recv | grep -v grep"):
fix = "Run ./setup_iscsi.sh from the datera-csi repository"
ff("iscsi-recv binary is not running.", "A8B6BA35", fix=fix)
# Agents are running?
pods = exe("kubectl --namespace=kube-system get pods").strip()
if not pods:
return ff("CSI plugin pods are not running.",
"49BDC893",
fix="Install the CSI plugin deployment yaml. "
"'kubectl create -f csi.yaml'")
controller_pod = False
node_pods = False
for line in pods.split("\n"):
if "csi-provisioner-0" in line:
controller_pod = True
if "csi-node-" in line:
node_pods = True
if not controller_pod:
ff("Controller pod not found", "17FF7B78")
if not node_pods:
ff("At least one Node pod not found", "2FD6A7B4")
def run_install_script():
with verbose():
if not exe_check("./{}".format(SCRIPT_NAME)):
print("Failed to run iscsi-recv install script successfully: {}"
"".format(SCRIPT_NAME))
def check_something(config):
if not exe_check("ping -C 2 -W 1 1.1.1.1"):
ff("This test failed")
return
if not exe_check("traceroute 1.1.1.1"):
wf("This is a warning")
