class KernelVirtualMachinePlayer(unittest.TestCase): """ This class is used by all Kernel Virtual Machine testing suite Override the setUpClass by selecting your custom environment with the following catalog: >>> @classmethod >>> def setUpClass(cls): >>> cls.running_requirements() >>> cls.set_acserver() >>> cls.set_api() >>> cls.set_matchbox() >>> cls.set_dnsmasq() >>> cls.set_lldp() >>> cls.set_rack0() >>> cls.pause(cls.wait_setup_teardown) Note: you may use 'reset -q' because of Link Layer Discovery Protocol Container's """ __name__ = "KernelVirtualMachinePlayer" p_matchbox = multiprocessing.Process p_dnsmasq = multiprocessing.Process p_api = multiprocessing.Process p_lldp = multiprocessing.Process p_list = [] gen = generator.Generator euid_path = "%s" % os.path.dirname(os.path.abspath(__file__)) tests_path = "%s" % os.path.dirname(euid_path) app_path = os.path.dirname(tests_path) project_path = os.path.dirname(app_path) matchbox_path = "%s/matchbox" % project_path assets_path = "%s/matchbox/assets" % project_path runtime_path = "%s/runtime" % project_path rkt_bin = "%s/rkt/rkt" % runtime_path helm_bin = "%s/helm/helm" % runtime_path matchbox_bin = "%s/matchbox/matchbox" % runtime_path acserver_bin = "%s/run_acserver.py" % runtime_path ssh_private_key = os.path.join(tests_path, "testing.id_rsa") test_certs_path = "%s/test_certs" % tests_path test_matchbox_path = "%s/test_matchbox" % tests_path matchbox_port = int(os.getenv("MATCHBOX_PORT", "8080")) api_port = int(os.getenv("API_PORT", "5000")) api_ip = "172.20.0.1" api_uri = "http://%s:%d" % (api_ip, api_port) dev_null = open("/dev/null", "w") testing_sleep_seconds = get_kvm_sleep() wait_setup_teardown = 3 os.environ["ENJOLIVER_API_URI"] = api_uri os.environ["ENJOLIVER_MATCHBOX_PATH"] = test_matchbox_path os.environ["ENJOLIVER_MATCHBOX_ASSETS"] = assets_path os.environ[ "ENJOLIVER_KUBERNETES_APISERVER_INSECURE_BIND_ADDRESS"] = "0.0.0.0" ec = configs.EnjoliverConfig(importer=__file__) # Memory needed for RAM nodes ram_kvm_node_memory_mb = 9216 @staticmethod def pause(t=600): """ Sleep for eventual side testing or tests/s.sh ... :param t: 10 minutes :return: None """ try: display("==> sleep %d..." % t) time.sleep(t) except KeyboardInterrupt: pass finally: display("==> sleep finish") @staticmethod def process_target_matchbox(): os.environ[ "MATCHBOX_PATH"] = KernelVirtualMachinePlayer.test_matchbox_path cmd = [ "%s" % sys.executable, "%s/manage.py" % KernelVirtualMachinePlayer.project_path, "matchbox", ] display("PID -> %s\n" "exec -> %s" % (os.getpid(), " ".join(cmd))) sys.stdout.flush() os.environ["TERM"] = "xterm" os.execve(cmd[0], cmd, os.environ) @staticmethod def process_target_acserver(): cmd = [ "%s" % KernelVirtualMachinePlayer.acserver_bin, ] display("PID -> %s\n" "exec -> %s" % (os.getpid(), " ".join(cmd))) sys.stdout.flush() os.environ["TERM"] = "xterm" os.execve(cmd[0], cmd, os.environ) @staticmethod def process_target_api(): os.environ[ "ENJOLIVER_DB_PATH"] = "%s/enjoliver.sqlite" % KernelVirtualMachinePlayer.euid_path os.environ[ "ENJOLIVER_IGNITION_JOURNAL_DIR"] = "%s/ignition_journal" % KernelVirtualMachinePlayer.euid_path try: os.remove(os.environ["ENJOLIVER_DB_PATH"]) except OSError: pass shutil.rmtree(os.environ["ENJOLIVER_IGNITION_JOURNAL_DIR"], ignore_errors=True) try: with open("%s/.config/enjoliver/config.json" % os.getenv("HOME")) as f: conf = json.load(f) os.environ["ENJOLIVER_AWS_ACCESS_KEY_ID"] = conf[ "AWS_ACCESS_KEY_ID"] os.environ["ENJOLIVER_AWS_SECRET_ACCESS_KEY"] = conf[ "AWS_SECRET_ACCESS_KEY"] except (IOError, ValueError): pass os.environ["ENJOLIVER_BACKUP_BUCKET_NAME"] = "bbcenjoliver-dev" os.environ["ENJOLIVER_SYNC_NOTIFY_TTL"] = "0" cmd = [ "%s" % sys.executable, "%s/manage.py" % KernelVirtualMachinePlayer.project_path, "gunicorn", ] display("PID -> %s\n" "exec -> %s" % (os.getpid(), " ".join(cmd))) os.execve(cmd[0], cmd, os.environ) @staticmethod def process_target_dnsmasq(): cmd = [ "%s" % KernelVirtualMachinePlayer.rkt_bin, "--local-config=%s" % KernelVirtualMachinePlayer.tests_path, "--mount", "volume=config,target=/etc/dnsmasq.conf", "--mount", "volume=resolv,target=/etc/resolv.conf", "run", "enjoliver.local/dnsmasq:latest", "--insecure-options=all", "--net=host", "--interactive", "--caps-retain=all", "--set-env=TERM=%s" % os.getenv("TERM", "xterm"), "--uuid-file-save=/tmp/dnsmasq.uuid", "--volume", "resolv,kind=host,source=/etc/resolv.conf", "--volume", "config,kind=host,source=%s/dnsmasq-rack0.conf" % KernelVirtualMachinePlayer.tests_path ] display("PID -> %s\n" "exec -> %s" % (os.getpid(), " ".join(cmd))) sys.stdout.flush() os.execve(cmd[0], cmd, os.environ) os._exit(2) @staticmethod def fetch_lldpd(): cmd = [ "%s" % KernelVirtualMachinePlayer.rkt_bin, "--local-config=%s" % KernelVirtualMachinePlayer.tests_path, "fetch", "--insecure-options=all", KernelVirtualMachinePlayer.ec.lldp_image_url ] assert subprocess.call(cmd) == 0 @staticmethod def process_target_lldpd(): cmd = [ "%s" % KernelVirtualMachinePlayer.rkt_bin, "--local-config=%s" % KernelVirtualMachinePlayer.tests_path, "run", KernelVirtualMachinePlayer.ec.lldp_image_url, "--insecure-options=all", "--net=host", "--interactive", "--set-env=TERM=%s" % os.getenv("TERM", "xterm"), "--exec", "/usr/sbin/lldpd", "--", "-dd" ] display("PID -> %s\n" "exec -> %s" % (os.getpid(), " ".join(cmd))) sys.stdout.flush() os.execve(cmd[0], cmd, os.environ) os._exit(2) # Should not happen @staticmethod def dns_masq_running(): display("DNSMASQ probing...") sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) result = 1 for i in range(120): result = sock.connect_ex(('172.20.0.1', 53)) if result == 0: break time.sleep(0.5) if i % 10 == 0: display("DNSMASQ still NOT ready") sock.close() assert result == 0 display("DNSMASQ ready") sys.stdout.flush() @staticmethod def acserver_is_running(): url = "http://enjoliver.local" for t in range(20): try: r = requests.get(url) r.close() return except Exception as e: display(" GET -> %s : %s" % (url, e)) time.sleep(0.5) r = requests.get(url) r.close() @classmethod def running_requirements(cls): warnings.simplefilter("ignore", ResourceWarning) # TODO validate the assets in this method if os.geteuid() != 0: raise RuntimeError("Need to be root EUID == %d" % os.geteuid()) cls.clean_sandbox() if os.path.isfile(KernelVirtualMachinePlayer.rkt_bin) is False or \ os.path.isfile(KernelVirtualMachinePlayer.matchbox_bin) is False or \ os.path.isfile(KernelVirtualMachinePlayer.acserver_bin) is False: display("Call 'make runtime' as user for:\n" "- %s\n" % KernelVirtualMachinePlayer.rkt_bin + "- %s\n" % KernelVirtualMachinePlayer.matchbox_bin + "- %s\n" % KernelVirtualMachinePlayer.acserver_bin) exit(2) if os.path.isfile(cls.ssh_private_key) is False: display("Call 'make testing.id_rsa' as user\n") exit(3) display("PID -> %s" % os.getpid()) @classmethod def set_matchbox(cls): cls.p_matchbox = multiprocessing.Process( target=KernelVirtualMachinePlayer.process_target_matchbox, name="matchbox") cls.p_matchbox.start() time.sleep(0.5) assert cls.p_matchbox.is_alive() is True cls.p_list.append(cls.p_matchbox) @classmethod def set_rack0(cls): cmd = [ "%s" % KernelVirtualMachinePlayer.rkt_bin, "--local-config=%s" % KernelVirtualMachinePlayer.tests_path, "run", "--net=rack0", "--interactive", "--set-env=TERM=%s" % os.getenv("TERM", "xterm"), "--insecure-options=all", "coreos.com/rkt/stage1-coreos", "--exec", "/bin/bash", "--", "-c", "exit", "0" ] display("call %s" % " ".join(cmd)) ret = subprocess.call(cmd) display("Bridge w/ iptables creation exit: %d" % ret) assert subprocess.call(["ip", "link", "show", "rack0"]) == 0 @classmethod def set_dnsmasq(cls): cls.p_dnsmasq = multiprocessing.Process( target=KernelVirtualMachinePlayer.process_target_dnsmasq, name="dnsmasq") cls.p_dnsmasq.start() time.sleep(0.5) assert cls.p_dnsmasq.is_alive() is True cls.dns_masq_running() cls.p_list.append(cls.p_dnsmasq) @classmethod def set_api(cls): cls.p_api = multiprocessing.Process( target=KernelVirtualMachinePlayer.process_target_api, name="api") cls.p_api.start() time.sleep(0.5) assert cls.p_api.is_alive() is True cls.p_list.append(cls.p_api) @classmethod def set_acserver(cls): cls.p_acserver = multiprocessing.Process( target=KernelVirtualMachinePlayer.process_target_acserver, name="acserver") cls.p_acserver.start() time.sleep(1) if cls.p_acserver.is_alive() is True: cls.p_list.append(cls.p_acserver) @classmethod def set_lldp(cls): cls.fetch_lldpd() cls.p_lldp = multiprocessing.Process( target=KernelVirtualMachinePlayer.process_target_lldpd, name="lldp") cls.p_lldp.start() time.sleep(0.5) assert cls.p_lldp.is_alive() is True cls.p_list.append(cls.p_lldp) @classmethod def setUpClass(cls): raise NotImplementedError @classmethod def tearDownClass(cls): for p in cls.p_list: if p.is_alive(): display("TERM -> %s %s" % (p.pid, p.name)) p.terminate() p.join(10) if p.is_alive(): os.kill(p.pid, 9) display("END -> %s %s" % (p.exitcode, p.name)) display("EXITED -> %s %s" % (p.exitcode, p.name)) subprocess.call([ "%s" % KernelVirtualMachinePlayer.rkt_bin, "--local-config=%s" % KernelVirtualMachinePlayer.tests_path, "gc", "--grace-period=0s" ]) cls.clean_sandbox() cls.pause(cls.wait_setup_teardown) cls.write_ending(cls.__name__) subprocess.check_output(["reset", "-q"]) @staticmethod def write_ending(message): with open("/tmp/unittest.end", "a") as f: f.write("%s %s" % (datetime.datetime.now(), message)) @staticmethod def clean_sandbox(): dirs = [ "%s/%s" % (KernelVirtualMachinePlayer.test_matchbox_path, k) for k in ("profiles", "groups") ] for d in dirs: for f in os.listdir(d): if ".json" in f: display("-> remove %s" % f) os.remove("%s/%s" % (d, f)) for f in os.listdir( os.path.join(KernelVirtualMachinePlayer.tests_path, "test_certs")): if f != ".gitkeep": os.remove( os.path.join(KernelVirtualMachinePlayer.tests_path, "test_certs", f)) def api_healthz(self, first=True): """ If the api just respond on the route it's fine :param first: :return: """ try: request = requests.get("%s/healthz" % self.api_uri) response_body = request.content request.close() _ = json.loads(response_body.decode()) except Exception as e: display("%s %s" % (self.api_healthz.__name__, e)) if first is True: time.sleep(0.5) self.api_healthz(False) else: raise def setUp(self): subprocess.call(["reset", "-q"]) self.clean_sandbox() self.api_healthz() def clean_up_virtual_machine(self, name: str): for elt in [["virsh", "destroy", name], ["virsh", "undefine", name], [ "virsh", "vol-delete", "%s.qcow2" % name, "--pool", "default" ]]: self.virsh(elt) def create_virtual_machine(self, name: str, nb_node: int, disk_gb=0): if disk_gb == 0: disk_opt = "size=10" else: disk_opt = "size=%d" % disk_gb virt_install = [ "virt-install", "--name", "%s" % name, "--network=bridge:rack0,model=virtio", "--memory=%d" % self.get_optimized_memory(nb_node, disk_gb), "--vcpus=%d" % self.get_optimized_cpu(nb_node), "--cpu", "host", "--pxe", "--mac=%s" % self.generate_mac_from_name(name), "--disk", disk_opt, "--os-type=linux", "--os-variant=generic", "--noautoconsole", "--check=disk_size=off", "--boot=hd,network" ] return virt_install @staticmethod def generate_mac_from_name(name: str): """ :param name: virtual machine name :return: """ nb = int(re.match('.*-(\d)$', name).group(1)) + 1 if nb > 99: raise AssertionError( "machine number extracted from name is incoherent: %d %s" % (nb, name)) return "54:52:00:00:00:%02d" % nb @staticmethod def virsh(cmd, assertion=False, v=None): ret = subprocess.call(cmd, stdout=v, stderr=v) if assertion is True and ret != 0: raise RuntimeError("\"%s\"" % " ".join(cmd)) def fetch_discovery_interfaces(self): request = self.fetch_discovery() interfaces = [k["interfaces"] for k in request if request] return interfaces def fetch_discovery(self): request = requests.get("%s/discovery" % self.api_uri) response_body = request.content request.close() self.assertEqual(request.status_code, 200) disco_data = json.loads(response_body.decode()) return disco_data def fetch_discovery_ignition_journal(self, uuid: str): request = requests.get("%s/discovery/ignition-journal/%s" % (self.api_uri, uuid)) response_body = request.content request.close() self.assertEqual(request.status_code, 200) disco_data = json.loads(response_body.decode()) return disco_data def kvm_restart_off_machines(self, to_start: list, tries=120): self.assertIs(list, type(to_start)) self.assertGreater(len(to_start), 0) for j in range(tries): if len(to_start) == 0: break for i, m in enumerate(to_start): start = ["virsh", "start", "%s" % m] try: self.virsh(start, assertion=True), display("") to_start.pop(i) time.sleep(self.testing_sleep_seconds) except RuntimeError: # virsh raise this pass time.sleep(self.testing_sleep_seconds) self.assertEqual(len(to_start), 0) def etcd_endpoint_health(self, ips: list, port: int, tries=30, verify=True, certs_name=""): self.assertIs(list, type(ips)) self.assertGreater(len(ips), 0) certs = tuple() if certs_name: verify, certs = self._get_certificates(certs_name) for t in range(tries): if len(ips) == 0: break for i, ip in enumerate(ips): try: endpoint = "https://%s:%d/health" % (ip, port) request = requests.get(endpoint, verify=verify, cert=certs) response_body = json.loads(request.content.decode()) request.close() display("-> RESULT %s %s" % (endpoint, response_body)) sys.stdout.flush() if response_body == {u"health": u"true"}: ips.pop(i) display("-> REMAIN %s for %s" % (str(ips), self.etcd_endpoint_health.__name__)) continue except Exception as e: display(e) display( "-> %d/%d NOT READY %s:%d for %s" % (t, tries, ip, port, self.etcd_endpoint_health.__name__)) time.sleep(self.testing_sleep_seconds * 2) self.assertEqual(len(ips), 0) def _get_vault_uri_by_initier(self, ip: str, port: int, tries=30): vault_uri = "" for t in range(tries): try: endpoint = "https://%s:%d/v2/keys/initier" % (ip, port) request = requests.get(endpoint, verify=False) request.close() content = request.content vault_uri = json.loads(content.decode())["node"]["value"] display("-> RESULT %s: %s" % (endpoint, vault_uri)) sys.stdout.flush() break except Exception as e: display(e) display("-> %d/%d NOT READY initier %s:%d for %s" % (t, tries, ip, port, self.vault_self_certs.__name__)) time.sleep(self.testing_sleep_seconds * 2) self.assertGreater(len(vault_uri), 0) return vault_uri def vault_self_certs(self, ip, port, tries=30): vault_uri = self._get_vault_uri_by_initier(ip, port, tries) token_vault_server = self._get_vault_token_in_etcd( ip, port, "token/vault/server", tries) self._vault_issue_certificate("%s/v1/pki/vault/issue/server" % vault_uri, token_vault_server, verify=False, parent="vault", component="server") def _get_vault_token_in_etcd(self, ip: str, port: int, etcd_key: str, tries=30): token_vault_server = "" for t in range(tries): try: endpoint = "https://%s:%d/v2/keys/%s" % (ip, port, etcd_key) request = requests.get(endpoint, verify=False) content = request.content request.close() token_vault_server = json.loads( content.decode())["node"]["value"].replace("\n", "") break except Exception as e: display(e) display( "-> %d/%d NOT READY token %s %s:%d for %s" % (t, tries, etcd_key, ip, port, self.vault_self_certs.__name__)) time.sleep(self.testing_sleep_seconds * 2) self.assertGreater(len(token_vault_server), 0) return token_vault_server def _vault_issue_certificate(self, url: str, token: str, verify: bool, parent: str, component: str, tries=30): certs = ["certificate", "issuing_ca", "private_key"] content = dict() for t in range(tries): try: request = requests.post(url, headers={'X-Vault-Token': token}, verify=verify, data=json.dumps({ "common_name": "enjoliver.local", "ttl": "17520h", "ip_sans": "%s" % self.api_ip, })) request.close() content = json.loads(request.content.decode())["data"] break except Exception as e: display(e) time.sleep(self.testing_sleep_seconds * 2) for c in certs: filename_ext = "%s_%s.%s" % (parent, component, c) with open(os.path.join(self.test_certs_path, filename_ext), 'w') as f: f.write(content[c]) display("vault issue %s token: %s -> %s" % (url, token, filename_ext)) def vault_verifing_issuing_ca(self, ip: str, port: int): vault_uri = self._get_vault_uri_by_initier(ip, port, tries=2) r = requests.get("%s/v1/" % vault_uri, verify=os.path.join(self.test_certs_path, "vault_server.issuing_ca")) r.close() self.assertEqual(404, r.status_code) self.assertEqual({"errors": []}, json.loads(r.content.decode())) def vault_issue_app_certs(self, ip: str, port: int, tries=30): vault_uri = self._get_vault_uri_by_initier(ip, port, tries=2) for vault_cert in [(parent, component) for parent, component in [( "etcd-kubernetes", "client"), ( "etcd-fleet", "client"), ("kubernetes", "kube-apiserver"), ("kubernetes", "kubelet")]]: for t in range(tries): parent, component = vault_cert[0], vault_cert[1] try: token = self._get_vault_token_in_etcd( ip, port, "token/%s/%s" % (parent, component)) self._vault_issue_certificate( "%s/v1/pki/%s/issue/%s" % (vault_uri, parent, component), token, verify=os.path.join(self.test_certs_path, "vault_server.issuing_ca"), parent=parent, component=component) break except Exception as e: display(e) display("-> %d/%d NOT READY %s/%s %s for %s" % (t, tries, parent, component, ip, self.vault_self_certs.__name__)) time.sleep(self.testing_sleep_seconds) def healthz_enjoliver_agent(self, ips: list): health_list = [] for ip in ips: for t in range(10): try: req = requests.get("http://%s:8000/healthz" % ip) health = json.loads(req.content.decode()) req.close() self.assertEqual({}, health["Errors"]) health_list.append(health) break except Exception as e: display("fail: %s" % e) time.sleep(self.testing_sleep_seconds) self.assertEqual(len(ips), len(health_list)) def _get_certificates(self, certs_name: str): verify, certs = True, tuple() if certs_name: verify = os.path.join(self.test_certs_path, "%s.issuing_ca" % certs_name) certs = (os.path.join(self.test_certs_path, "%s.certificate" % certs_name), os.path.join(self.test_certs_path, "%s.private_key" % certs_name)) for c in certs: self.assertTrue(os.path.exists(c)) self.assertTrue(os.path.exists(verify)) return verify, certs def save_unseal_key(self, ips: list): unseal_file = os.path.join(self.test_certs_path, "unseal.key") for ip in ips: stdout = subprocess.check_output([ "ssh", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", "-o", "ConnectTimeout=1", "-i", self.ssh_private_key, "-lcore", ip, 'sudo grep "Unseal Key 1:" /etc/vault.d/keys | cut -f4 -d \' \'' ]).decode().replace("\n", "") if stdout: with open(unseal_file, "w") as f: f.write(stdout) break self.assertTrue(os.path.isfile(unseal_file)) def unseal_all_vaults(self, ips: list, tries=30): with open(os.path.join(self.test_certs_path, "unseal.key")) as f: key = f.read() self.assertGreater(len(key), 0) for ip in ips: for t in range(tries): url = "https://%s:8200/v1/sys/unseal" % ip try: request = requests.post(url, verify=os.path.join( self.test_certs_path, "vault_server.issuing_ca"), data=json.dumps({ "key": key, })) content = json.loads(request.content.decode()) request.close() self.assertFalse(content["sealed"]) request = requests.get( "https://%s:8200/v1/" % ip, verify=os.path.join(self.test_certs_path, "vault_server.issuing_ca"), ) content = json.loads(request.content.decode()) request.close() self.assertEqual([], content["errors"]) break except Exception as e: display(e) display("-> NOT READY %d/%d %s %s key: %s" % (t, tries, url, self.unseal_all_vaults.__name__, key)) self.assertFalse(t == tries - 1) time.sleep(self.testing_sleep_seconds * 2) def etcd_member_len(self, ip: str, members_nb: int, port: int, tries=30, verify=True, certs_name=""): result = {} certs = tuple() if certs_name: verify, certs = self._get_certificates(certs_name) for t in range(tries): try: endpoint = "https://%s:%d/v2/members" % (ip, port) request = requests.get(endpoint, verify=verify, cert=certs) content = request.content request.close() result = json.loads(content.decode()) display("-> RESULT %s %s" % (endpoint, result)) sys.stdout.flush() if len(result["members"]) == members_nb: break except Exception as e: display(e) display("-> %d/%d NOT READY %s:%d for %s" % (t, tries, ip, port, self.etcd_member_len.__name__)) time.sleep(self.testing_sleep_seconds * 2) self.assertEqual(len(result["members"]), members_nb) def kubernetes_node_nb(self, api_server_ip: str, nodes_nb: int, tries=200): c = kubeclient.ApiClient( host="%s:%d" % (api_server_ip, self.ec.kubernetes_apiserver_insecure_port)) core = kubeclient.CoreV1Api(c) items = [] for t in range(tries): try: nodes = core.list_node() if nodes and len(nodes.items) == nodes_nb: items = nodes.items break except Exception as e: display(e) display("-> %d/%d NOT READY %s for %s %d/%d" % (t, tries, api_server_ip, self.kubernetes_node_nb.__name__, len(items), nodes_nb)) time.sleep(self.testing_sleep_seconds) self.assertEqual(len(items), nodes_nb) def kube_apiserver_health(self, ips: list, tries=200): self.assertIs(list, type(ips)) self.assertGreater(len(ips), 0) for t in range(tries): if len(ips) == 0: break for i, ip in enumerate(ips): try: endpoint = "http://%s:%d/healthz" % ( ip, self.ec.kubernetes_apiserver_insecure_port) request = requests.get(endpoint) response_body = request.content request.close() display("-> RESULT %s %s" % (endpoint, response_body)) if response_body == b"ok": display( "## kubectl -s %s:%d get cs" % (ip, self.ec.kubernetes_apiserver_insecure_port)) ips.pop(i) display( "-> REMAIN %s for %s" % (str(ips), self.kube_apiserver_health.__name__)) continue except Exception as e: display(e) display( "-> %d/%d NOT READY %s for %s" % (t + 1, tries, ip, self.kube_apiserver_health.__name__)) time.sleep(self.testing_sleep_seconds) self.assertEqual(len(ips), 0) def create_tiller(self, api_server_ip: str): c = kubeclient.ApiClient( host="http://%s:%d" % (api_server_ip, self.ec.kubernetes_apiserver_insecure_port)) with open("%s/manifests/tiller/tiller-service.yaml" % self.euid_path) as f: service_manifest = yaml.load(f) with open("%s/manifests/tiller/tiller-deploy.yaml" % self.euid_path) as f: deploy_manifest = yaml.load(f) with open("%s/manifests/tiller/tiller-service-account.yaml" % self.euid_path) as f: serviceaccount_manifest = yaml.load(f) with open("%s/manifests/tiller/clusterrolebinding.yaml" % self.euid_path) as f: clusterrolebinding_manifest = yaml.load(f) core, beta = kubeclient.CoreV1Api(c), kubeclient.ExtensionsV1beta1Api( c) rbac = kubeclient.RbacAuthorizationV1beta1Api(c) core.create_namespaced_service("kube-system", service_manifest) rbac.create_cluster_role_binding(clusterrolebinding_manifest) core.create_namespaced_service_account("kube-system", serviceaccount_manifest) beta.create_namespaced_deployment("kube-system", deploy_manifest) def pod_tiller_is_running(self, api_server_ip: str, tries=100): code = 0 c = kubeclient.ApiClient( host="http://%s:%d" % (api_server_ip, self.ec.kubernetes_apiserver_insecure_port)) core = kubeclient.CoreV1Api(c) for t in range(tries): if code == 200: break try: r = core.list_namespaced_pod("kube-system", label_selector="app=tiller") for p in r.items: ip = p.status.pod_ip try: g = requests.get("http://%s:44135/liveness" % ip) code = g.status_code g.close() display( "-> RESULT %s %s for %s" % (ip, code, self.pod_tiller_is_running.__name__)) except Exception as e: display("-> %d/%d NOT READY %s for %s %s" % (t + 1, tries, ip, self.pod_tiller_is_running.__name__, e)) except ValueError: display("-> %d/%d NOT READY %s for %s" % (t + 1, tries, "ValueError", self.pod_tiller_is_running.__name__)) time.sleep(self.testing_sleep_seconds) self.assertEqual(200, code) def _tiller_is_gc(self, node_ip): output = subprocess.check_output([ "ssh", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", "-o", "ConnectTimeout=1", "-i", self.ssh_private_key, "-lcore", node_ip, 'sudo bash -c "/opt/bin/rkt l --no-legend | grep -c tiller"' ]) tiller_containers = int(output.decode().replace("\n", "")) return tiller_containers == 1 def tiller_can_restart(self, api_server_ip: str): c = kubeclient.ApiClient( host="http://%s:%d" % (api_server_ip, self.ec.kubernetes_apiserver_insecure_bind_address)) core = kubeclient.CoreV1Api(c) r = core.list_namespaced_pod("kube-system", label_selector="app=tiller") pod_ip, node_ip, req, tiller_endpoint = "", "", "", "" for p in r.items: pod_ip = p.status.pod_ip node_ip = p.status.host_ip try: req = "http://%s:44135/liveness" % pod_ip g = requests.get(req) g.close() self.assertEqual(200, g.status_code) tiller_endpoint = "%s:44134" % pod_ip display("-> tiller endpoint to kill: %s" % tiller_endpoint) subprocess.check_output([ "ssh", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", "-o", "ConnectTimeout=1", "-i", self.ssh_private_key, "-lcore", node_ip, 'sudo /usr/bin/pkill tiller' ]) break except Exception as e: display(e) with self.assertRaises(requests.ConnectionError): g = requests.get(req) g.close() ts = time.time() for i in range(10): try: new_tiller_endpoint = self._get_tiller_grpc_endpoint( api_server_ip=api_server_ip) display("-> new tiller endpoint: %s" % new_tiller_endpoint) break except Exception as e: display(e) time.sleep(1) new_tiller_endpoint = self._get_tiller_grpc_endpoint( api_server_ip=api_server_ip) self.assertNotEqual( self._get_tiller_grpc_endpoint(api_server_ip=api_server_ip), tiller_endpoint) display("-> polling tiller Pod %s during 70s or until its GC" % new_tiller_endpoint) while time.time() < ts + 120: try: loop_tiller_endpoint = self._get_tiller_grpc_endpoint( api_server_ip=api_server_ip) except RuntimeWarning: time.sleep(1) loop_tiller_endpoint = self._get_tiller_grpc_endpoint( api_server_ip=api_server_ip) self.assertEqual(new_tiller_endpoint, loop_tiller_endpoint) if self._tiller_is_gc(node_ip): return time.sleep(1) raise AssertionError("tiller is not gc on node %s" % node_ip) def _get_tiller_grpc_endpoint(self, api_server_ip: str): c = kubeclient.ApiClient( host="http://%s:%d" % (api_server_ip, self.ec.kubernetes_apiserver_insecure_port)) core = kubeclient.CoreV1Api(c) r = core.list_namespaced_pod("kube-system", label_selector="app=tiller") exception = None for p in r.items: ip = p.status.pod_ip try: g = requests.get("http://%s:44135/liveness" % ip) g.close() self.assertEqual(200, g.status_code) return "%s:44134" % ip except Exception as e: display(e) exception = e raise exception def create_helm_etcd_backup(self, api_server_ip: str, etcd_app_name: str): tiller = self._get_tiller_grpc_endpoint(api_server_ip) c = kubeclient.ApiClient( host="http://%s:%d" % (api_server_ip, self.ec.kubernetes_apiserver_insecure_port)) core = kubeclient.CoreV1Api(c) try: core.create_namespace( body={ "kind": "Namespace", "apiVersion": "v1", "metadata": { "name": "backup" } }) except Exception as e: self.assertEqual("Conflict", e.reason) ret = subprocess.call([ self.helm_bin, "--host", tiller, "install", "-f", "%s/manifests/etcd3-backup/%s.yaml" % (self.euid_path, etcd_app_name), "%s/manifests/etcd3-backup/" % self.euid_path ]) self.assertEqual(0, ret) def create_helm_by_name(self, api_server_ip: str, name: str): tiller = self._get_tiller_grpc_endpoint(api_server_ip) ret = subprocess.call([ self.helm_bin, "--host", tiller, "install", "%s/manifests/%s" % (self.euid_path, name) ]) self.assertEqual(0, ret) def _snapshot_status(self, core: kubeclient.CoreV1Api, etcd_app_name: str, tries: int): for t in range(tries): r = core.list_namespaced_pod("backup", label_selector="etcd=%s" % etcd_app_name) for p in r.items: ip = p.status.host_ip if p.status.phase != "Succeeded": display("%d/%d pod %s status.phase: %s" % (t, tries, p.metadata.name, p.status.phase)) continue try: stdout = subprocess.check_output([ "ssh", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", "-o", "ConnectTimeout=1", "-i", self.ssh_private_key, "-lcore", ip, 'sudo /opt/bin/etcdctl3 snapshot status /var/lib/backup/etcd3/%s.snap -w json' % etcd_app_name ]) return json.loads(stdout.decode()) except Exception as e: display(e) time.sleep(self.testing_sleep_seconds) def etcd_backup_done(self, api_server_ip: str, etcd_app_name: str, tries=120): c = kubeclient.ApiClient( host="%s:%d" % (api_server_ip, self.ec.kubernetes_apiserver_insecure_port)) core = kubeclient.CoreV1Api(c) summary = self._snapshot_status(core, etcd_app_name, tries) for k in ["revision", "totalKey", "totalSize"]: self.assertGreater(summary[k], 0) def daemonset_node_exporter_are_running(self, ips: list, tries=200): assert type(ips) is list assert len(ips) > 0 for t in range(tries): if len(ips) == 0: break for i, ip in enumerate(ips): try: g = requests.get("http://%s:9100" % ip) code = g.status_code g.close() display("-> RESULT %s %s" % (ip, code)) if code == 200: ips.pop(i) display("-> REMAIN %s for %s" % ( str(ips), self.daemonset_node_exporter_are_running.__name__)) continue except Exception as e: display(e) display("-> %d/%d NOT READY %s for %s" % (t + 1, tries, ip, self.daemonset_node_exporter_are_running.__name__)) time.sleep(self.testing_sleep_seconds) self.assertEqual(len(ips), 0) def get_optimized_memory(self, nb_nodes: int, disk: int): mem_bytes = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES') mem_gib = mem_bytes / (1024.**3) node_memory = (mem_gib // nb_nodes) * 1024 default_memory = self.ram_kvm_node_memory_mb * 0.8 if disk else self.ram_kvm_node_memory_mb return default_memory * 1.2 if node_memory > default_memory else default_memory @staticmethod def get_optimized_cpu(nb_nodes: int): cpu = float(multiprocessing.cpu_count()) cores = cpu / nb_nodes return int(round(cores)) def kubectl_proxy(self, proxy_port: int): def run(): cmd = [ "%s/hyperkube/hyperkube" % self.project_path, "kubectl", "--kubeconfig", os.path.join(self.tests_path, "testing_kubeconfig.yaml"), "proxy", "-p", "%d" % proxy_port ] display("-> %s" % " ".join(cmd)) os.execve(cmd[0], cmd, os.environ) return run def create_kubeconfig(self, api_server_ip: str): with open("%s/manifests/kubeconfig/clusterrolebinding.yaml" % self.euid_path) as f: clusterrolebinding_manifest = yaml.load(f) c = kubeclient.ApiClient( host="http://%s:%d" % (api_server_ip, self.ec.kubernetes_apiserver_insecure_port)) rbac = kubeclient.RbacAuthorizationV1beta1Api(c) rbac.create_cluster_role_binding(clusterrolebinding_manifest) kube_config = { 'preferences': { 'colors': True }, 'users': [{ 'user': { 'client-key': '%s/kubernetes_kubelet.private_key' % self.test_certs_path, 'client-certificate': '%s/kubernetes_kubelet.certificate' % self.test_certs_path }, 'name': 'enjoliver.local' }], 'kind': 'Config', 'apiVersion': 'v1', 'clusters': [{ 'cluster': { 'server': "https://%s:6443" % api_server_ip, 'certificate-authority': '%s/kubernetes_kubelet.issuing_ca' % self.test_certs_path }, 'name': 'enjoliver' }], 'contexts': [{ 'name': 'e', 'context': { 'cluster': 'enjoliver', 'namespace': 'kube-system', 'user': '******' } }], 'current-context': 'e' } with open(os.path.join(self.tests_path, "testing_kubeconfig.yaml"), "w") as kc: yaml.dump(kube_config, kc) def iteractive_usage(self, stop="/tmp/e.stop", api_server_ip=None, fns=None): display("-> Starting %s" % self.iteractive_usage.__name__) kp, proxy_port = None, 8001 if api_server_ip: self.create_kubeconfig(api_server_ip) kp = multiprocessing.Process(target=self.kubectl_proxy( proxy_port=proxy_port)) kp.start() maxi = 12 for i in range(maxi): if kp.is_alive(): try: r = requests.get("http://127.0.0.1:%d/healthz" % proxy_port) r.close() if r.status_code == 200: display("\n#####################################\n" "mkdir -pv ~/.kube\n" "cat << EOF >> ~/.kube/config\n" "apiVersion: v1\n" "clusters:\n" "- cluster:\n" " server: http://127.0.0.1:8001\n" " name: enjoliver\n" "contexts:\n" "- context:\n" " cluster: enjoliver\n" " namespace: default\n" " user: "******"\n" " name: e\n" "current-context: e\n" "kind: Config\n" "preferences:\n" " colors: true\n" "EOF\n" "kubectl config use-context e\n" "#####################################\n") break except Exception as e: display("-> %d/%d %s" % (i + 1, maxi, e)) time.sleep(0.5) with open(stop, "w") as f: f.write("") os.chmod(stop, 777) try: while os.path.isfile(stop) is True and os.stat(stop).st_size == 0: if fns: [fn() for fn in fns] if int(time.time()) % 120 == 0: display( "-> Stop with \"sudo rm -v\" %s or \"echo 1 > %s\"" % (stop, stop)) time.sleep(self.wait_setup_teardown) if api_server_ip and kp.is_alive(): kp.terminate() kp.join(timeout=5) finally: display("-> Stopping %s" % self.iteractive_usage.__name__) def replace_ignition_metadata(self, metadata, new_value): req = requests.get("%s/scheduler" % self.api_uri) scheduler = json.loads(req.content.decode()) req.close() for mac in scheduler: req = requests.post("%s/lifecycle/rolling/mac=%s" % (self.api_uri, mac)) req.close() for j in os.listdir("%s/groups/" % self.test_matchbox_path): if j == "discovery.json" or ".json" not in j: continue with open("%s/groups/%s" % (self.test_matchbox_path, j), 'r') as f: group = json.loads(f.read()) group["metadata"][metadata] = new_value with open("%s/groups/%s" % (self.test_matchbox_path, j), 'w') as f: json.dump(group, f, indent=4)
import datetime import time import unittest import ops from app import configs from app import crud from app import model from common import posts ec = configs.EnjoliverConfig() @unittest.skipIf(__name__ == "__main__", "") class TestModel(unittest.TestCase): @classmethod def setUpClass(cls): raise NotImplementedError @staticmethod def set_up_class_checks(smart): model.BASE.metadata.drop_all(smart.get_engine_connection()) model.BASE.metadata.create_all(smart.get_engine_connection()) with smart.new_session() as session: ops.health_check(session, time.time(), "unittest") # @unittest.skip("") def test_00(self): for _ in range(3): self.repositories.discovery.upsert(posts.M01) disco = self.repositories.discovery.fetch_all_discovery()
class TestValidateAcserverStorage(unittest.TestCase): cwd = os.path.dirname(os.path.abspath(__file__)) acserver_d = os.path.join(cwd, "runtime/acserver.d/enjoliver.local") ec = configs.EnjoliverConfig(importer=__file__) @staticmethod def format_image_url(image_url: str): image_url = image_url.replace("enjoliver.local/", "") image_url = image_url.replace(":", "-") return "%s-linux-amd64.aci" % image_url def test_cni(self): list_dir = os.listdir(os.path.join(self.acserver_d, "cni")) self.assertEqual(1, len(list_dir)) def test_etcd(self): list_dir = os.listdir(os.path.join(self.acserver_d, "etcd")) self.assertEqual(1, len(list_dir)) def test_fleet(self): list_dir = os.listdir(os.path.join(self.acserver_d, "fleet")) self.assertEqual(1, len(list_dir)) def test_hyperkube(self): list_dir = os.listdir(os.path.join(self.acserver_d, "hyperkube")) self.assertIn(self.format_image_url(self.ec.hyperkube_image_url), list_dir) self.assertEqual(1, len(list_dir)) def test_lldp(self): list_dir = os.listdir(os.path.join(self.acserver_d, "lldp")) self.assertIn(self.format_image_url(self.ec.lldp_image_url), list_dir) self.assertEqual(1, len(list_dir)) def test_iproute2(self): list_dir = os.listdir(os.path.join(self.acserver_d, "iproute2")) self.assertEqual(1, len(list_dir)) def test_rkt(self): list_dir = os.listdir(os.path.join(self.acserver_d, "rkt")) self.assertEqual(1, len(list_dir)) def test_vault(self): list_dir = os.listdir(os.path.join(self.acserver_d, "vault")) self.assertEqual(1, len(list_dir)) def test_ceph_tools(self): list_dir = os.listdir(os.path.join(self.acserver_d, "ceph-tools")) self.assertIn(self.format_image_url(self.ec.cephtools_image_url), list_dir) self.assertEqual(1, len(list_dir)) def test_dnsmasq(self): list_dir = os.listdir(os.path.join(self.acserver_d, "dnsmasq")) self.assertEqual(1, len(list_dir)) def test_tiller(self): list_dir = os.listdir(os.path.join(self.acserver_d, "tiller")) self.assertEqual(1, len(list_dir)) def test_heapster(self): list_dir = os.listdir(os.path.join(self.acserver_d, "heapster")) self.assertEqual(1, len(list_dir)) def test_node_exporter(self): list_dir = os.listdir(os.path.join(self.acserver_d, "node-exporter")) self.assertEqual(1, len(list_dir)) def test_kube_state_metrics(self): list_dir = os.listdir( os.path.join(self.acserver_d, "kube-state-metrics")) self.assertEqual(1, len(list_dir)) def test_prometheus(self): list_dir = os.listdir(os.path.join(self.acserver_d, "prometheus")) self.assertEqual(1, len(list_dir)) def test_haproxy(self): list_dir = os.listdir(os.path.join(self.acserver_d, "haproxy")) self.assertEqual(1, len(list_dir))
import json import os import shutil import sys import time import unittest from multiprocessing import Process import requests from app import api from app import configs from app import generator from common import posts ec = configs.EnjoliverConfig(importer=__file__) class TestAPI(unittest.TestCase): p_matchbox = Process int_path = "%s" % os.path.dirname(__file__) dbs_path = "%s/dbs" % int_path tests_path = "%s" % os.path.dirname(int_path) app_path = os.path.dirname(tests_path) project_path = os.path.dirname(app_path) matchbox_path = "%s/matchbox" % project_path assets_path = "%s/matchbox/assets" % project_path test_matchbox_path = "%s/test_matchbox" % tests_path
if __name__ == '__main__': parser = argparse.ArgumentParser(prog='Enjoliver') parser.add_argument( 'task', type=str, choices=["gunicorn", "plan", "matchbox", "show-configs", "validate"], help="Choose the task to run") parser.add_argument('--configs', type=str, default="%s/configs.yaml" % APP_PATH, help="Choose the yaml config file") task = parser.parse_args().task f = parser.parse_args().configs ec = configs.EnjoliverConfig(yaml_full_path=f, importer=__file__) if task == "gunicorn": init_db(ec) init_journal_dir(ec) gunicorn(ec) elif task == "plan": plan(ec) elif task == "matchbox": matchbox(ec) elif task == "show-configs": show_configs(ec) elif task == "validate": validate() else: raise AttributeError("%s not a choice" % task)