def plan():
ec = configs.EnjoliverConfig(importer=__file__)
cmd = [
'python',
"%s/enjoliver/k8s_2t.py" % APP_PATH,
]
click.echo("exec[%s] -> %s\n" % (os.getpid(), " ".join(cmd)))
with open(ec.plan_pid_file, "w") as f:
f.write("%d" % os.getpid())
os.execvpe(cmd[0], cmd, os.environ)
def matchbox():
ec = configs.EnjoliverConfig(importer=__file__)
cmd = [
"%s/runtime/matchbox/matchbox" % PROJECT_PATH,
"-address",
ec.matchbox_uri.replace("https://", "").replace("http://", ""),
"-assets-path",
"%s" % ec.matchbox_assets,
"-data-path",
"%s" % ec.matchbox_path,
"-log-level",
ec.matchbox_logging_level.lower(),
]
click.echo("exec[%s] -> %s\n" % (os.getpid(), " ".join(cmd)))
with open(ec.matchbox_pid_file, "w") as f:
f.write("%d" % os.getpid())
os.execve(cmd[0], cmd, os.environ)
def setUpClass(cls):
cls.ec = configs.EnjoliverConfig(importer=__file__)
app = create_app('EnjoliverTest', ec=cls.ec)
app.testing = True
cls.engine = create_engine(
'postgresql+psycopg2://localhost/enjoliver_testing')
cls.init_db()
sess_maker = sessionmaker(bind=cls.engine)
registry = RepositoryRegistry(sess_maker=sess_maker)
register_routes(app=app,
ec=cls.ec,
cache=SimpleCache(),
sess_maker=sess_maker,
registry=registry)
cls.app = app.test_client()
def gunicorn():
ec = configs.EnjoliverConfig(importer=__file__)
_init_db(ec)
_init_journal_dir(ec)
cmd = [
"gunicorn",
"--chdir",
APP_PATH,
"enjoliver.api:gunicorn()",
"--worker-class",
ec.gunicorn_worker_type,
"-b",
ec.gunicorn_bind,
"--log-level",
ec.logging_level.lower(),
"-w",
"%s" % ec.gunicorn_workers,
"-c",
gunicorn_conf.__file__
]
if not os.getenv('prometheus_multiproc_dir', None):
os.environ["prometheus_multiproc_dir"] = ec.prometheus_multiproc_dir
_fs_gunicorn_cleaning(ec)
p = multiprocessing.Process(target=lambda: os.execvpe(cmd[0], cmd, os.environ))
def stop(signum, frame):
click.echo("terminating %d" % p.pid)
p.terminate()
click.echo("starting gunicorn: %s" % " ".join(cmd))
p.start()
with open(ec.gunicorn_pid_file, "w") as f:
f.write("%d" % p.pid)
for sig in [signal.SIGINT, signal.SIGTERM]:
signal.signal(sig, stop)
p.join()
_fs_gunicorn_cleaning(ec)
class KernelVirtualMachinePlayer(unittest.TestCase):
"""
This class is used by all Kernel Virtual Machine testing suite
Override the setUpClass by selecting your custom environment with the following catalog:
>>> @classmethod
>>> def setUpClass(cls):
>>> cls.running_requirements()
>>> cls.set_acserver()
>>> cls.set_api()
>>> cls.set_matchbox()
>>> cls.set_dnsmasq()
>>> cls.set_lldp()
>>> cls.set_rack0()
>>> cls.pause(cls.wait_setup_teardown)
Note: you may use 'reset -q' because of Link Layer Discovery Protocol Container's
"""
__name__ = "KernelVirtualMachinePlayer"
p_matchbox = multiprocessing.Process
p_dnsmasq = multiprocessing.Process
p_api = multiprocessing.Process
p_lldp = multiprocessing.Process
p_list = []
gen = generator.Generator
euid_path = "%s" % os.path.dirname(os.path.abspath(__file__))
tests_path = "%s" % os.path.dirname(euid_path)
project_path = os.path.dirname(tests_path)
matchbox_path = "%s/matchbox" % project_path
assets_path = "%s/matchbox/assets" % project_path
runtime_path = "%s/runtime" % project_path
rkt_bin = "%s/rkt/rkt" % runtime_path
helm_bin = "%s/helm/helm" % runtime_path
matchbox_bin = "%s/matchbox/matchbox" % runtime_path
acserver_bin = "%s/run_acserver.py" % runtime_path
ssh_private_key = os.path.join(tests_path, "testing.id_rsa")
test_certs_path = "%s/test_certs" % tests_path
test_matchbox_path = "%s/test_matchbox" % tests_path
matchbox_port = int(os.getenv("MATCHBOX_PORT", "8080"))
api_port = int(os.getenv("API_PORT", "5000"))
api_ip = "172.20.0.1"
api_uri = "http://%s:%d" % (api_ip, api_port)
dev_null = open("/dev/null", "w")
testing_sleep_seconds = get_kvm_sleep()
wait_setup_teardown = 3
os.environ["ENJOLIVER_API_URI"] = api_uri
os.environ["ENJOLIVER_MATCHBOX_PATH"] = test_matchbox_path
os.environ["ENJOLIVER_MATCHBOX_ASSETS"] = assets_path
os.environ["ENJOLIVER_KUBERNETES_APISERVER_INSECURE_BIND_ADDRESS"] = "0.0.0.0"
ec = configs.EnjoliverConfig(importer=__file__)
# Memory needed for RAM nodes
ram_kvm_node_memory_mb = 9216
@staticmethod
def pause(t=600):
"""
Sleep for eventual side testing or tests/s.sh ...
:param t: 10 minutes
:return: None
"""
try:
display("==> sleep %d..." % t)
time.sleep(t)
except KeyboardInterrupt:
pass
finally:
display("==> sleep finish")
@staticmethod
def process_target_matchbox():
os.environ["MATCHBOX_PATH"] = KernelVirtualMachinePlayer.test_matchbox_path
cmd = [
"%s" % sys.executable,
"%s/manage.py" % KernelVirtualMachinePlayer.project_path,
"matchbox",
]
display("PID -> %s\n"
"exec -> %s" % (os.getpid(), " ".join(cmd)))
sys.stdout.flush()
os.environ["TERM"] = "xterm"
os.execve(cmd[0], cmd, os.environ)
@staticmethod
def process_target_acserver():
cmd = [
"%s" % KernelVirtualMachinePlayer.acserver_bin,
]
display("PID -> %s\n"
"exec -> %s" % (os.getpid(), " ".join(cmd)))
sys.stdout.flush()
os.environ["TERM"] = "xterm"
os.execve(cmd[0], cmd, os.environ)
@staticmethod
def process_target_api():
os.environ["ENJOLIVER_IGNITION_JOURNAL_DIR"] = "%s/ignition_journal" % KernelVirtualMachinePlayer.euid_path
shutil.rmtree(os.environ["ENJOLIVER_IGNITION_JOURNAL_DIR"], ignore_errors=True)
# XXX: tests require with an empty db
subprocess.run("sudo -u postgres psql -c 'drop database enjoliver_local;'", shell=True)
subprocess.run("sudo -u postgres psql -c 'create database enjoliver_local;'", shell=True)
os.environ["ENJOLIVER_SYNC_NOTIFY_TTL"] = "0"
cmd = [
"%s" % sys.executable,
"%s/manage.py" % KernelVirtualMachinePlayer.project_path,
"gunicorn",
]
display("PID -> %s\n"
"exec -> %s" % (os.getpid(), " ".join(cmd)))
os.execve(cmd[0], cmd, os.environ)
@staticmethod
def process_target_dnsmasq():
cmd = [
"%s" % KernelVirtualMachinePlayer.rkt_bin,
"--local-config=%s" % KernelVirtualMachinePlayer.tests_path,
"--mount",
"volume=config,target=/etc/dnsmasq.conf",
"--mount",
"volume=resolv,target=/etc/resolv.conf",
"run",
"enjoliver.local/dnsmasq:latest",
"--insecure-options=all",
"--net=host",
"--interactive",
"--caps-retain=all",
"--set-env=TERM=%s" % os.getenv("TERM", "xterm"),
"--uuid-file-save=/tmp/dnsmasq.uuid",
"--volume",
"resolv,kind=host,source=/etc/resolv.conf",
"--volume",
"config,kind=host,source=%s/dnsmasq-rack0.conf" % KernelVirtualMachinePlayer.tests_path
]
display("PID -> %s\n"
"exec -> %s" % (os.getpid(), " ".join(cmd)))
sys.stdout.flush()
os.execve(cmd[0], cmd, os.environ)
os._exit(2)
@staticmethod
def fetch_lldpd():
cmd = [
"%s" % KernelVirtualMachinePlayer.rkt_bin,
"--local-config=%s" % KernelVirtualMachinePlayer.tests_path,
"fetch",
"--insecure-options=all",
KernelVirtualMachinePlayer.ec.lldp_image_url]
assert subprocess.call(cmd) == 0
@staticmethod
def process_target_lldpd():
cmd = [
"%s" % KernelVirtualMachinePlayer.rkt_bin,
"--local-config=%s" % KernelVirtualMachinePlayer.tests_path,
"run",
KernelVirtualMachinePlayer.ec.lldp_image_url,
"--insecure-options=all",
"--net=host",
"--interactive",
"--set-env=TERM=%s" % os.getenv("TERM", "xterm"),
"--exec",
"/usr/sbin/lldpd",
"--",
"-dd"]
display("PID -> %s\n"
"exec -> %s" % (os.getpid(), " ".join(cmd)))
sys.stdout.flush()
os.execve(cmd[0], cmd, os.environ)
os._exit(2) # Should not happen
@staticmethod
def dns_masq_running():
display("DNSMASQ probing...")
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
result = 1
for i in range(120):
result = sock.connect_ex(('172.20.0.1', 53))
if result == 0:
break
time.sleep(0.5)
if i % 10 == 0:
display("DNSMASQ still NOT ready")
sock.close()
assert result == 0
display("DNSMASQ ready")
sys.stdout.flush()
@staticmethod
def acserver_is_running():
url = "http://enjoliver.local"
for t in range(20):
try:
r = requests.get(url)
r.close()
return
except Exception as e:
display(" GET -> %s : %s" % (url, e))
time.sleep(0.5)
r = requests.get(url)
r.close()
@classmethod
def running_requirements(cls):
warnings.simplefilter("ignore", ResourceWarning)
# TODO validate the assets in this method
if os.geteuid() != 0:
raise RuntimeError("Need to be root EUID == %d" % os.geteuid())
cls.clean_sandbox()
if os.path.isfile(KernelVirtualMachinePlayer.rkt_bin) is False or \
os.path.isfile(KernelVirtualMachinePlayer.matchbox_bin) is False or \
os.path.isfile(KernelVirtualMachinePlayer.acserver_bin) is False:
display("Call 'make runtime' as user for:\n"
"- %s\n" % KernelVirtualMachinePlayer.rkt_bin +
"- %s\n" % KernelVirtualMachinePlayer.matchbox_bin +
"- %s\n" % KernelVirtualMachinePlayer.acserver_bin)
exit(2)
if os.path.isfile(cls.ssh_private_key) is False:
display("Call 'make testing.id_rsa' as user\n")
exit(3)
display("PID -> %s" % os.getpid())
@classmethod
def set_matchbox(cls):
cls.p_matchbox = multiprocessing.Process(target=KernelVirtualMachinePlayer.process_target_matchbox,
name="matchbox")
cls.p_matchbox.start()
time.sleep(0.5)
assert cls.p_matchbox.is_alive() is True
cls.p_list.append(cls.p_matchbox)
@classmethod
def set_rack0(cls):
cmd = [
"%s" % KernelVirtualMachinePlayer.rkt_bin,
"--local-config=%s" % KernelVirtualMachinePlayer.tests_path,
"run",
"--net=rack0",
"--interactive",
"--set-env=TERM=%s" % os.getenv("TERM", "xterm"),
"--insecure-options=all",
"coreos.com/rkt/stage1-coreos",
"--exec",
"/bin/bash", "--", "-c", "exit", "0"]
display("call %s" % " ".join(cmd))
ret = subprocess.call(cmd)
display("Bridge w/ iptables creation exit: %d" % ret)
assert subprocess.call(["ip", "link", "show", "rack0"]) == 0
@classmethod
def set_dnsmasq(cls):
cls.p_dnsmasq = multiprocessing.Process(target=KernelVirtualMachinePlayer.process_target_dnsmasq,
name="dnsmasq")
cls.p_dnsmasq.start()
time.sleep(0.5)
assert cls.p_dnsmasq.is_alive() is True
cls.dns_masq_running()
cls.p_list.append(cls.p_dnsmasq)
@classmethod
def set_api(cls):
cls.p_api = multiprocessing.Process(target=KernelVirtualMachinePlayer.process_target_api, name="api")
cls.p_api.start()
time.sleep(5)
assert cls.p_api.is_alive() is True
cls.p_list.append(cls.p_api)
@classmethod
def set_acserver(cls):
cls.p_acserver = multiprocessing.Process(target=KernelVirtualMachinePlayer.process_target_acserver,
name="acserver")
cls.p_acserver.start()
time.sleep(1)
if cls.p_acserver.is_alive() is True:
cls.p_list.append(cls.p_acserver)
@classmethod
def set_lldp(cls):
cls.fetch_lldpd()
cls.p_lldp = multiprocessing.Process(target=KernelVirtualMachinePlayer.process_target_lldpd, name="lldp")
cls.p_lldp.start()
time.sleep(0.5)
assert cls.p_lldp.is_alive() is True
cls.p_list.append(cls.p_lldp)
@classmethod
def setUpClass(cls):
raise NotImplementedError
@classmethod
def tearDownClass(cls):
for p in cls.p_list:
if p.is_alive():
display("TERM -> %s %s" % (p.pid, p.name))
p.terminate()
p.join(10)
if p.is_alive():
os.kill(p.pid, 9)
display("END -> %s %s" % (p.exitcode, p.name))
display("EXITED -> %s %s" % (p.exitcode, p.name))
subprocess.call([
"%s" % KernelVirtualMachinePlayer.rkt_bin,
"--local-config=%s" % KernelVirtualMachinePlayer.tests_path,
"gc",
"--grace-period=0s"])
cls.clean_sandbox()
cls.pause(cls.wait_setup_teardown)
cls.write_ending(cls.__name__)
#subprocess.check_output(["reset", "-q"])
@staticmethod
def write_ending(message):
with open("/tmp/unittest.end", "a") as f:
f.write("%s %s" % (datetime.datetime.now(), message))
@staticmethod
def clean_sandbox():
dirs = ["%s/%s" % (KernelVirtualMachinePlayer.test_matchbox_path, k)
for k in ("profiles", "groups")]
for d in dirs:
for f in os.listdir(d):
if ".json" in f:
display("-> remove %s" % f)
os.remove("%s/%s" % (d, f))
for f in os.listdir(os.path.join(KernelVirtualMachinePlayer.tests_path, "test_certs")):
if f != ".gitkeep":
os.remove(os.path.join(KernelVirtualMachinePlayer.tests_path, "test_certs", f))
def api_healthz(self, first=True):
"""
If the api just respond on the route it's fine
:param first:
:return:
"""
try:
request = requests.get("%s/healthz" % self.api_uri)
response_body = request.content
request.close()
_ = json.loads(response_body.decode())
except Exception as e:
display("%s %s" % (self.api_healthz.__name__, e))
if first is True:
time.sleep(0.5)
self.api_healthz(False)
else:
raise
def setUp(self):
#subprocess.call(["reset", "-q"])
self.clean_sandbox()
self.api_healthz()
def clean_up_virtual_machine(self, name: str):
pass
for elt in [["virsh", "destroy", name], ["virsh", "undefine", name],
["virsh", "vol-delete", "%s.qcow2" % name, "--pool", "default"]]:
self.virsh(elt)
def create_virtual_machine(self, name: str, nb_node: int, disk_gb=0):
if disk_gb == 0:
disk_opt = "size=10"
else:
disk_opt = "size=%d" % disk_gb
virt_install = [
"virt-install",
"--name",
"%s" % name,
"--network=bridge:rack0,model=virtio",
"--memory=%d" % self.get_optimized_memory(nb_node, disk_gb),
"--vcpus=%d" % self.get_optimized_cpu(nb_node),
"--cpu",
"host",
"--pxe",
"--mac=%s" % self.generate_mac_from_name(name),
"--disk",
disk_opt,
"--os-type=linux",
"--os-variant=generic",
"--noautoconsole",
"--check=disk_size=off",
"--boot=hd,network"
]
return virt_install
@staticmethod
def generate_mac_from_name(name: str):
"""
:param name: virtual machine name
:return:
"""
nb = int(re.match('.*-(\d)$', name).group(1)) + 1
if nb > 99:
raise AssertionError("machine number extracted from name is incoherent: %d %s" % (nb, name))
return "54:52:00:00:00:%02d" % nb
@staticmethod
def virsh(cmd, assertion=False, v=None):
ret = subprocess.call(cmd, stdout=v, stderr=v)
if assertion is True and ret != 0:
raise RuntimeError("\"%s\"" % " ".join(cmd))
def fetch_discovery_interfaces(self):
request = self.fetch_discovery()
interfaces = [k["interfaces"] for k in request if request]
return interfaces
def fetch_discovery(self):
request = requests.get("%s/discovery" % self.api_uri)
response_body = request.content
request.close()
self.assertEqual(request.status_code, 200)
disco_data = json.loads(response_body.decode())
return disco_data
def fetch_discovery_ignition_journal(self, uuid: str):
request = requests.get("%s/discovery/ignition-journal/%s" % (self.api_uri, uuid))
response_body = request.content
request.close()
self.assertEqual(request.status_code, 200)
disco_data = json.loads(response_body.decode())
return disco_data
def kvm_restart_off_machines(self, to_start: list, tries=120):
self.assertIs(list, type(to_start))
self.assertGreater(len(to_start), 0)
for j in range(tries):
if len(to_start) == 0:
break
for i, m in enumerate(to_start):
start = ["virsh", "start", "%s" % m]
try:
self.virsh(start, assertion=True), display("")
to_start.pop(i)
time.sleep(self.testing_sleep_seconds)
except RuntimeError:
# virsh raise this
pass
time.sleep(self.testing_sleep_seconds)
self.assertEqual(len(to_start), 0)
def assertFuncTrueInLessThanXTries(self, func: typing.Callable, tries: int=30, *args, **kwargs):
"""
assert a function returned True in less than x tries.
:param func: the func to test
:param tries: the max number of tries
:param args: the args to func
:param kwargs: the kwargs to func
"""
for step in range(tries):
display("-> TRY: {}/{} {}".format(step+1, tries, func))
if func(*args, **kwargs):
return
time.sleep(self.testing_sleep_seconds * 2)
raise AssertionError('{}() was called more than {} times, and none of these calls returned True'.format(
func, tries
))
def etcd_endpoint_health(self, ips: list, port: int, tries: int=30, verify=True, certs_name=""):
"""
assert health is true for all nodes whose IP address is in ips.
:param ips: the ips to check
:param port: the TCP port where to query the status
:param tries: the number of tries before failing
:param verify: should we verify the SSL CA or not
:param certs_name: the CA name
:return: True or False
"""
self.assertIs(list, type(ips))
self.assertGreater(len(ips), 0)
certs = ()
if certs_name:
verify, certs = self._get_certificates(certs_name)
self.assertFuncTrueInLessThanXTries(
self._etcd_endpoint_health,
tries,
ips=ips,
port=port,
verify=verify,
certs=certs,
)
def _etcd_endpoint_health(self, ips: list, port: int, verify, certs):
"""
return True if health is true for all nodes whose IP address is in ips.
:param ips: the ips to check
:param port: the TCP port where to query the status
:param verify: passed to requests lib
:param certs: passed to requests lib
:return: True or False
"""
healths = {}
for i, ip in enumerate(ips):
try:
endpoint = "https://%s:%d/health" % (ip, port)
request = requests.get(endpoint, verify=verify, cert=certs)
response_body = json.loads(request.content.decode())
request.close()
sys.stdout.flush()
healths[ip] = response_body == {u"health": u"true"}
except Exception as e:
healths[ip] = False
display(e)
if not healths[ip]:
display("-> NOT READY %s:%d for %s" % (
ip, port, self._etcd_endpoint_health.__name__
))
return False not in healths.values()
def _get_vault_uri_by_initier(self, ip: str, port: int, tries=30):
vault_uri = ""
for t in range(tries):
try:
endpoint = "https://%s:%d/v2/keys/initier" % (ip, port)
request = requests.get(endpoint, verify=False)
request.close()
content = request.content
vault_uri = json.loads(content.decode())["node"]["value"]
display("-> RESULT %s: %s" % (endpoint, vault_uri))
sys.stdout.flush()
break
except Exception as e:
display(e)
display(
"-> %d/%d NOT READY initier %s:%d for %s" % (t, tries, ip, port, self.vault_self_certs.__name__))
time.sleep(self.testing_sleep_seconds * 2)
self.assertGreater(len(vault_uri), 0)
return vault_uri
def vault_self_certs(self, ip, port, tries=30):
vault_uri = self._get_vault_uri_by_initier(ip, port, tries)
token_vault_server = self._get_vault_token_in_etcd(ip, port, "token/vault/server", tries)
self._vault_issue_certificate(
"%s/v1/pki/vault/issue/server" % vault_uri, token_vault_server, verify=False, parent="vault",
component="server")
def _get_vault_token_in_etcd(self, ip: str, port: int, etcd_key: str, tries=30):
token_vault_server = ""
for t in range(tries):
try:
endpoint = "https://%s:%d/v2/keys/%s" % (ip, port, etcd_key)
request = requests.get(endpoint, verify=False)
content = request.content
request.close()
token_vault_server = json.loads(content.decode())["node"]["value"].replace("\n", "")
break
except Exception as e:
display(e)
display("-> %d/%d NOT READY token %s %s:%d for %s" % (
t, tries, etcd_key, ip, port, self.vault_self_certs.__name__))
time.sleep(self.testing_sleep_seconds * 2)
self.assertGreater(len(token_vault_server), 0)
return token_vault_server
def _vault_issue_certificate(self, url: str, token: str, verify: bool, parent: str, component: str, tries=30):
certs = ["certificate", "issuing_ca", "private_key"]
content = dict()
for t in range(tries):
try:
request = requests.post(
url,
headers={'X-Vault-Token': token},
verify=verify,
data=json.dumps({
"common_name": "enjoliver.local",
"ttl": "17520h",
"ip_sans": "%s" % self.api_ip,
}))
request.close()
content = json.loads(request.content.decode())["data"]
break
except Exception as e:
display(e)
time.sleep(self.testing_sleep_seconds * 2)
for c in certs:
filename_ext = "%s_%s.%s" % (parent, component, c)
with open(os.path.join(self.test_certs_path, filename_ext), 'w') as f:
f.write(content[c])
display("vault issue %s token: %s -> %s" % (url, token, filename_ext))
def vault_verifing_issuing_ca(self, ip: str, port: int):
vault_uri = self._get_vault_uri_by_initier(ip, port, tries=2)
r = requests.get("%s/v1/" % vault_uri,
verify=os.path.join(self.test_certs_path, "vault_server.issuing_ca"))
r.close()
self.assertEqual(404, r.status_code)
self.assertEqual({"errors": []}, json.loads(r.content.decode()))
def vault_issue_app_certs(self, ip: str, port: int, tries=30):
vault_uri = self._get_vault_uri_by_initier(ip, port, tries=2)
for vault_cert in [(parent, component) for parent, component in [
("etcd-kubernetes", "client"),
("etcd-fleet", "client"),
("kubernetes", "kube-apiserver"),
("kubernetes", "kubelet")
]]:
for t in range(tries):
parent, component = vault_cert[0], vault_cert[1]
try:
token = self._get_vault_token_in_etcd(ip, port, "token/%s/%s" % (parent, component))
self._vault_issue_certificate(
"%s/v1/pki/%s/issue/%s" % (vault_uri, parent, component),
token,
verify=os.path.join(self.test_certs_path, "vault_server.issuing_ca"),
parent=parent,
component=component
)
break
except Exception as e:
display(e)
display("-> %d/%d NOT READY %s/%s %s for %s" % (
t, tries, parent, component, ip, self.vault_self_certs.__name__))
time.sleep(self.testing_sleep_seconds)
def healthz_enjoliver_agent(self, ips: list):
health_list = []
for ip in ips:
for t in range(10):
try:
req = requests.get("http://%s:8000/healthz" % ip)
health = json.loads(req.content.decode())
req.close()
self.assertEqual({}, health["Errors"])
health_list.append(health)
break
except Exception as e:
display("fail: %s" % e)
time.sleep(self.testing_sleep_seconds)
self.assertEqual(len(ips), len(health_list))
def _get_certificates(self, certs_name: str):
verify, certs = True, tuple()
if certs_name:
verify = os.path.join(self.test_certs_path, "%s.issuing_ca" % certs_name)
certs = (
os.path.join(self.test_certs_path, "%s.certificate" % certs_name),
os.path.join(self.test_certs_path, "%s.private_key" % certs_name)
)
for c in certs:
self.assertTrue(os.path.exists(c))
self.assertTrue(os.path.exists(verify))
return verify, certs
def save_unseal_key(self, ips: list):
unseal_file = os.path.join(self.test_certs_path, "unseal.key")
for ip in ips:
stdout = subprocess.check_output([
"ssh", "-o", "StrictHostKeyChecking=no",
"-o", "UserKnownHostsFile=/dev/null",
"-o", "ConnectTimeout=1",
"-i", self.ssh_private_key,
"-lcore", ip, 'sudo grep "Unseal Key 1:" /etc/vault.d/keys | cut -f4 -d \' \''
]).decode().replace("\n", "")
if stdout:
with open(unseal_file, "w") as f:
f.write(stdout)
break
self.assertTrue(os.path.isfile(unseal_file))
def unseal_all_vaults(self, ips: list, tries=30):
with open(os.path.join(self.test_certs_path, "unseal.key")) as f:
key = f.read()
self.assertGreater(len(key), 0)
for ip in ips:
for t in range(tries):
url = "https://%s:8200/v1/sys/unseal" % ip
try:
request = requests.post(
url,
verify=os.path.join(self.test_certs_path, "vault_server.issuing_ca"),
data=json.dumps({
"key": key,
}))
content = json.loads(request.content.decode())
request.close()
self.assertFalse(content["sealed"])
request = requests.get(
"https://%s:8200/v1/" % ip,
verify=os.path.join(self.test_certs_path, "vault_server.issuing_ca"),
)
content = json.loads(request.content.decode())
request.close()
self.assertEqual([], content["errors"])
break
except Exception as e:
display(e)
display("-> NOT READY %d/%d %s %s key: %s" % (
t, tries, url, self.unseal_all_vaults.__name__, key))
self.assertFalse(t == tries - 1)
time.sleep(self.testing_sleep_seconds * 2)
def etcd_member_len(self, ip: str, members_nb: int, port: int, tries=30, verify=True, certs_name=""):
result = {}
certs = tuple()
if certs_name:
verify, certs = self._get_certificates(certs_name)
for t in range(tries):
try:
endpoint = "https://%s:%d/v2/members" % (ip, port)
request = requests.get(endpoint, verify=verify, cert=certs)
content = request.content
request.close()
result = json.loads(content.decode())
display("-> RESULT %s %s" % (endpoint, result))
sys.stdout.flush()
if len(result["members"]) == members_nb:
break
except Exception as e:
display(e)
display("-> %d/%d NOT READY %s:%d for %s" % (t, tries, ip, port, self.etcd_member_len.__name__))
time.sleep(self.testing_sleep_seconds * 2)
self.assertEqual(len(result["members"]), members_nb)
def kubernetes_node_nb(self, api_server_ip: str, nodes_nb: int, tries=200):
c = kubeclient.ApiClient(host="%s:%d" % (api_server_ip, self.ec.kubernetes_apiserver_insecure_port))
core = kubeclient.CoreV1Api(c)
items = []
for t in range(tries):
try:
nodes = core.list_node()
if nodes and len(nodes.items) == nodes_nb:
items = nodes.items
break
except Exception as e:
display(e)
display("-> %d/%d NOT READY %s for %s %d/%d" % (
t, tries, api_server_ip, self.kubernetes_node_nb.__name__, len(items), nodes_nb))
time.sleep(self.testing_sleep_seconds)
self.assertEqual(len(items), nodes_nb)
def kube_apiserver_health(self, ips: list, tries=200):
self.assertIs(list, type(ips))
self.assertGreater(len(ips), 0)
for t in range(tries):
if len(ips) == 0:
break
for i, ip in enumerate(ips):
try:
endpoint = "http://%s:%d/healthz" % (ip, self.ec.kubernetes_apiserver_insecure_port)
request = requests.get(endpoint)
response_body = request.content
request.close()
display("-> RESULT %s %s" % (endpoint, response_body))
if response_body == b"ok":
display("## kubectl -s %s:%d get cs" % (ip, self.ec.kubernetes_apiserver_insecure_port))
ips.pop(i)
display("-> REMAIN %s for %s" % (str(ips), self.kube_apiserver_health.__name__))
continue
except Exception as e:
display(e)
display("-> %d/%d NOT READY %s for %s" % (t + 1, tries, ip, self.kube_apiserver_health.__name__))
time.sleep(self.testing_sleep_seconds)
self.assertEqual(len(ips), 0)
def create_tiller(self, api_server_ip: str):
c = kubeclient.ApiClient(host="http://%s:%d" % (api_server_ip, self.ec.kubernetes_apiserver_insecure_port))
with open("%s/manifests/tiller/tiller-service.yaml" % self.euid_path) as f:
service_manifest = yaml.load(f)
with open("%s/manifests/tiller/tiller-deploy.yaml" % self.euid_path) as f:
deploy_manifest = yaml.load(f)
with open("%s/manifests/tiller/tiller-service-account.yaml" % self.euid_path) as f:
serviceaccount_manifest = yaml.load(f)
with open("%s/manifests/tiller/clusterrolebinding.yaml" % self.euid_path) as f:
clusterrolebinding_manifest = yaml.load(f)
core, beta = kubeclient.CoreV1Api(c), kubeclient.ExtensionsV1beta1Api(c)
rbac = kubeclient.RbacAuthorizationV1beta1Api(c)
core.create_namespaced_service("kube-system", service_manifest)
rbac.create_cluster_role_binding(clusterrolebinding_manifest)
core.create_namespaced_service_account("kube-system", serviceaccount_manifest)
beta.create_namespaced_deployment("kube-system", deploy_manifest)
def pod_tiller_is_running(self, api_server_ip: str, tries=100):
code = 0
c = kubeclient.ApiClient(host="http://%s:%d" % (api_server_ip, self.ec.kubernetes_apiserver_insecure_port))
core = kubeclient.CoreV1Api(c)
for t in range(tries):
if code == 200:
break
try:
r = core.list_namespaced_pod("kube-system", label_selector="app=tiller")
for p in r.items:
ip = p.status.pod_ip
try:
g = requests.get("http://%s:44135/liveness" % ip)
code = g.status_code
g.close()
display("-> RESULT %s %s for %s" % (ip, code, self.pod_tiller_is_running.__name__))
except Exception as e:
display("-> %d/%d NOT READY %s for %s %s" % (
t + 1, tries, ip, self.pod_tiller_is_running.__name__, e))
except ValueError:
display("-> %d/%d NOT READY %s for %s" % (
t + 1, tries, "ValueError", self.pod_tiller_is_running.__name__))
time.sleep(self.testing_sleep_seconds)
self.assertEqual(200, code)
def _tiller_is_gc(self, node_ip):
output = subprocess.check_output([
"ssh", "-o", "StrictHostKeyChecking=no",
"-o", "UserKnownHostsFile=/dev/null",
"-o", "ConnectTimeout=1",
"-i", self.ssh_private_key,
"-lcore", node_ip,
'sudo bash -c "/opt/bin/rkt l --no-legend | grep -c tiller"'
])
tiller_containers = int(output.decode().replace("\n", ""))
return tiller_containers == 1
def tiller_can_restart(self, api_server_ip: str):
c = kubeclient.ApiClient(
host="http://%s:%d" % (api_server_ip, self.ec.kubernetes_apiserver_insecure_bind_address))
core = kubeclient.CoreV1Api(c)
r = core.list_namespaced_pod("kube-system", label_selector="app=tiller")
pod_ip, node_ip, req, tiller_endpoint = "", "", "", ""
for p in r.items:
pod_ip = p.status.pod_ip
node_ip = p.status.host_ip
try:
req = "http://%s:44135/liveness" % pod_ip
g = requests.get(req)
g.close()
self.assertEqual(200, g.status_code)
tiller_endpoint = "%s:44134" % pod_ip
display("-> tiller endpoint to kill: %s" % tiller_endpoint)
subprocess.check_output([
"ssh", "-o", "StrictHostKeyChecking=no",
"-o", "UserKnownHostsFile=/dev/null",
"-o", "ConnectTimeout=1",
"-i", self.ssh_private_key,
"-lcore", node_ip,
'sudo /usr/bin/pkill tiller'
])
break
except Exception as e:
display(e)
with self.assertRaises(requests.ConnectionError):
g = requests.get(req)
g.close()
ts = time.time()
for i in range(10):
try:
new_tiller_endpoint = self._get_tiller_grpc_endpoint(api_server_ip=api_server_ip)
display("-> new tiller endpoint: %s" % new_tiller_endpoint)
break
except Exception as e:
display(e)
time.sleep(1)
new_tiller_endpoint = self._get_tiller_grpc_endpoint(api_server_ip=api_server_ip)
self.assertNotEqual(self._get_tiller_grpc_endpoint(api_server_ip=api_server_ip), tiller_endpoint)
display("-> polling tiller Pod %s during 70s or until its GC" % new_tiller_endpoint)
while time.time() < ts + 120:
try:
loop_tiller_endpoint = self._get_tiller_grpc_endpoint(api_server_ip=api_server_ip)
except RuntimeWarning:
time.sleep(1)
loop_tiller_endpoint = self._get_tiller_grpc_endpoint(api_server_ip=api_server_ip)
self.assertEqual(new_tiller_endpoint, loop_tiller_endpoint)
if self._tiller_is_gc(node_ip):
return
time.sleep(1)
raise AssertionError("tiller is not gc on node %s" % node_ip)
def _get_tiller_grpc_endpoint(self, api_server_ip: str):
c = kubeclient.ApiClient(host="http://%s:%d" % (api_server_ip, self.ec.kubernetes_apiserver_insecure_port))
core = kubeclient.CoreV1Api(c)
r = core.list_namespaced_pod("kube-system", label_selector="app=tiller")
exception = None
for p in r.items:
ip = p.status.pod_ip
try:
g = requests.get("http://%s:44135/liveness" % ip)
g.close()
self.assertEqual(200, g.status_code)
return "%s:44134" % ip
except Exception as e:
display(e)
exception = e
raise exception
def create_helm_etcd_backup(self, api_server_ip: str, etcd_app_name: str):
tiller = self._get_tiller_grpc_endpoint(api_server_ip)
c = kubeclient.ApiClient(host="http://%s:%d" % (api_server_ip, self.ec.kubernetes_apiserver_insecure_port))
core = kubeclient.CoreV1Api(c)
try:
core.create_namespace(body={"kind": "Namespace", "apiVersion": "v1", "metadata": {"name": "backup"}})
except Exception as e:
self.assertEqual("Conflict", e.reason)
ret = subprocess.call([
self.helm_bin,
"--host",
tiller,
"install",
"-f",
"%s/manifests/etcd3-backup/%s.yaml" % (self.euid_path, etcd_app_name),
"%s/manifests/etcd3-backup/" % self.euid_path
])
self.assertEqual(0, ret)
def create_helm_by_name(self, api_server_ip: str, name: str):
tiller = self._get_tiller_grpc_endpoint(api_server_ip)
ret = subprocess.call([
self.helm_bin,
"--host",
tiller,
"install",
"%s/manifests/%s" % (self.euid_path, name)
])
self.assertEqual(0, ret)
def _snapshot_status(self, core: kubeclient.CoreV1Api, etcd_app_name: str, tries: int):
for t in range(tries):
r = core.list_namespaced_pod("backup", label_selector="etcd=%s" % etcd_app_name)
for p in r.items:
ip = p.status.host_ip
if p.status.phase != "Succeeded":
display("%d/%d pod %s status.phase: %s" % (t, tries, p.metadata.name, p.status.phase))
continue
try:
stdout = subprocess.check_output([
"ssh", "-o", "StrictHostKeyChecking=no",
"-o", "UserKnownHostsFile=/dev/null",
"-o", "ConnectTimeout=1",
"-i", self.ssh_private_key,
"-lcore", ip,
'sudo /opt/bin/etcdctl3 snapshot status /var/lib/backup/etcd3/%s.snap -w json' % etcd_app_name
])
return json.loads(stdout.decode())
except Exception as e:
display(e)
time.sleep(self.testing_sleep_seconds)
def etcd_backup_done(self, api_server_ip: str, etcd_app_name: str, tries=120):
c = kubeclient.ApiClient(host="%s:%d" % (api_server_ip, self.ec.kubernetes_apiserver_insecure_port))
core = kubeclient.CoreV1Api(c)
summary = self._snapshot_status(core, etcd_app_name, tries)
for k in ["revision", "totalKey", "totalSize"]:
self.assertGreater(summary[k], 0)
def daemonset_node_exporter_are_running(self, ips: list, tries=200):
assert type(ips) is list
assert len(ips) > 0
for t in range(tries):
if len(ips) == 0:
break
for i, ip in enumerate(ips):
try:
g = requests.get("http://%s:9100" % ip)
code = g.status_code
g.close()
display("-> RESULT %s %s" % (ip, code))
if code == 200:
ips.pop(i)
display("-> REMAIN %s for %s" % (str(ips), self.daemonset_node_exporter_are_running.__name__))
continue
except Exception as e:
display(e)
display("-> %d/%d NOT READY %s for %s" % (
t + 1, tries, ip, self.daemonset_node_exporter_are_running.__name__))
time.sleep(self.testing_sleep_seconds)
self.assertEqual(len(ips), 0)
def get_optimized_memory(self, nb_nodes: int, disk: int):
mem_bytes = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
mem_gib = mem_bytes / (1024. ** 3)
node_memory = (mem_gib // nb_nodes) * 1024
default_memory = self.ram_kvm_node_memory_mb * 0.8 if disk else self.ram_kvm_node_memory_mb
return default_memory * 1.2 if node_memory > default_memory else default_memory
@staticmethod
def get_optimized_cpu(nb_nodes: int):
cpu = float(multiprocessing.cpu_count())
cores = cpu / nb_nodes
return int(round(cores))
def kubectl_proxy(self, proxy_port: int):
def run():
cmd = [
"%s/hyperkube/hyperkube" % self.project_path,
"kubectl",
"--kubeconfig",
os.path.join(self.tests_path, "testing_kubeconfig.yaml"),
"proxy",
"-p",
"%d" % proxy_port
]
display("-> %s" % " ".join(cmd))
os.execve(cmd[0], cmd, os.environ)
return run
def create_kubeconfig(self, api_server_ip: str):
with open("%s/manifests/kubeconfig/clusterrolebinding.yaml" % self.euid_path) as f:
clusterrolebinding_manifest = yaml.load(f)
c = kubeclient.ApiClient(host="http://%s:%d" % (api_server_ip, self.ec.kubernetes_apiserver_insecure_port))
rbac = kubeclient.RbacAuthorizationV1beta1Api(c)
rbac.create_cluster_role_binding(clusterrolebinding_manifest)
kube_config = {
'preferences': {'colors': True},
'users': [
{
'user': {
'client-key': '%s/kubernetes_kubelet.private_key' % self.test_certs_path,
'client-certificate': '%s/kubernetes_kubelet.certificate' % self.test_certs_path
},
'name': 'enjoliver.local'
}
],
'kind': 'Config',
'apiVersion': 'v1',
'clusters': [
{
'cluster': {
'server': "https://%s:6443" % api_server_ip,
'certificate-authority': '%s/kubernetes_kubelet.issuing_ca' % self.test_certs_path
},
'name': 'enjoliver'
}
],
'contexts': [
{
'name': 'e',
'context': {
'cluster': 'enjoliver',
'namespace': 'kube-system',
'user': '******'
}
}
],
'current-context': 'e'
}
with open(os.path.join(self.tests_path, "testing_kubeconfig.yaml"), "w") as kc:
yaml.dump(kube_config, kc)
def iteractive_usage(self, stop="/tmp/e.stop", api_server_ip=None, fns=None):
display("-> Starting %s" % self.iteractive_usage.__name__)
kp, proxy_port = None, 8001
if api_server_ip:
self.create_kubeconfig(api_server_ip)
kp = multiprocessing.Process(target=self.kubectl_proxy(proxy_port=proxy_port))
kp.start()
maxi = 12
for i in range(maxi):
if kp.is_alive():
try:
r = requests.get("http://127.0.0.1:%d/healthz" % proxy_port)
r.close()
if r.status_code == 200:
display(
"\n#####################################\n"
"mkdir -pv ~/.kube\n"
"cat << EOF >> ~/.kube/config\n"
"apiVersion: v1\n"
"clusters:\n"
"- cluster:\n"
" server: http://127.0.0.1:8001\n"
" name: enjoliver\n"
"contexts:\n"
"- context:\n"
" cluster: enjoliver\n"
" namespace: default\n"
" user: ""\n"
" name: e\n"
"current-context: e\n"
"kind: Config\n"
"preferences:\n"
" colors: true\n"
"EOF\n"
"kubectl config use-context e\n"
"#####################################\n"
)
break
except Exception as e:
display("-> %d/%d %s" % (i + 1, maxi, e))
time.sleep(0.5)
with open(stop, "w") as f:
f.write("")
os.chmod(stop, 777)
try:
while os.path.isfile(stop) is True and os.stat(stop).st_size == 0:
if fns:
[fn() for fn in fns]
if int(time.time()) % 120 == 0:
display("-> Stop with \"sudo rm -v\" %s or \"echo 1 > %s\"" % (stop, stop))
time.sleep(self.wait_setup_teardown)
if api_server_ip and kp.is_alive():
kp.terminate()
kp.join(timeout=5)
finally:
display("-> Stopping %s" % self.iteractive_usage.__name__)
def replace_ignition_metadata(self, metadata, new_value):
req = requests.get("%s/scheduler" % self.api_uri)
scheduler = json.loads(req.content.decode())
req.close()
for mac in scheduler:
req = requests.post("%s/lifecycle/rolling/mac=%s" % (self.api_uri, mac))
req.close()
for j in os.listdir("%s/groups/" % self.test_matchbox_path):
if j == "discovery.json" or ".json" not in j:
continue
with open("%s/groups/%s" % (self.test_matchbox_path, j), 'r') as f:
group = json.loads(f.read())
group["metadata"][metadata] = new_value
with open("%s/groups/%s" % (self.test_matchbox_path, j), 'w') as f:
json.dump(group, f, indent=4)
--exec /usr/bin/cockroach -- start --port 26259 --http-port 8083 --insecure --join=127.0.0.1:26257,127.0.0.1:26258
./runtime/runtime.rkt run --net=host --insecure-options=all --interactive enjoliver.local/cockroach:latest \
--exec /usr/bin/cockroach -- sql
"""
import os
import sys
import time
import unittest
from multiprocessing import Process
import requests
from enjoliver import configs, smartdb
EC = configs.EnjoliverConfig(importer=__file__)
EC.api_uri = "http://127.0.0.1:5000"
EC.db_uri = "cockroachdb://root@localhost:26257,cockroachdb://root@localhost:26258,cockroachdb://root@localhost:26259"
@unittest.skip("Manual Trigger -> TODO")
class TestEnjoliverCockroach(unittest.TestCase):
p_matchbox = Process
p_api = Process
inte_path = "%s" % os.path.dirname(__file__)
dbs_path = "%s/dbs" % inte_path
tests_path = "%s" % os.path.dirname(inte_path)
app_path = os.path.dirname(tests_path)
project_path = os.path.dirname(app_path)
matchbox_path = "%s/matchbox" % project_path
class TestValidateAcserverStorage(unittest.TestCase):
cwd = os.path.dirname(os.path.abspath(__file__))
acserver_d = os.path.join(cwd, "runtime/acserver.d/enjoliver.local")
ec = configs.EnjoliverConfig(importer=__file__)
@staticmethod
def format_image_url(image_url: str):
image_url = image_url.replace("enjoliver.local/", "")
image_url = image_url.replace(":", "-")
return "%s-linux-amd64.aci" % image_url
def test_cni(self):
list_dir = os.listdir(os.path.join(self.acserver_d, "cni"))
self.assertEqual(1, len(list_dir))
def test_etcd(self):
list_dir = os.listdir(os.path.join(self.acserver_d, "etcd"))
self.assertEqual(1, len(list_dir))
def test_fleet(self):
list_dir = os.listdir(os.path.join(self.acserver_d, "fleet"))
self.assertEqual(1, len(list_dir))
def test_hyperkube(self):
list_dir = os.listdir(os.path.join(self.acserver_d, "hyperkube"))
self.assertIn(self.format_image_url(self.ec.hyperkube_image_url),
list_dir)
self.assertEqual(1, len(list_dir))
def test_lldp(self):
list_dir = os.listdir(os.path.join(self.acserver_d, "lldp"))
self.assertIn(self.format_image_url(self.ec.lldp_image_url), list_dir)
self.assertEqual(1, len(list_dir))
def test_iproute2(self):
list_dir = os.listdir(os.path.join(self.acserver_d, "iproute2"))
self.assertEqual(1, len(list_dir))
def test_rkt(self):
list_dir = os.listdir(os.path.join(self.acserver_d, "rkt"))
self.assertEqual(1, len(list_dir))
def test_vault(self):
list_dir = os.listdir(os.path.join(self.acserver_d, "vault"))
self.assertEqual(1, len(list_dir))
def test_ceph_tools(self):
list_dir = os.listdir(os.path.join(self.acserver_d, "ceph-tools"))
self.assertIn(self.format_image_url(self.ec.cephtools_image_url),
list_dir)
self.assertEqual(1, len(list_dir))
def test_dnsmasq(self):
list_dir = os.listdir(os.path.join(self.acserver_d, "dnsmasq"))
self.assertEqual(1, len(list_dir))
def test_tiller(self):
list_dir = os.listdir(os.path.join(self.acserver_d, "tiller"))
self.assertEqual(1, len(list_dir))
def test_heapster(self):
list_dir = os.listdir(os.path.join(self.acserver_d, "heapster"))
self.assertEqual(1, len(list_dir))
def test_node_exporter(self):
list_dir = os.listdir(os.path.join(self.acserver_d, "node-exporter"))
self.assertEqual(1, len(list_dir))
def test_kube_state_metrics(self):
list_dir = os.listdir(
os.path.join(self.acserver_d, "kube-state-metrics"))
self.assertEqual(1, len(list_dir))
def test_prometheus(self):
list_dir = os.listdir(os.path.join(self.acserver_d, "prometheus"))
self.assertEqual(1, len(list_dir))
def test_haproxy(self):
list_dir = os.listdir(os.path.join(self.acserver_d, "haproxy"))
self.assertEqual(1, len(list_dir))
def test_socat(self):
list_dir = os.listdir(os.path.join(self.acserver_d, "socat"))
self.assertEqual(1, len(list_dir))
def show_configs():
ec = configs.EnjoliverConfig(importer=__file__)
for k, v in ec.__dict__.items():
click.echo("%s=%s" % (k, v))