def clean(host, request):
logging.info("running devops clean_between_tests")
host.Iptables.reset_state()
host.ProxyContainer.run()
waiter.wait_nothrow(host.SSH.connect, timeout=30)
host.Admin.flush_journal()
host.Admin.log_to_journal(f">>>>> Test {request.node.nodeid} <<<<")
def _join_master(host, cluster_ip, cluster_token):
join_cmd = f"curl -sfL https://get.k3s.io | K3S_URL=https://{cluster_ip}:6443 K3S_TOKEN={cluster_token} sh -s - server || true"
ssh = host.SshDirect
ssh.execute("sudo systemctl stop k3s")
ssh.execute("sudo rm -rf /var/lib/rancher/k3s/server")
ssh.execute(join_cmd)
ssh.execute("sudo chmod o+r /etc/rancher/k3s/k3s.yaml")
waiter.wait_nothrow(lambda: host.SshDirect.execute("sudo kubectl get nodes"), timeout=60)
def pytest_after_base_config(base_config, request):
for cluster in base_config.clusters.values():
logging.info("deploying proxy_container on remote")
cluster.ProxyDaemonSet.run()
logging.info("waiting for SSH connection to proxy_container")
for host in base_config.hosts.values():
waiter.wait_nothrow(lambda: host.SSH.connect(port=host.tunnelport), timeout=60)
logging.info("Successfully connected to proxy pods")
# Install product devops_docker_installer is invoked
request.config.hook.pytest_after_proxy_container(base_config=base_config, request=request)
def pytest_after_base_config(base_config, request):
for host in base_config.hosts.values():
logging.info("deploying proxy_container on remote")
host.ProxyContainer.run()
logging.info("waiting for SSH connection to proxy_container")
waiter.wait_nothrow(lambda: host.SSH.connect(port=host.tunnelport),
timeout=60)
# import pdb; pdb.set_trace()
# Install product devops_docker_installer is invoked
request.config.hook.pytest_after_proxy_container(base_config=base_config,
request=request)
def list_masters(self):
masters = []
for host in self._cluster.hosts.values():
try:
waiter.wait_nothrow(
lambda: host.SshDirect.execute("sudo kubectl get po"),
timeout=100)
masters.append(host)
except:
continue
return masters
def setup_cluster(cluster, request):
for host_name, config in request.function.__hardware_reqs.items():
host = dict(cluster.hosts.items())[host_name]
host.k3s_config = config['k3s_config']
host.internal_ip = host.SshDirect.execute("hostname -I | awk {'print $1'}").strip()
logging.info("Setting up k3s cluster")
hosts = list(cluster.hosts.values())
masters = [host for host in hosts if host.k3s_config["role"] == "master"]
if not masters:
raise Exception("Couldn't find any master node")
main_master = next(iter(masters))
main_master.k8s_name = "k3s-master"
main_master.SshDirect.execute(
"curl -sfL https://get.k3s.io | sh -s - --cluster-init --cluster-reset --cluster-reset-restore-path=/root/k3s-infra-1174-snapshot")
waiter.wait_nothrow(lambda: main_master.SshDirect.execute("journalctl --since='1 min ago' | grep 'restart without'"))
main_master.SshDirect.execute(
"curl -sfL https://get.k3s.io | sh -s - --node-name=k3s-master --disable='servicelb,traefik,local-storage,metrics-server'")
main_master.SshDirect.execute("sudo chmod o+r /etc/rancher/k3s/k3s.yaml")
cluster_token = main_master.SshDirect.execute("sudo cat /var/lib/rancher/k3s/server/token").strip()
cluster_ip = main_master.SshDirect.execute("hostname -I").strip()
waiter.wait_nothrow(lambda: main_master.SshDirect.execute("kubectl get nodes"))
nodes = [host for host in hosts if host.k3s_config['role'] == "node"]
masters.remove(main_master)
jobs = {}
nodes_jobs = {f"{host.alias}": partial(_join_agent, host, cluster_ip, cluster_token) for host in nodes}
masters_jobs = {f"{master.alias}": partial(_join_master, master, cluster_ip, cluster_token) for master in masters}
jobs.update(nodes_jobs)
jobs.update(masters_jobs)
if jobs:
concurrently.run(jobs)
logging.info("Waiting for cluster to be Ready...")
k8s_client = cluster.Kubectl.client()
v1 = kubernetes.client.CoreV1Api(k8s_client)
waiter.wait_for_predicate(lambda: len(v1.list_node().items) == len(hosts), timeout=30)
logging.info(f"Number of nodes in cluster: {len(v1.list_node().items)}")
waiter.wait_for_predicate(lambda: kubectl.is_cluster_ready(k8s_client), timeout=60)
logging.info("Adding node labels and taints")
_label_and_taint_nodes(k8s_client, hosts)
def run(self):
self.kill()
logging.debug("Deploying automation-proxy DaemonSet")
with open(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
"../../proxy_container/daemonset.yaml")) as f:
ds_yaml = yaml.safe_load(f)
ds_yaml['spec']['template']['spec']['containers'][0][
'image'] = f'gcr.io/anyvision-training/automation-proxy:{self._automation_proxy_version()}'
try:
res = self._k8s_v1_client.create_namespaced_daemon_set(
namespace="default", body=ds_yaml)
except ApiException as e:
logging.exception(
"Exception when calling AppsV1Api->create_namespaced_daemon_set: %s\n"
% e)
waiter.wait_nothrow(
lambda: self._num_ready_pods() == len(self._cluster.hosts),
timeout=30)
logging.debug(f"Deployment created. status={res.metadata.name}")
def test_power_plugin(base_config):
logging.info("starting power test")
host = base_config.hosts.host
power = host.Power
try:
power.verify_available()
except (NotImplementedError, AssertionError):
logging.info(
"Skipping test_power on HUT which Power plugin doesnt support...")
return
logging.info("powering off")
power.off()
logging.info("powered off")
assert power.status() == 'off'
with pytest.raises(Exception):
logging.info("trying to connect, should get exception")
host.SshDirect.connect(timeout=3)
logging.info("powered off successfully, powering on")
power.on()
logging.info("powered on, trying to connect")
waiter.wait_nothrow(lambda: power.status() == 'on')
host.SshDirect.connect(timeout=30)
logging.info("connected succesfully, plugin working!")
def deploy_proxy_pod(cluster, request):
concurrently.run([partial(ssh.ssh_direct_connect_session, host, request) for host in cluster.hosts.values()])
logging.info("Deploying proxy daemon-set")
cluster.ProxyDaemonSet.run()
for host in cluster.hosts.values():
waiter.wait_nothrow(lambda: host.SSH.connect(port=host.tunnelport), timeout=60)
def clean(host, request):
logging.info("running devops clean_between_tests")
host.TunnelManager.clear()
waiter.wait_nothrow(host.SshDirect.connect, timeout=30)
host.ProxyContainer.restart()
waiter.wait_nothrow(host.SSH.connect, timeout=30)
def deploy_proxy_container(host, request):
host.ProxyContainer.run()
waiter.wait_nothrow(host.SSH.connect, timeout=30)
def _kubectl(self, host):
return waiter.wait_nothrow(
lambda: host.SshDirect.execute("sudo kubectl get po"), timeout=150)
def deploy_proxy_container(host, request):
ssh.ssh_direct_connect_session(host, request)
host.ProxyContainer.run()
waiter.wait_nothrow(host.SSH.connect, timeout=30)
def clear_and_start(self):
self._redis.flushall()
container.start_container_by_service(self._host, "_redis")
waiter.wait_nothrow(self.ping, timeout=30)
def _start_tunnel(self):
self._forward_server, self._local_bind_port = waiter.wait_nothrow(
lambda: self.try_start_tunnel(self.remote_dns_name, self.
remote_port, self.transport, self.
_local_bind_port))
def start(self):
container.restart_container_by_service(self._host, "_consul")
waiter.wait_nothrow(self.ping, timeout=30)
def clear_and_start(self):
self.delete_storage_compose()
container.start_container_by_service(self._host, "_consul")
waiter.wait_nothrow(self.ping, timeout=30)
def clean(cluster, request):
concurrently.run([partial(ssh.ssh_direct_connect_session, host, request) for host in cluster.hosts.values()])
logging.info("running devops clean_base_btwn_tests")
cluster.ProxyDaemonSet.restart()
for host in cluster.hosts.values():
waiter.wait_nothrow(host.SSH.connect, timeout=30)
def start_service(self):
self._host.Docker.start_container("memsql")
self._host.Docker.wait_container_up("memsql")
waiter.wait_nothrow(self.ping, timeout=30)
def pytest_clean_base_btwn_tests(base_config, item):
logging.info("running devops clean_base_btwn_tests")
concurrently.run([lambda: cluster.ProxyDaemonSet.restart() for cluster in base_config.clusters.values()])
for host in base_config.hosts.values():
waiter.wait_nothrow(host.SSH.connect, timeout=30)
def pytest_clean_between_tests(host, item):
logging.info("running devops clean_between_tests")
host.TunnelManager.clear()
host.ProxyContainer.restart()
waiter.wait_nothrow(host.SSH.connect, timeout=30)
def _join_agent(host, cluster_ip, cluster_token):
join_cmd = f"curl -sfL https://get.k3s.io | K3S_URL=https://{cluster_ip}:6443 K3S_TOKEN={cluster_token} sh -s -"
host.SshDirect.execute(join_cmd)
waiter.wait_nothrow(lambda: host.SshDirect.execute("systemctl is-active --quiet k3s-agent"), timeout=60)