def _install_kubeadm(self): logger.info('Starting installing kubeadm on %s nodes' % len(self.hosts)) logger.debug('Turning off Firewall on hosts') cmd = '''cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf net.bridge.bridge-nf-call-ip6tables = 1 net.bridge.bridge-nf-call-iptables = 1''' execute_cmd(cmd, self.hosts) cmd = 'sudo sysctl --system' execute_cmd(cmd, self.hosts) logger.debug('Turning off swap on hosts') cmd = 'swapoff -a' execute_cmd(cmd, self.hosts) logger.debug('Installing kubeadm kubelet kubectl') configurator = packages_configurator() configurator.install_packages(['apt-transport-https', 'curl'], self.hosts) cmd = 'curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -' execute_cmd(cmd, self.hosts) cmd = '''cat <<EOF | sudo tee /etc/apt/sources.list.d/kubernetes.list deb https://apt.kubernetes.io/ kubernetes-xenial main''' execute_cmd(cmd, self.hosts) configurator.install_packages(['kubelet', 'kubeadm', 'kubectl'], self.hosts)
def config_host(self, kube_master, kube_namespace): logger.info('Starting configuring nodes') # configuring Kubernetes environment kube_workers = [host for host in self.hosts if host != kube_master] if self.args.kube_master is None: kube_workers = self.deploy_k8s(kube_master) self.setup_k8s_env(kube_master, kube_namespace, kube_workers) elif self.args.setup_k8s_env: logger.info('Kubernetes master: %s' % kube_master) self.setup_k8s_env(kube_master, kube_namespace, kube_workers) else: self._get_credential(kube_master) if self.args.attach_volume: self._setup_ovh_kube_volumes(kube_workers, n_pv=3) # Install elmerfs dependencies configurator = packages_configurator() configurator.install_packages(['libfuse2', 'jq'], kube_workers) # Installing benchmark for running the experiments if self.configs['parameters']['benchmarks'] in ['mailserver', 'videoserver']: logger.info('Installing Filebench') self.install_filebench(kube_workers) logger.info('Finish configuring nodes') return kube_master
def install_benchsoftware(self, hosts): logger.info("Starting installing benchmark software") logger.info("Installing tccommand") cmd = "curl -sSL https://raw.githubusercontent.com/thombashi/tcconfig/master/scripts/installer.sh | bash" execute_cmd(cmd, hosts) configurator = packages_configurator() configurator.install_packages(["bonnie++", "libfuse2", "wget", "jq"], hosts) logger.info("Installing crefi") cmd = "pip install pyxattr" execute_cmd(cmd, hosts) cmd = "pip install crefi" execute_cmd(cmd, hosts) # create folder to store the results on all hosts cmd = "mkdir -p /tmp/results" execute_cmd(cmd, hosts) # prepare data on all host for convergence experiment cmd = "mkdir -p /tmp/convergence_files" execute_cmd(cmd, hosts) cmd = "wget https://raw.githubusercontent.com/ntlinh16/elmerfs-eval/main/resources_convergence_exp/timing_copy_file.sh -P /tmp/convergence_files/ -N" execute_cmd(cmd, hosts) cmd = "wget https://raw.githubusercontent.com/ntlinh16/elmerfs-eval/main/resources_convergence_exp/periodically_checksum.sh -P /tmp/convergence_files/ -N" execute_cmd(cmd, hosts) test_file = self.configs['exp_env']['convergence_test_file'] cmd = "wget %s -P /tmp/convergence_files/ -N" % test_file execute_cmd(cmd, hosts) cmd = "mv /tmp/convergence_files/%s /tmp/convergence_files/sample" % ( test_file.split('/')[-1]) execute_cmd(cmd, hosts) # prepare data on all host for performance experiment cmd = "mkdir -p /tmp/performance_files" execute_cmd(cmd, hosts) cmd = "wget https://raw.githubusercontent.com/ntlinh16/elmerfs-eval/main/resources_performance_exp/elmerfs_setup.sh -P /tmp/performance_files/ -N" execute_cmd(cmd, hosts) cmd = "wget https://raw.githubusercontent.com/ntlinh16/elmerfs-eval/main/resources_performance_exp/elmerfs_run.sh -P /tmp/performance_files/ -N" execute_cmd(cmd, hosts) cmd = "chmod +x /tmp/performance_files/*" execute_cmd(cmd, hosts) # create random UID and set it as ELMERFS_UID env variable on each hot for running elmerfs uids = random.sample(range(10**4, 10**6), len(self.hosts)) for host, uid in zip(hosts, uids): cmd = 'echo "export ELMERFS_UID=%s" >> ~/.bashrc' % uid execute_cmd(cmd, host) logger.info("Finish installing benchmark software on hosts\n")
def config_host(self, kube_master_site, kube_namespace): kube_master = self.args.kube_master if self.args.kube_master is None: antidote_hosts = list() for cluster in self.configs['clusters']: cluster_name = cluster['cluster'] if cluster_name == self.configs['exp_env']['kube_master_site']: antidote_hosts += [ host for host in self.hosts if host.startswith(cluster_name) ][0:cluster['n_nodes'] + 1] else: antidote_hosts += [ host for host in self.hosts if host.startswith(cluster_name) ][0:cluster['n_nodes']] for host in antidote_hosts: if host.startswith(kube_master_site): kube_master = host break kube_workers = self.config_kube(kube_master, antidote_hosts, kube_namespace) else: logger.info('Kubernetes master: %s' % kube_master) self._get_credential(kube_master) configurator = k8s_resources_configurator() deployed_hosts = configurator.get_k8s_resources(resource='node') kube_workers = [ host.metadata.name for host in deployed_hosts.items ] kube_workers.remove(kube_master) logger.info('Installing elmerfs dependencies') configurator = packages_configurator() configurator.install_packages(['libfuse2', 'wget', 'jq'], kube_workers) # Create mount point on elmerfs hosts cmd = 'mkdir -p /tmp/dc-$(hostname)' execute_cmd(cmd, kube_workers) # Installing filebench for running the experiments logger.info('Installing Filebench') self.install_filebench(kube_workers)
def install_filebench(self, hosts): configurator = packages_configurator() configurator.install_packages(['build-essential', 'bison', 'flex', 'libtool'], hosts) cmd = 'wget https://github.com/filebench/filebench/archive/refs/tags/1.5-alpha3.tar.gz -P /tmp/ -N' execute_cmd(cmd, hosts) cmd = 'tar -xf /tmp/1.5-alpha3.tar.gz --directory /tmp/' execute_cmd(cmd, hosts) cmd = '''cd /tmp/filebench-1.5-alpha3/ && libtoolize && aclocal && autoheader && automake --add-missing && autoconf && ./configure && make && make install''' execute_cmd(cmd, hosts)
def config_docker(self): """Install Docker on the given hosts Parameters ---------- hosts: str a list of hosts """ logger.info('Starting installing Docker on %s hosts' % len(self.hosts)) configurator = packages_configurator() configurator.install_packages(['wget'], self.hosts) logger.info('Downloading the official get_docker script') cmd = 'wget https://get.docker.com -O get-docker.sh' self.error_hosts = execute_cmd(cmd, self.hosts) logger.info('Installing Docker by using get_docker script') cmd = 'sh get-docker.sh' self.error_hosts = execute_cmd(cmd, self.hosts) logger.info('Finish installing Docker on %s hosts' % len(self.hosts))
def install_glusterfs(self, hosts): logger.info('Installing GlusterFS') configurator = packages_configurator() configurator.install_packages(["glusterfs-server"], hosts) cmd = 'systemctl start glusterd' execute_cmd(cmd, hosts) gluster_configuration = list() for index, host in enumerate(hosts): cmd = "hostname -I | awk '{print $1}'" _, r = execute_cmd(cmd, host) host_ip = r.processes[0].stdout.strip() gluster_configuration.append("%s gluster-%s.%s.local gluster-%s " % (host_ip, index, host, index)) gluster_configuration = "\n".join(gluster_configuration) cmd = "echo '%s' >> /etc/hosts" % gluster_configuration execute_cmd(cmd, hosts) for index, _ in enumerate(hosts): cmd = 'gluster peer probe gluster-%s' % index execute_cmd(cmd, hosts[0])
def setup_env(self): """Setting the experiment environment base on the user's requirements This function normally contains two steps: 1. Provisioning hosts on G5k if needed (if you provided the OAR_JOB_ID of the already reserved hosts, the provisioner will not make a reservation again) 2. Configuring all your necessary packages/services on those hosts. """ provisioner = g5k_provisioner( config_file_path=self.args.config_file_path, keep_alive=self.args.keep_alive, out_of_chart=self.args.out_of_chart, oar_job_ids=self.args.oar_job_ids, no_deploy_os=self.args.no_deploy_os, is_reservation=self.args.is_reservation, job_name="cloudal") provisioner.provisioning() self.hosts = provisioner.hosts oar_job_ids = provisioner.oar_result self.oar_result = provisioner.oar_result ################################################## # Configuring hosts with your applications here # ################################################## # For example: install some dependencies configurator = packages_configurator() configurator.install_packages(['sysstat', 'htop'], self.hosts) # or call the provided configurator (by cloudal) to deploy some well-known services configurator = docker_configurator(self.hosts) configurator.config_docker() # or perform some commands on all of hosts logger.info("Downloading cloudal") cmd = "cd /tmp/ && git clone https://github.com/ntlinh16/cloudal.git" execute_cmd(cmd, self.hosts) return oar_job_ids
def deploy_elmerfs(self, kube_master, kube_namespace, elmerfs_hosts): logger.info("Starting deploying elmerfs on hosts") configurator = packages_configurator() # configurator.install_packages(["libfuse2", "wget", "jq"], elmerfs_hosts) elmerfs_repo = self.configs["exp_env"]["elmerfs_repo"] elmerfs_version = self.configs["exp_env"]["elmerfs_version"] elmerfs_file_path = self.configs["exp_env"]["elmerfs_path"] if elmerfs_repo is None: elmerfs_repo = "https://github.com/scality/elmerfs" if elmerfs_version is None: elmerfs_version = "latest" logger.info("Killing elmerfs process if it is running") for host in elmerfs_hosts: cmd = "pidof elmerfs" _, r = execute_cmd(cmd, host) pids = r.processes[0].stdout.strip().split(" ") if len(pids) >= 1 and pids[0] != '': for pid in pids: cmd = "kill %s" % pid.strip() execute_cmd(cmd, host) cmd = "umount /tmp/dc-$(hostname)" execute_cmd(cmd, host) cmd = "rm -rf /tmp/dc-$(hostname)" execute_cmd(cmd, host) logger.info("Delete elmerfs project folder on host (if existing)") cmd = "rm -rf /tmp/elmerfs_repo" execute_cmd(cmd, kube_master) if elmerfs_file_path is None: logger.info("Downloading elmerfs project from the repo") cmd = """curl \ -H "Accept: application/vnd.github.v3+json" \ https://api.github.com/repos/scality/elmerfs/releases/%s | jq ".tag_name" \ | xargs -I tag_name git clone https://github.com/scality/elmerfs.git --branch tag_name --single-branch /tmp/elmerfs_repo """ % elmerfs_version execute_cmd(cmd, kube_master) cmd = "cd /tmp/elmerfs_repo \ && git submodule update --init --recursive" execute_cmd(cmd, kube_master) cmd = """cat <<EOF | sudo tee /tmp/elmerfs_repo/Dockerfile FROM rust:1.47 RUN mkdir /elmerfs WORKDIR /elmerfs COPY . . RUN apt-get update \ && apt-get -y install libfuse-dev RUN cargo build --release CMD ["/bin/bash"] """ execute_cmd(cmd, kube_master) logger.info("Building elmerfs") cmd = " cd /tmp/elmerfs_repo/ \ && docker build -t elmerfs ." execute_cmd(cmd, kube_master) cmd = "docker run --name elmerfs elmerfs \ && docker cp -L elmerfs:/elmerfs/target/release/main /tmp/elmerfs \ && docker rm elmerfs" execute_cmd(cmd, kube_master) getput_file( hosts=[kube_master], file_paths=["/tmp/elmerfs"], dest_location="/tmp", action="get", ) elmerfs_file_path = "/tmp/elmerfs" logger.info( "Uploading elmerfs binary file from local to %s elmerfs hosts" % len(elmerfs_hosts)) getput_file( hosts=elmerfs_hosts, file_paths=[elmerfs_file_path], dest_location="/tmp", action="put", ) cmd = "chmod +x /tmp/elmerfs \ && mkdir -p /tmp/dc-$(hostname)" execute_cmd(cmd, elmerfs_hosts) logger.debug("Getting IP of antidoteDB instances on nodes") antidote_ips = dict() configurator = k8s_resources_configurator() pod_list = configurator.get_k8s_resources( resource="pod", label_selectors="app=antidote", kube_namespace=kube_namespace, ) for pod in pod_list.items: node = pod.spec.node_name if node not in antidote_ips: antidote_ips[node] = list() antidote_ips[node].append(pod.status.pod_ip) for host in elmerfs_hosts: antidote_options = [ "--antidote=%s:8087" % ip for ip in antidote_ips[host] ] elmerfs_cmd = "RUST_BACKTRACE=1 RUST_LOG=debug nohup /tmp/elmerfs %s --mount=/tmp/dc-$(hostname) --force-view=$ELMERFS_UID > /tmp/elmer.log 2>&1" % " ".join( antidote_options) logger.info("Starting elmerfs on %s with cmd: %s" % (host, elmerfs_cmd)) execute_cmd(elmerfs_cmd, host, mode='start') sleep(10) for i in range(10): cmd = "pidof elmerfs" _, r = execute_cmd(cmd, host) pid = r.processes[0].stdout.strip().split(" ") if len(pid) >= 1 and pid[0].strip(): break else: execute_cmd(elmerfs_cmd, host, mode="start") sleep(10) else: logger.info("Cannot deploy elmerfs on host %s" % host) return False logger.info("Finish deploying elmerfs\n") return True
def install_elmerfs(self, kube_master, elmerfs_hosts, elmerfs_mountpoint, elmerfs_repo, elmerfs_version, elmerfs_path): # Create folder to build the elmerfs from the repo # cmd = 'rm -rf /tmp/elmerfs_repo && mkdir -p /tmp/elmerfs_repo' # execute_cmd(cmd, elmerfs_hosts) if elmerfs_repo is None: elmerfs_repo = 'https://github.com/scality/elmerfs' if elmerfs_version is None: elmerfs_version = 'latest' if elmerfs_path is None: logger.info('Installing elmerfs') configurator = packages_configurator() configurator.install_packages(['libfuse2', 'wget', 'jq'], elmerfs_hosts) logger.info('Create folder to build the elmerfs from the repo') # Create folder to build the elmerfs from the repo cmd = 'rm -rf /tmp/elmerfs_repo && mkdir -p /tmp/elmerfs_repo' execute_cmd(cmd, kube_master) logger.info('Downloading elmerfs project from the repo') cmd = '''curl \ -H 'Accept: application/vnd.github.v3+json' \ https://api.github.com/repos/scality/elmerfs/releases/%s | jq '.tag_name' \ | xargs -I tag_name git clone https://github.com/scality/elmerfs.git --branch tag_name --single-branch /tmp/elmerfs_repo ''' % elmerfs_version execute_cmd(cmd, kube_master) logger.info('update the repo') cmd = 'cd /tmp/elmerfs_repo \ && git submodule update --init --recursive' execute_cmd(cmd, kube_master) logger.info('Creating the Docker file') cmd = '''cat <<EOF | sudo tee /tmp/elmerfs_repo/Dockerfile FROM rust:1.47 RUN mkdir /elmerfs WORKDIR /elmerfs COPY . . RUN apt-get update \ && apt-get -y install libfuse-dev RUN cargo build --release CMD ['/bin/bash'] ''' execute_cmd(cmd, kube_master) logger.info('Building elmerfs Docker image') cmd = 'cd /tmp/elmerfs_repo/ \ && docker build -t elmerfs .' execute_cmd(cmd, kube_master) logger.info('Building elmerfs') cmd = 'docker run --name elmerfs elmerfs \ && docker cp -L elmerfs:/elmerfs/target/release/main /tmp/elmerfs \ && docker rm elmerfs' execute_cmd(cmd, kube_master) getput_file(hosts=[kube_master], file_paths=['/tmp/elmerfs'], dest_location='/tmp', action='get',) elmerfs_path = '/tmp/elmerfs' logger.info('Uploading elmerfs binary file from local to %s elmerfs hosts' % len(elmerfs_hosts)) getput_file(hosts=elmerfs_hosts, file_paths=[elmerfs_path], dest_location='/tmp', action='put') cmd = 'chmod +x /tmp/elmerfs' execute_cmd(cmd, elmerfs_hosts) logger.info('Create mountpoint and result folder') cmd = 'rm -rf /tmp/results && mkdir -p /tmp/results && \ rm -rf %s && mkdir -p %s' % (elmerfs_mountpoint, elmerfs_mountpoint) execute_cmd(cmd, elmerfs_hosts)
def deploy_elmerfs(self, kube_master, kube_namespace, elmerfs_hosts): logger.info("Starting deploying elmerfs on hosts") configurator = packages_configurator() configurator.install_packages(['libfuse2', 'wget', 'jq'], elmerfs_hosts) elmerfs_repo = self.configs['exp_env']['elmerfs_repo'] elmerfs_version = self.configs['exp_env']['elmerfs_version'] if elmerfs_repo is None: elmerfs_repo = 'https://github.com/scality/elmerfs' if elmerfs_version is None: elmerfs_version = 'latest' logger.info('Killing elmerfs process if it is running') for host in elmerfs_hosts: cmd = "ps aux | grep elmerfs | awk '{print$2}'" _, r = execute_cmd(cmd, host) pids = r.processes[0].stdout.strip().split('\r\n') if len(pids) >= 3: cmd = "kill %s && umount /tmp/dc-$(hostname)" % pids[0] execute_cmd(cmd, host) logger.info("Downloading elmerfs project from the repo") cmd = '''curl \ -H "Accept: application/vnd.github.v3+json" \ https://api.github.com/repos/scality/elmerfs/releases/%s | jq ".tag_name" \ | xargs -I tag_name git clone https://github.com/scality/elmerfs.git --branch tag_name --single-branch /tmp/elmerfs_repo ''' % elmerfs_version execute_cmd(cmd, kube_master) cmd = "cd /tmp/elmerfs_repo \ && git submodule update --init --recursive" execute_cmd(cmd, kube_master) cmd = '''cat <<EOF | sudo tee /tmp/elmerfs_repo/Dockerfile FROM rust:1.47 RUN mkdir /elmerfs WORKDIR /elmerfs COPY . . RUN apt-get update \ && apt-get -y install libfuse-dev RUN cargo build --release CMD ["/bin/bash"] ''' execute_cmd(cmd, kube_master) logger.info("Building elmerfs") cmd = " cd /tmp/elmerfs_repo/ \ && docker build -t elmerfs ." execute_cmd(cmd, kube_master) cmd = "docker run --name elmerfs elmerfs \ && docker cp -L elmerfs:/elmerfs/target/release/main /tmp/elmerfs \ && docker rm elmerfs" execute_cmd(cmd, kube_master) getput_file(hosts=[kube_master], file_paths=['/tmp/elmerfs'], dest_location='/tmp', action='get') elmerfs_file_path = '/tmp/elmerfs' logger.info( "Uploading elmerfs binary file from local to %s elmerfs hosts" % len(elmerfs_hosts)) getput_file(hosts=elmerfs_hosts, file_paths=[elmerfs_file_path], dest_location='/tmp', action='put') cmd = "chmod +x /tmp/elmerfs \ && mkdir -p /tmp/dc-$(hostname)" execute_cmd(cmd, elmerfs_hosts) logger.debug('Getting IP of antidoteDB instances on nodes') antidote_ips = dict() configurator = k8s_resources_configurator() pod_list = configurator.get_k8s_resources( resource='pod', label_selectors='app=antidote', kube_namespace=kube_namespace) for pod in pod_list.items: node = pod.spec.node_name if node not in antidote_ips: antidote_ips[node] = list() antidote_ips[node].append(pod.status.pod_ip) for host in elmerfs_hosts: antidote_options = [ "--antidote=%s:8087" % ip for ip in antidote_ips[host] ] cmd = "RUST_BACKTRACE=1 RUST_LOG=debug nohup /tmp/elmerfs %s --mount=/tmp/dc-$(hostname) --no-locks > /tmp/elmer.log 2>&1" % " ".join( antidote_options) logger.info("Starting elmerfs on %s with cmd: %s" % (host, cmd)) execute_cmd(cmd, host, mode='start') sleep(5) logger.info('Finish deploying elmerfs\n')
def deploy_monitoring(self, node, monitoring_yaml_path, kube_namespace='default'): """Deploy monitoring system for AntidoteDB cluster on the given K8s cluster Parameters ---------- node: str the IP or hostname used to connect to the node to deploy monitoring system on monitoring_yaml_path: str a path to the K8s yaml deployment files kube_namespace: str the name of K8s namespace """ logger.info("Deleting old deployment") cmd = "rm -rf /root/antidote_stats" execute_cmd(cmd, node) configurator = packages_configurator() configurator.install_packages(['git'], [node]) cmd = "git clone https://github.com/AntidoteDB/antidote_stats.git" execute_cmd(cmd, node) logger.info("Setting to allow pods created on kube mater node") cmd = "kubectl taint nodes --all node-role.kubernetes.io/master-" execute_cmd(cmd, node, is_continue=True) logger.debug("Init configurator: k8s_resources_configurator") configurator = k8s_resources_configurator() pods = configurator.get_k8s_resources_name( resource='pod', label_selectors='app=antidote', kube_namespace=kube_namespace) antidote_info = ["%s.antidote:3001" % pod for pod in pods] logger.debug( 'Modify the prometheus.yml file with AntidoteDB instances info') file_path = os.path.join(monitoring_yaml_path, 'prometheus.yml.template') with open(file_path) as f: doc = f.read().replace('antidotedc_info', '%s' % antidote_info) prometheus_configmap_file = os.path.join(monitoring_yaml_path, 'prometheus.yml') with open(prometheus_configmap_file, 'w') as f: f.write(doc) configurator.create_configmap(file=prometheus_configmap_file, namespace=kube_namespace, configmap_name='prometheus-configmap') logger.debug('Modify the deploy_prometheus.yaml file with node info') if not is_ip(node): node_info = configurator.get_k8s_resources( resource='node', label_selectors='kubernetes.io/hostname=%s' % node) for item in node_info.items[0].status.addresses: if item.type == 'InternalIP': node_ip = item.address node_hostname = node else: node_ip = node cmd = 'hostname' _, r = execute_cmd(cmd, node) node_hostname = r.processes[0].stdout.strip().lower() file_path = os.path.join(monitoring_yaml_path, 'deploy_prometheus.yaml.template') with open(file_path) as f: doc = f.read().replace('node_ip', '%s' % node_ip) doc = doc.replace("node_hostname", '%s' % node_hostname) prometheus_deploy_file = os.path.join(monitoring_yaml_path, 'deploy_prometheus.yaml') with open(prometheus_deploy_file, 'w') as f: f.write(doc) logger.info("Starting Prometheus service") configurator.deploy_k8s_resources(files=[prometheus_deploy_file], namespace=kube_namespace) logger.info('Waiting until Prometheus instance is up') configurator.wait_k8s_resources(resource='pod', label_selectors="app=prometheus", kube_namespace=kube_namespace) logger.debug('Modify the deploy_grafana.yaml file with node info') file_path = os.path.join(monitoring_yaml_path, 'deploy_grafana.yaml.template') with open(file_path) as f: doc = f.read().replace('node_ip', '%s' % node_ip) doc = doc.replace("node_hostname", '%s' % node_hostname) grafana_deploy_file = os.path.join(monitoring_yaml_path, 'deploy_grafana.yaml') with open(grafana_deploy_file, 'w') as f: f.write(doc) file = '/root/antidote_stats/monitoring/grafana-config/provisioning/datasources/all.yml' cmd = """ sed -i "s/localhost/%s/" %s """ % (node_ip, file) execute_cmd(cmd, node) logger.info("Starting Grafana service") configurator.deploy_k8s_resources(files=[grafana_deploy_file], namespace=kube_namespace) logger.info('Waiting until Grafana instance is up') configurator.wait_k8s_resources(resource='pod', label_selectors="app=grafana", kube_namespace=kube_namespace) logger.info("Finish deploying monitoring system\n") prometheus_url = "http://%s:9090" % node_ip grafana_url = "http://%s:3000" % node_ip logger.info("Connect to Grafana at: %s" % grafana_url) logger.info("Connect to Prometheus at: %s" % prometheus_url) return prometheus_url, grafana_url