def get_results(comb, hosts, remote_result_files, local_result_dir): """Get all the results files from remote hosts to a local result directory Parameters ---------- comb: dict a dictionary that contains the set of parameters for a specific run key: str, the name of the experiment parameter value: object, the value of the experiment parameter in this combination hosts: list a list of hosts to get the results from remote_result_files: list a list of results files on the remote nodes local_result_dir: str the path to the directory to store the results on the local node """ logger.info('Create combination dir locally') comb_dir = create_combination_dir(comb, local_result_dir) logger.info('Download the result') getput_file(hosts=hosts, file_paths=remote_result_files, dest_location=comb_dir, action='get') return comb_dir
def _get_credential(self, kube_master): home = os.path.expanduser('~') kube_dir = os.path.join(home, '.kube') if not os.path.exists(kube_dir): os.mkdir(kube_dir) getput_file(hosts=[kube_master], file_paths=['~/.kube/config'], dest_location=kube_dir, action='get') kube_config_file = os.path.join(kube_dir, 'config') config.load_kube_config(config_file=kube_config_file) logger.info('Kubernetes config file is stored at: %s' % kube_config_file)
def _get_credential(self, kube_master): home = os.path.expanduser("~") kube_dir = os.path.join(home, ".kube") if not os.path.exists(kube_dir): os.mkdir(kube_dir) getput_file( hosts=[kube_master], file_paths=["~/.kube/config"], dest_location=kube_dir, action="get", ) kube_config_file = os.path.join(kube_dir, "config") config.load_kube_config(config_file=kube_config_file) logger.info("Kubernetes config file is stored at: %s" % kube_config_file)
def perform_combination(self, kube_namespace, concurrent_clients): logger.info('-----------------------------------------------------------------') logger.info('5. Starting deploying fmke client to stress the Antidote database') fmke_client_k8s_dir = self.configs['exp_env']['fmke_yaml_path'] logger.debug('Delete old k8s yaml files if exists') for filename in os.listdir(fmke_client_k8s_dir): if filename.startswith('create_fmke_client_') or filename.startswith('fmke_client_'): if '.template' not in filename: try: os.remove(os.path.join(fmke_client_k8s_dir, filename)) except OSError: logger.debug("Error while deleting file") logger.debug('Create fmke_client folder on each fmke_client node') configurator = k8s_resources_configurator() exp_nodes = configurator.get_k8s_resources_name(resource='node', label_selectors='service_g5k=fmke_client') cmd = 'mkdir -p /tmp/fmke_client' execute_cmd(cmd, exp_nodes) logger.debug('Create fmke_client config files to stress database for each Antidote DC') file_path = os.path.join(fmke_client_k8s_dir, 'fmke_client.config.template') fmke_list = configurator.get_k8s_resources(resource='pod', label_selectors='app=fmke', kube_namespace=kube_namespace) for cluster in self.configs['exp_env']['clusters']: fmke_IPs = list() for fmke in fmke_list.items: if cluster in fmke.metadata.name: fmke_IPs.append(fmke.status.pod_ip) fmke_ports = [9090 for i in range(0, len(fmke_IPs))] # Modify fmke_client config files with new values with open(file_path) as f: doc = f.read().replace('["127.0.0.1"]', '%s' % fmke_IPs) doc = doc.replace("[9090]", '%s' % fmke_ports) doc = doc.replace("{concurrent, 16}.", '{concurrent, %s}.' % concurrent_clients) doc = doc.replace("'", '"') file_path2 = os.path.join(fmke_client_k8s_dir, 'fmke_client_%s.config' % cluster) with open(file_path2, 'w') as f: f.write(doc) logger.debug('Upload fmke_client config files to kube_master to be used by kubectl to run fmke_client pods') getput_file(hosts=exp_nodes, file_paths=[file_path2], dest_location='/tmp/fmke_client/', action='put') logger.debug('Create create_fmke_client.yaml files to run job stress for each Antidote DC') file_path = os.path.join(fmke_client_k8s_dir, 'create_fmke_client.yaml.template') with open(file_path) as f: doc = yaml.safe_load(f) fmke_client_files = list() for cluster in self.configs['exp_env']['clusters']: doc['spec']['parallelism'] = self.configs['exp_env']['n_fmke_client_per_dc'] doc['spec']['completions'] = self.configs['exp_env']['n_fmke_client_per_dc'] doc['metadata']['name'] = 'fmke-client-%s' % cluster doc['spec']['template']['spec']['containers'][0]['lifecycle']['postStart']['exec']['command'] = [ "cp", "/cluster_node/fmke_client_%s.config" % cluster, "/fmke_client/fmke_client.config"] doc['spec']['template']['spec']['nodeSelector'] = { 'service_g5k': 'fmke_client', 'cluster_g5k': '%s' % cluster} file_path = os.path.join(fmke_client_k8s_dir, 'create_fmke_client_%s.yaml' % cluster) with open(file_path, 'w') as f: yaml.safe_dump(doc, f) fmke_client_files.append(file_path) logger.info("Running fmke client instances on each DC") logger.debug("Init configurator: k8s_resources_configurator") configurator = k8s_resources_configurator() configurator.deploy_k8s_resources(files=fmke_client_files, namespace=kube_namespace) t = '0' with open(os.path.join(fmke_client_k8s_dir, 'fmke_client.config.template')) as search: for line in search: line = line.rstrip() # remove '\n' at end of line if "{duration" in line: t = line.split(',')[1].split('}')[0].strip() timeout = (int(t) + 5)*60 logger.info("Stressing database in %s minutes ....." % t) configurator.wait_k8s_resources(resource='job', label_selectors="app=fmke-client", timeout=timeout, kube_namespace=kube_namespace) logger.info("Finish stressing Antidote database")
def deploy_fmke_client(self, fmke_yaml_path, test_duration, concurrent_clients, n_total_fmke_clients, workload=None, kube_namespace='default'): """Deploy FMKe client on the given K8s cluster Parameters ---------- fmke_yaml_path: str a path to the K8s yaml deployment files test_duration: int the duration to perform the workload concurrent_clients: int the number of concurrent clients n_total_fmke_clients: int the total number of clients need to be deployed on the system workload: dict the workload ratio of FMKe benchmark kube_namespace: str the name of K8s namespace """ logger.debug('Delete old k8s yaml files if exists') for filename in os.listdir(fmke_yaml_path): if filename.startswith('create_fmke_client_') or filename.startswith('fmke_client_'): if '.template' not in filename: try: os.remove(os.path.join(fmke_yaml_path, filename)) except OSError: logger.debug('Error while deleting file') if workload: logger.debug('Create the new workload ratio') new_workload = ',\n'.join([' {%s, %s}' % (key, val) for key, val in workload.items()]) operations = '{operations,[\n%s\n]}.' % new_workload logger.debug('Init configurator: k8s_resources_configurator') configurator = k8s_resources_configurator() fmke_list = configurator.get_k8s_resources(resource='pod', label_selectors='app=fmke', kube_namespace=kube_namespace) fmke_client_files = list() config_file_path = os.path.join(fmke_yaml_path, 'fmke_client.config.template') create_file_path = os.path.join(fmke_yaml_path, 'create_fmke_client.yaml.template') for fmke in fmke_list.items: node = fmke.spec.node_name.split('.')[0] # Modify fmke_client config files with new values logger.debug('Create fmke_client config files to stress database for each AntidoteDB DC') with open(config_file_path) as f: doc = f.read() doc = doc.replace('127.0.0.1', '%s' % fmke.status.pod_ip) doc = doc.replace('{concurrent, 16}.', '{concurrent, %s}.' % concurrent_clients) doc = doc.replace('{duration, 3}.', '{duration, %s}.' % test_duration) doc = doc.replace("'", '"') if workload: doc = re.sub(r'{operations.*', operations, doc, flags=re.S) file_path = os.path.join(fmke_yaml_path, 'fmke_client_%s.config' % node) with open(file_path, 'w') as f: f.write(doc) logger.debug('Create fmke_client folder on each fmke_client node') cmd = 'mkdir -p /tmp/fmke_client' execute_cmd(cmd, fmke.status.host_ip) logger.debug('Upload fmke_client config files to kube_master to be used by kubectl to run fmke_client pods') getput_file(hosts=fmke.status.host_ip, file_paths=[file_path], dest_location='/tmp/fmke_client/', action='put') logger.debug('Create create_fmke_client.yaml files to deploy one FMKe client') with open(create_file_path) as f: doc = yaml.safe_load(f) doc['metadata']['name'] = 'fmke-client-%s' % node doc['spec']['template']['spec']['containers'][0]['lifecycle']['postStart']['exec']['command'] = [ 'cp', '/cluster_node/fmke_client_%s.config' % node, '/fmke_client/fmke_client.config'] doc['spec']['template']['spec']['nodeSelector'] = { 'service': 'fmke', 'kubernetes.io/hostname': '%s' % fmke.spec.node_name} file_path = os.path.join(fmke_yaml_path, 'create_fmke_client_%s.yaml' % node) with open(file_path, 'w') as f: yaml.safe_dump(doc, f) fmke_client_files.append(file_path) logger.info('Starting FMKe client instances on each AntidoteDB DC') configurator.deploy_k8s_resources(files=fmke_client_files, namespace=kube_namespace) sleep(20) logger.info('Checking if deploying enough the number of running FMKe_client or not') fmke_client_list = configurator.get_k8s_resources_name(resource='pod', label_selectors='app=fmke-client', kube_namespace=kube_namespace) if len(fmke_client_list) != n_total_fmke_clients: logger.info('n_fmke_client = %s, n_deployed_fmke_client = %s' %(n_total_fmke_clients, len(fmke_client_list))) raise CancelException('Cannot deploy enough FMKe_client') logger.info('Stressing database in %s minutes .....' % test_duration) deploy_ok = configurator.wait_k8s_resources(resource='job', label_selectors='app=fmke-client', timeout=(test_duration + 5)*60, kube_namespace=kube_namespace) if not deploy_ok: logger.error('Cannot wait until all FMKe client instances running completely') raise CancelException('Cannot wait until all FMKe client instances running completely') logger.info('Finish stressing AntidoteDB cluster')
def deploy_elmerfs(self, kube_master, kube_namespace, elmerfs_hosts): logger.info("Starting deploying elmerfs on hosts") configurator = packages_configurator() # configurator.install_packages(["libfuse2", "wget", "jq"], elmerfs_hosts) elmerfs_repo = self.configs["exp_env"]["elmerfs_repo"] elmerfs_version = self.configs["exp_env"]["elmerfs_version"] elmerfs_file_path = self.configs["exp_env"]["elmerfs_path"] if elmerfs_repo is None: elmerfs_repo = "https://github.com/scality/elmerfs" if elmerfs_version is None: elmerfs_version = "latest" logger.info("Killing elmerfs process if it is running") for host in elmerfs_hosts: cmd = "pidof elmerfs" _, r = execute_cmd(cmd, host) pids = r.processes[0].stdout.strip().split(" ") if len(pids) >= 1 and pids[0] != '': for pid in pids: cmd = "kill %s" % pid.strip() execute_cmd(cmd, host) cmd = "umount /tmp/dc-$(hostname)" execute_cmd(cmd, host) cmd = "rm -rf /tmp/dc-$(hostname)" execute_cmd(cmd, host) logger.info("Delete elmerfs project folder on host (if existing)") cmd = "rm -rf /tmp/elmerfs_repo" execute_cmd(cmd, kube_master) if elmerfs_file_path is None: logger.info("Downloading elmerfs project from the repo") cmd = """curl \ -H "Accept: application/vnd.github.v3+json" \ https://api.github.com/repos/scality/elmerfs/releases/%s | jq ".tag_name" \ | xargs -I tag_name git clone https://github.com/scality/elmerfs.git --branch tag_name --single-branch /tmp/elmerfs_repo """ % elmerfs_version execute_cmd(cmd, kube_master) cmd = "cd /tmp/elmerfs_repo \ && git submodule update --init --recursive" execute_cmd(cmd, kube_master) cmd = """cat <<EOF | sudo tee /tmp/elmerfs_repo/Dockerfile FROM rust:1.47 RUN mkdir /elmerfs WORKDIR /elmerfs COPY . . RUN apt-get update \ && apt-get -y install libfuse-dev RUN cargo build --release CMD ["/bin/bash"] """ execute_cmd(cmd, kube_master) logger.info("Building elmerfs") cmd = " cd /tmp/elmerfs_repo/ \ && docker build -t elmerfs ." execute_cmd(cmd, kube_master) cmd = "docker run --name elmerfs elmerfs \ && docker cp -L elmerfs:/elmerfs/target/release/main /tmp/elmerfs \ && docker rm elmerfs" execute_cmd(cmd, kube_master) getput_file( hosts=[kube_master], file_paths=["/tmp/elmerfs"], dest_location="/tmp", action="get", ) elmerfs_file_path = "/tmp/elmerfs" logger.info( "Uploading elmerfs binary file from local to %s elmerfs hosts" % len(elmerfs_hosts)) getput_file( hosts=elmerfs_hosts, file_paths=[elmerfs_file_path], dest_location="/tmp", action="put", ) cmd = "chmod +x /tmp/elmerfs \ && mkdir -p /tmp/dc-$(hostname)" execute_cmd(cmd, elmerfs_hosts) logger.debug("Getting IP of antidoteDB instances on nodes") antidote_ips = dict() configurator = k8s_resources_configurator() pod_list = configurator.get_k8s_resources( resource="pod", label_selectors="app=antidote", kube_namespace=kube_namespace, ) for pod in pod_list.items: node = pod.spec.node_name if node not in antidote_ips: antidote_ips[node] = list() antidote_ips[node].append(pod.status.pod_ip) for host in elmerfs_hosts: antidote_options = [ "--antidote=%s:8087" % ip for ip in antidote_ips[host] ] elmerfs_cmd = "RUST_BACKTRACE=1 RUST_LOG=debug nohup /tmp/elmerfs %s --mount=/tmp/dc-$(hostname) --force-view=$ELMERFS_UID > /tmp/elmer.log 2>&1" % " ".join( antidote_options) logger.info("Starting elmerfs on %s with cmd: %s" % (host, elmerfs_cmd)) execute_cmd(elmerfs_cmd, host, mode='start') sleep(10) for i in range(10): cmd = "pidof elmerfs" _, r = execute_cmd(cmd, host) pid = r.processes[0].stdout.strip().split(" ") if len(pid) >= 1 and pid[0].strip(): break else: execute_cmd(elmerfs_cmd, host, mode="start") sleep(10) else: logger.info("Cannot deploy elmerfs on host %s" % host) return False logger.info("Finish deploying elmerfs\n") return True
def install_elmerfs(self, kube_master, elmerfs_hosts, elmerfs_mountpoint, elmerfs_repo, elmerfs_version, elmerfs_path): # Create folder to build the elmerfs from the repo # cmd = 'rm -rf /tmp/elmerfs_repo && mkdir -p /tmp/elmerfs_repo' # execute_cmd(cmd, elmerfs_hosts) if elmerfs_repo is None: elmerfs_repo = 'https://github.com/scality/elmerfs' if elmerfs_version is None: elmerfs_version = 'latest' if elmerfs_path is None: logger.info('Installing elmerfs') configurator = packages_configurator() configurator.install_packages(['libfuse2', 'wget', 'jq'], elmerfs_hosts) logger.info('Create folder to build the elmerfs from the repo') # Create folder to build the elmerfs from the repo cmd = 'rm -rf /tmp/elmerfs_repo && mkdir -p /tmp/elmerfs_repo' execute_cmd(cmd, kube_master) logger.info('Downloading elmerfs project from the repo') cmd = '''curl \ -H 'Accept: application/vnd.github.v3+json' \ https://api.github.com/repos/scality/elmerfs/releases/%s | jq '.tag_name' \ | xargs -I tag_name git clone https://github.com/scality/elmerfs.git --branch tag_name --single-branch /tmp/elmerfs_repo ''' % elmerfs_version execute_cmd(cmd, kube_master) logger.info('update the repo') cmd = 'cd /tmp/elmerfs_repo \ && git submodule update --init --recursive' execute_cmd(cmd, kube_master) logger.info('Creating the Docker file') cmd = '''cat <<EOF | sudo tee /tmp/elmerfs_repo/Dockerfile FROM rust:1.47 RUN mkdir /elmerfs WORKDIR /elmerfs COPY . . RUN apt-get update \ && apt-get -y install libfuse-dev RUN cargo build --release CMD ['/bin/bash'] ''' execute_cmd(cmd, kube_master) logger.info('Building elmerfs Docker image') cmd = 'cd /tmp/elmerfs_repo/ \ && docker build -t elmerfs .' execute_cmd(cmd, kube_master) logger.info('Building elmerfs') cmd = 'docker run --name elmerfs elmerfs \ && docker cp -L elmerfs:/elmerfs/target/release/main /tmp/elmerfs \ && docker rm elmerfs' execute_cmd(cmd, kube_master) getput_file(hosts=[kube_master], file_paths=['/tmp/elmerfs'], dest_location='/tmp', action='get',) elmerfs_path = '/tmp/elmerfs' logger.info('Uploading elmerfs binary file from local to %s elmerfs hosts' % len(elmerfs_hosts)) getput_file(hosts=elmerfs_hosts, file_paths=[elmerfs_path], dest_location='/tmp', action='put') cmd = 'chmod +x /tmp/elmerfs' execute_cmd(cmd, elmerfs_hosts) logger.info('Create mountpoint and result folder') cmd = 'rm -rf /tmp/results && mkdir -p /tmp/results && \ rm -rf %s && mkdir -p %s' % (elmerfs_mountpoint, elmerfs_mountpoint) execute_cmd(cmd, elmerfs_hosts)
def deploy_elmerfs(self, kube_master, kube_namespace, elmerfs_hosts): logger.info("Starting deploying elmerfs on hosts") configurator = packages_configurator() configurator.install_packages(['libfuse2', 'wget', 'jq'], elmerfs_hosts) elmerfs_repo = self.configs['exp_env']['elmerfs_repo'] elmerfs_version = self.configs['exp_env']['elmerfs_version'] if elmerfs_repo is None: elmerfs_repo = 'https://github.com/scality/elmerfs' if elmerfs_version is None: elmerfs_version = 'latest' logger.info('Killing elmerfs process if it is running') for host in elmerfs_hosts: cmd = "ps aux | grep elmerfs | awk '{print$2}'" _, r = execute_cmd(cmd, host) pids = r.processes[0].stdout.strip().split('\r\n') if len(pids) >= 3: cmd = "kill %s && umount /tmp/dc-$(hostname)" % pids[0] execute_cmd(cmd, host) logger.info("Downloading elmerfs project from the repo") cmd = '''curl \ -H "Accept: application/vnd.github.v3+json" \ https://api.github.com/repos/scality/elmerfs/releases/%s | jq ".tag_name" \ | xargs -I tag_name git clone https://github.com/scality/elmerfs.git --branch tag_name --single-branch /tmp/elmerfs_repo ''' % elmerfs_version execute_cmd(cmd, kube_master) cmd = "cd /tmp/elmerfs_repo \ && git submodule update --init --recursive" execute_cmd(cmd, kube_master) cmd = '''cat <<EOF | sudo tee /tmp/elmerfs_repo/Dockerfile FROM rust:1.47 RUN mkdir /elmerfs WORKDIR /elmerfs COPY . . RUN apt-get update \ && apt-get -y install libfuse-dev RUN cargo build --release CMD ["/bin/bash"] ''' execute_cmd(cmd, kube_master) logger.info("Building elmerfs") cmd = " cd /tmp/elmerfs_repo/ \ && docker build -t elmerfs ." execute_cmd(cmd, kube_master) cmd = "docker run --name elmerfs elmerfs \ && docker cp -L elmerfs:/elmerfs/target/release/main /tmp/elmerfs \ && docker rm elmerfs" execute_cmd(cmd, kube_master) getput_file(hosts=[kube_master], file_paths=['/tmp/elmerfs'], dest_location='/tmp', action='get') elmerfs_file_path = '/tmp/elmerfs' logger.info( "Uploading elmerfs binary file from local to %s elmerfs hosts" % len(elmerfs_hosts)) getput_file(hosts=elmerfs_hosts, file_paths=[elmerfs_file_path], dest_location='/tmp', action='put') cmd = "chmod +x /tmp/elmerfs \ && mkdir -p /tmp/dc-$(hostname)" execute_cmd(cmd, elmerfs_hosts) logger.debug('Getting IP of antidoteDB instances on nodes') antidote_ips = dict() configurator = k8s_resources_configurator() pod_list = configurator.get_k8s_resources( resource='pod', label_selectors='app=antidote', kube_namespace=kube_namespace) for pod in pod_list.items: node = pod.spec.node_name if node not in antidote_ips: antidote_ips[node] = list() antidote_ips[node].append(pod.status.pod_ip) for host in elmerfs_hosts: antidote_options = [ "--antidote=%s:8087" % ip for ip in antidote_ips[host] ] cmd = "RUST_BACKTRACE=1 RUST_LOG=debug nohup /tmp/elmerfs %s --mount=/tmp/dc-$(hostname) --no-locks > /tmp/elmer.log 2>&1" % " ".join( antidote_options) logger.info("Starting elmerfs on %s with cmd: %s" % (host, cmd)) execute_cmd(cmd, host, mode='start') sleep(5) logger.info('Finish deploying elmerfs\n')