def config_fmke_pop(self, kube_namespace): logger.info('---------------------------') logger.info('4. Starting populating data') fmke_k8s_dir = self.configs['exp_env']['fmke_yaml_path'] logger.debug('Modify the populate_data template file') configurator = k8s_resources_configurator() fmke_list = configurator.get_k8s_resources(resource='pod', label_selectors='app=fmke', kube_namespace=kube_namespace) logger.info('fmke_list: %s' % len(fmke_list.items)) fmke_IPs = list() for cluster in self.configs['exp_env']['clusters']: for fmke in fmke_list.items: if cluster in fmke.metadata.name: fmke_IPs.append('fmke@%s' % fmke.status.pod_ip) with open(os.path.join(fmke_k8s_dir, 'populate_data.yaml.template')) as f: doc = yaml.safe_load(f) doc['metadata']['name'] = 'populate-data-without-prescriptions' doc['spec']['template']['spec']['containers'][0]['args'] = ['-f -d small --noprescriptions'] + fmke_IPs with open(os.path.join(fmke_k8s_dir, 'populate_data.yaml'), 'w') as f: yaml.safe_dump(doc, f) logger.info("Populating the FMKe benchmark data") logger.debug("Init configurator: k8s_resources_configurator") configurator = k8s_resources_configurator() configurator.deploy_k8s_resources(files=[os.path.join(fmke_k8s_dir, 'populate_data.yaml')], namespace=kube_namespace) logger.info('Waiting for populating data') configurator.wait_k8s_resources(resource='job', label_selectors="app=fmke_pop", kube_namespace=kube_namespace) logger.debug('Modify the populate_data file to populate prescriptions') with open(os.path.join(fmke_k8s_dir, 'populate_data.yaml.template')) as f: doc = yaml.safe_load(f) doc['metadata']['name'] = 'populate-data-with-onlyprescriptions' doc['spec']['template']['spec']['containers'][0]['args'] = ['-f -d small --onlyprescriptions -p 1'] + fmke_IPs with open(os.path.join(fmke_k8s_dir, 'populate_data.yaml'), 'w') as f: yaml.safe_dump(doc, f) logger.info("Populating the FMKe benchmark data with prescriptions") configurator.deploy_k8s_resources(files=[os.path.join(fmke_k8s_dir, 'populate_data.yaml')], namespace=kube_namespace) logger.info('Waiting for populating data') configurator.wait_k8s_resources(resource='job', label_selectors="app=fmke_pop", timeout=90, kube_namespace=kube_namespace) logger.info('Finish populating data')
def config_fmke(self, kube_master, kube_namespace): logger.info('------------------------------------') logger.info('3. Starting deploying FMKe benchmark') fmke_k8s_dir = self.configs['exp_env']['fmke_yaml_path'] logger.debug('Delete old deployment files') for filename in os.listdir(fmke_k8s_dir): if '.template' not in filename: try: os.remove(os.path.join(fmke_k8s_dir, filename)) except OSError: logger.debug("Error while deleting file") logger.debug('Create headless service file') file1 = os.path.join(fmke_k8s_dir, 'headlessService.yaml.template') file2 = os.path.join(fmke_k8s_dir, 'headlessService.yaml') shutil.copyfile(file1, file2) logger.debug('Create FMKe statefulSet files for each DC') file_path = os.path.join(fmke_k8s_dir, 'statefulSet_fmke.yaml.template') with open(file_path) as f: doc = yaml.safe_load(f) configurator = k8s_resources_configurator() service_list = configurator.get_k8s_resources(resource='service', label_selectors='app=antidote,type=exposer-service', kube_namespace=kube_namespace) for cluster in self.configs['exp_env']['clusters']: # Get IP of antidote DC exposer service for each cluster for service in service_list.items: if cluster in service.metadata.name: ip = service.spec.cluster_ip doc['spec']['replicas'] = self.configs['exp_env']['n_fmke_app_per_dc'] doc['metadata']['name'] = 'fmke-%s' % cluster doc['spec']['template']['spec']['containers'][0]['env'] = [ {'name': 'DATABASE_ADDRESSES', 'value': ip}] doc['spec']['template']['spec']['nodeSelector'] = {'service_g5k': 'fmke', 'cluster_g5k': '%s' % cluster} file_path = os.path.join(fmke_k8s_dir, 'statefulSet_fmke_%s.yaml' % cluster) with open(file_path, 'w') as f: yaml.safe_dump(doc, f) logger.info("Starting FMKe instances on each Antidote DC") logger.debug("Init configurator: k8s_resources_configurator") configurator = k8s_resources_configurator() configurator.deploy_k8s_resources(path=fmke_k8s_dir, namespace=kube_namespace) logger.info('Waiting until all fmke app servers are up') configurator.wait_k8s_resources(resource='pod', label_selectors="app=fmke", kube_namespace=kube_namespace) logger.info('Finish deploying FMKe benchmark')
def _setup_g5k_kube_volumes(self, kube_workers, n_pv=3): logger.info("Setting volumes on %s kubernetes workers" % len(kube_workers)) cmd = """umount /dev/sda5; mount -t ext4 /dev/sda5 /tmp""" execute_cmd(cmd, kube_workers) logger.debug( "Create n_pv partitions on the physical disk to make a PV can be shared" ) cmd = """for i in $(seq 1 %s); do mkdir -p /tmp/pv/vol${i} mkdir -p /mnt/disks/vol${i} mount --bind /tmp/pv/vol${i} /mnt/disks/vol${i} done""" % n_pv execute_cmd(cmd, kube_workers) logger.info("Creating local persistance volumes on Kubernetes cluster") logger.debug("Init configurator: k8s_resources_configurator") configurator = k8s_resources_configurator() antidote_k8s_dir = self.configs["exp_env"]["antidote_yaml_path"] deploy_files = [ os.path.join(antidote_k8s_dir, "local_persistentvolume.yaml"), os.path.join(antidote_k8s_dir, "storageClass.yaml"), ] configurator.deploy_k8s_resources(files=deploy_files) logger.info("Waiting for setting local persistance volumes") configurator.wait_k8s_resources( resource="pod", label_selectors= "app.kubernetes.io/instance=local-volume-provisioner", )
def _set_kube_workers_label(self, kube_workers): logger.info("Set labels for all kubernetes workers") configurator = k8s_resources_configurator() for host in kube_workers: cluster = host.split("-")[0] labels = "cluster_g5k=%s,service_g5k=antidote" % cluster configurator.set_labels_node(host, labels)
def _set_kube_workers_label(self, kube_workers): logger.info('Set labels for all kubernetes workers') configurator = k8s_resources_configurator() for host in kube_workers: cluster = host.split('-')[0] labels = 'cluster=%s,service=antidote' % cluster configurator.set_labels_node(host, labels)
def _setup_g5k_kube_volumes(self, kube_workers, n_pv=3): logger.info('Setting volumes on %s kubernetes workers' % len(kube_workers)) cmd = '''umount /dev/sda5; mount -t ext4 /dev/sda5 /tmp''' execute_cmd(cmd, kube_workers) logger.debug( 'Create n_pv partitions on the physical disk to make a PV can be shared' ) cmd = '''for i in $(seq 1 %s); do mkdir -p /tmp/pv/vol${i} mkdir -p /mnt/disks/vol${i} mount --bind /tmp/pv/vol${i} /mnt/disks/vol${i} done''' % n_pv execute_cmd(cmd, kube_workers) logger.info('Creating local persistance volumes on Kubernetes cluster') logger.debug('Init configurator: k8s_resources_configurator') configurator = k8s_resources_configurator() antidote_k8s_dir = self.configs['exp_env']['antidotedb_yaml_path'] deploy_files = [ os.path.join(antidote_k8s_dir, 'local_persistentvolume.yaml'), os.path.join(antidote_k8s_dir, 'storageClass.yaml'), ] configurator.deploy_k8s_resources(files=deploy_files) logger.info('Waiting for setting local persistance volumes') configurator.wait_k8s_resources( resource='pod', label_selectors= 'app.kubernetes.io/instance=local-volume-provisioner', )
def clean_exp_env(self, kube_namespace): logger.info('1. Cleaning the experiment environment') logger.info('Deleting all k8s running resources from the previous run in namespace "%s"' % kube_namespace) logger.debug('Delete namespace "%s" to delete all the running resources, then create it again' % kube_namespace) configurator = k8s_resources_configurator() configurator.delete_namespace(kube_namespace) configurator.create_namespace(kube_namespace)
def clean_exp_env(self, kube_namespace, n_fmke_client_per_dc): logger.info('1. Cleaning the experiment environment') logger.info( 'Deleting all k8s running resources from the previous run in namespace "%s"' % kube_namespace) logger.debug( 'Delete namespace "%s" to delete all the running resources, then create it again' % kube_namespace) configurator = k8s_resources_configurator() configurator.delete_namespace(kube_namespace) configurator.create_namespace(kube_namespace) if n_fmke_client_per_dc > 0: logger.info( 'Delete all files in /tmp/results folder on fmke_client nodes') fmke_nodes_info = configurator.get_k8s_resources( resource='node', label_selectors='service=fmke', kube_namespace=kube_namespace) fmke_nodes = [ r.metadata.annotations['flannel.alpha.coreos.com/public-ip'] for r in fmke_nodes_info.items ] cmd = 'rm -rf /tmp/results && mkdir -p /tmp/results' execute_cmd(cmd, fmke_nodes)
def connect_antidote_DCs(self, antidote_services_ips, kube_config, kube_namespace): antidote_k8s_dir = self.args.yaml_path logger.debug( 'Creating connectDCs_antidote.yaml to connect all Antidote DCs') file_path = os.path.join(antidote_k8s_dir, 'connectDCs.yaml.template') with open(file_path) as f: doc = yaml.safe_load(f) doc['spec']['template']['spec']['containers'][0]['args'] = [ '--connectDcs' ] + antidote_services_ips file_path = os.path.join(antidote_k8s_dir, 'connectDCs_antidote.yaml') with open(file_path, 'w') as f: yaml.safe_dump(doc, f) logger.info("Connecting all Antidote DCs into a cluster") configurator = k8s_resources_configurator() configurator.deploy_k8s_resources(kube_config=kube_config, files=[file_path], namespace=kube_namespace) logger.info('Waiting until connecting all Antidote DCs') configurator.wait_k8s_resources(resource='job', label_selectors='app=antidote', kube_config=kube_config, kube_namespace=kube_namespace)
def _setup_g5k_kube_volumes(self, kube_workers): logger.info("Setting volumes on %s kubernetes workers" % len(kube_workers)) N_PV = 3 cmd = '''umount /dev/sda5; mount -t ext4 /dev/sda5 /tmp''' execute_cmd(cmd, kube_workers) cmd = '''for i in $(seq 1 %s); do mkdir -p /tmp/pv/vol${i} mkdir -p /mnt/disks/vol${i} mount --bind /tmp/pv/vol${i} /mnt/disks/vol${i} done''' % N_PV execute_cmd(cmd, kube_workers) logger.info("Creating local persistance volumes on Kubernetes workers") logger.debug("Init configurator: k8s_resources_configurator") configurator = k8s_resources_configurator() antidote_k8s_dir = self.args.yaml_path deploy_files = [ os.path.join(antidote_k8s_dir, 'local_persistentvolume.yaml'), os.path.join(antidote_k8s_dir, 'storageClass.yaml') ] configurator.deploy_k8s_resources(files=deploy_files) logger.info('Waiting for setting local persistance volumes') configurator.wait_k8s_resources( resource='pod', label_selectors= "app.kubernetes.io/instance=local-volume-provisioner")
def _set_kube_workers_label(self, kube_master): configurator = k8s_resources_configurator() for node in self.nodes: if node['ipAddresses'][0]['ip'] == kube_master: pass else: r = configurator.set_labels_node(nodename=node['name'], labels='cluster=%s,service=antidote' % node['region'].lower()) if r is None: logger.info('Cannot Set labels for kubernetes workers') exit()
def run_workflow(self, kube_namespace, kube_master, comb, sweeper): comb_ok = False try: logger.info('=======================================') logger.info('Performing combination: ' + slugify(comb)) self.clean_exp_env(kube_namespace) self.deploy_antidote(kube_namespace, comb) logger.debug( 'Getting hosts and IP of antidoteDB instances on their nodes') antidote_ips = dict() configurator = k8s_resources_configurator() pod_list = configurator.get_k8s_resources( resource='pod', label_selectors='app=antidote', kube_namespace=kube_namespace) for pod in pod_list.items: node_ip = pod.status.host_ip if node_ip not in antidote_ips: antidote_ips[node_ip] = list() antidote_ips[node_ip].append(pod.status.pod_ip) antidote_hosts = list(antidote_ips.keys()) elmerfs_hosts = antidote_hosts is_elmerfs = self.deploy_elmerfs(kube_master, kube_namespace, elmerfs_hosts, antidote_ips) if is_elmerfs: if self.args.monitoring: self.deploy_monitoring(kube_master, kube_namespace) is_finished, hosts = self.run_benchmark(comb, elmerfs_hosts) if is_finished: comb_ok = True self.save_results(comb, hosts) else: raise CancelException('Cannot deploy elmerfs') except (ExecuteCommandException, CancelException) as e: logger.error('Combination exception: %s' % e) comb_ok = False finally: if comb_ok: sweeper.done(comb) logger.info('Finish combination: %s' % slugify(comb)) else: sweeper.cancel(comb) logger.warning(slugify(comb) + ' is canceled') logger.info('%s combinations remaining\n' % len(sweeper.get_remaining())) return sweeper
def clean_k8s_resources(self, kube_namespace): logger.info('1. Deleting all k8s resource from the previous run in namespace "%s"' % kube_namespace) logger.debug('Delete namespace "%s" to delete all the resources, then create it again' % kube_namespace) configurator = k8s_resources_configurator() configurator.delete_namespace(kube_namespace) configurator.create_namespace(kube_namespace) logger.debug('Delete all files in /tmp/results folder on fmke_client nodes') results_nodes = configurator.get_k8s_resources_name(resource='node', label_selectors='service_g5k=fmke_client', kube_namespace=kube_namespace) cmd = 'rm -rf /tmp/results && mkdir -p /tmp/results' execute_cmd(cmd, results_nodes)
def setup_k8s_env(self, kube_master, kube_namespace, kube_workers): self._get_credential(kube_master) logger.info('Create k8s namespace "%s" for this experiment' % kube_namespace) configurator = k8s_resources_configurator() configurator.create_namespace(namespace=kube_namespace) logger.info('Set labels for all kubernetes workers') self._set_kube_workers_label(kube_master) self._setup_ovh_kube_volumes(kube_workers, n_pv=3) logger.info('Finish deploying the Kubernetes cluster')
def save_results(self, comb): logger.info("----------------------------------") logger.info("6. Starting dowloading the results") configurator = k8s_resources_configurator() results_nodes = configurator.get_k8s_resources_name(resource='node', label_selectors='service_g5k=fmke_client') get_results(comb=comb, hosts=results_nodes, remote_result_files=['/tmp/results/'], local_result_dir=self.configs['exp_env']['results_dir']) logger.info("Finish dowloading the results")
def _set_kube_workers_label(self, kube_master): configurator = k8s_resources_configurator() for node in self.nodes: if node['ipAddresses'][0]['ip'] == kube_master: pass else: configurator.set_labels_node(nodename=node['name'], labels='cluster=%s' % node['region'].lower()) if node['id'] in self.data_node_ids: configurator.set_labels_node(nodename=node['name'], labels='service=antidote') else: configurator.set_labels_node(nodename=node['name'], labels='service=fmke')
def config_host(self, kube_master, kube_namespace): logger.debug("Init configurator: kubernetes_configurator") configurator = kubernetes_configurator(hosts=self.hosts, kube_master=kube_master) _, kube_workers = configurator.deploy_kubernetes_cluster() self._get_credential(kube_master) logger.info('Create k8s namespace "%s" for this experiment' % kube_namespace) configurator = k8s_resources_configurator() configurator.create_namespace(namespace=kube_namespace) self._setup_g5k_kube_volumes(kube_workers, n_pv=3) logger.info('Set labels for all kubernetes workers') self._set_kube_workers_label(kube_workers) logger.info("Finish deploying the Kubernetes cluster")
def run_exp_workflow(self, kube_namespace, comb, kube_master, sweeper): comb_ok = False try: logger.info('=======================================') logger.info('Performing combination: ' + slugify(comb)) self.clean_exp_env(kube_namespace, comb['n_fmke_client_per_dc']) self.deploy_antidote(kube_namespace, comb) if self.args.monitoring: prometheus_url, _ = self.deploy_monitoring( kube_master, kube_namespace) self.deploy_fmke_app(kube_namespace, comb) pop_result = self.deploy_fmke_pop(kube_namespace, comb) pop_errors = None if self.args.monitoring: pop_errors = self.get_prometheus_metric( 'antidote_error_count', prometheus_url) pop_errors = sum(pop_errors.values()) logger.info('Total ops errors of population: %s' % pop_errors) if comb['n_fmke_client_per_dc'] > 0: self.deploy_fmke_client(kube_namespace, comb) configurator = k8s_resources_configurator() fmke_nodes_info = configurator.get_k8s_resources( resource='node', label_selectors='service=fmke') fmke_nodes = [ r.metadata. annotations['flannel.alpha.coreos.com/public-ip'] for r in fmke_nodes_info.items ] self.save_results(comb, pop_result, pop_errors, fmke_nodes) else: self.save_results(comb, pop_result, pop_errors) comb_ok = True except (ExecuteCommandException, CancelException) as e: logger.error('Combination exception: %s' % e) comb_ok = False finally: if comb_ok: sweeper.done(comb) logger.info('Finish combination: %s' % slugify(comb)) else: sweeper.cancel(comb) logger.warning(slugify(comb) + ' is canceled') logger.info('%s combinations remaining\n' % len(sweeper.get_remaining())) return sweeper
def config_host(self, kube_master_site, kube_namespace): kube_master = self.args.kube_master if self.args.kube_master is None: antidote_hosts = list() for cluster in self.configs['clusters']: cluster_name = cluster['cluster'] if cluster_name == self.configs['exp_env']['kube_master_site']: antidote_hosts += [ host for host in self.hosts if host.startswith(cluster_name) ][0:cluster['n_nodes'] + 1] else: antidote_hosts += [ host for host in self.hosts if host.startswith(cluster_name) ][0:cluster['n_nodes']] for host in antidote_hosts: if host.startswith(kube_master_site): kube_master = host break kube_workers = self.config_kube(kube_master, antidote_hosts, kube_namespace) else: logger.info('Kubernetes master: %s' % kube_master) self._get_credential(kube_master) configurator = k8s_resources_configurator() deployed_hosts = configurator.get_k8s_resources(resource='node') kube_workers = [ host.metadata.name for host in deployed_hosts.items ] kube_workers.remove(kube_master) logger.info('Installing elmerfs dependencies') configurator = packages_configurator() configurator.install_packages(['libfuse2', 'wget', 'jq'], kube_workers) # Create mount point on elmerfs hosts cmd = 'mkdir -p /tmp/dc-$(hostname)' execute_cmd(cmd, kube_workers) # Installing filebench for running the experiments logger.info('Installing Filebench') self.install_filebench(kube_workers)
def _set_kube_workers_label(self, kube_workers): configurator = k8s_resources_configurator() clusters = dict() for host in kube_workers: cluster = host.split('-')[0] clusters[cluster] = [host] + clusters.get(cluster, list()) configurator.set_labels_node(nodename=host, labels='cluster=%s' % cluster) n_fmke_per_dc = max(self.normalized_parameters['n_fmke_client_per_dc']) n_antidotedb_per_dc = max(self.normalized_parameters['n_nodes_per_dc']) for cluster, list_of_hosts in clusters.items(): for n, service_name in [(n_antidotedb_per_dc, 'antidote'), (n_fmke_per_dc, 'fmke')]: for host in list_of_hosts[0:n]: configurator.set_labels_node(nodename=host, labels='service=%s' % service_name) list_of_hosts = list_of_hosts[n:]
def clean_exp_env(self, kube_namespace, n_fmke_client_per_dc): logger.info('1. Cleaning the experiment environment') logger.info( 'Deleting all k8s running resources from the previous run in namespace "%s"' % kube_namespace) logger.debug( 'Delete namespace "%s" to delete all the running resources, then create it again' % kube_namespace) configurator = k8s_resources_configurator() configurator.delete_namespace(kube_namespace) configurator.create_namespace(kube_namespace) if n_fmke_client_per_dc > 0: logger.info( 'Delete old result files in /tmp/results on FMKe client nodes') fmke_nodes = configurator.get_k8s_resources_name( resource='node', label_selectors='service=fmke', kube_namespace=kube_namespace) cmd = 'rm -rf /tmp/results && mkdir -p /tmp/results' execute_cmd(cmd, fmke_nodes)
def config_host(self, kube_master_site, kube_namespace): kube_master = self.args.kube_master if self.args.kube_master is None: antidote_hosts = list() for cluster in self.configs["exp_env"]["antidote_clusters"]: cluster_name = cluster["cluster"] if cluster_name == self.configs["exp_env"]["kube_master_site"]: antidote_hosts += [ host for host in self.hosts if host.startswith(cluster_name) ][0:cluster["n_antidotedb_per_dc"] + 1] else: antidote_hosts += [ host for host in self.hosts if host.startswith(cluster_name) ][0:cluster["n_antidotedb_per_dc"]] for host in antidote_hosts: if host.startswith(kube_master_site): kube_master = host break elmerfs_hosts = antidote_hosts elmerfs_hosts.remove(kube_master) self.config_kube(kube_master, antidote_hosts, kube_namespace) else: logger.info("Kubernetes master: %s" % kube_master) self._get_credential(kube_master) configurator = k8s_resources_configurator() antidote_hosts = configurator.get_k8s_resources_name( resource="node", label_selectors="service_g5k=antidote") elmerfs_hosts = antidote_hosts self.install_benchsoftware(elmerfs_hosts) return elmerfs_hosts
def config_kube(self, kube_master, antidote_hosts, kube_namespace): logger.info("Starting configuring a Kubernetes cluster") logger.debug("Init configurator: kubernetes_configurator") configurator = kubernetes_configurator(hosts=self.hosts, kube_master=kube_master) configurator.deploy_kubernetes_cluster() self._get_credential(kube_master) logger.info('Create k8s namespace "%s" for this experiment' % kube_namespace) logger.debug("Init configurator: k8s_resources_configurator") configurator = k8s_resources_configurator() configurator.create_namespace(kube_namespace) kube_workers = [host for host in antidote_hosts if host != kube_master] self._setup_g5k_kube_volumes(kube_workers, n_pv=3) self._set_kube_workers_label(kube_workers) logger.info("Finish configuring the Kubernetes cluster\n")
def config_host(self, kube_master_site, kube_namespace): kube_master = self.args.kube_master if self.args.kube_master is None: antidote_hosts = list() for cluster in self.configs['exp_env']['antidote_clusters']: if cluster == self.configs['exp_env']['kube_master_site']: antidote_hosts += [ host for host in self.hosts if host.startswith(cluster) ][0:self.configs['exp_env']['n_antidotedb_per_dc'] + 1] else: antidote_hosts += [ host for host in self.hosts if host.startswith(cluster) ][0:self.configs['exp_env']['n_antidotedb_per_dc']] for host in antidote_hosts: if host.startswith(kube_master_site): kube_master = host break elmerfs_hosts = antidote_hosts elmerfs_hosts.remove(kube_master) self.config_kube(kube_master, antidote_hosts, kube_namespace) else: logger.info('Kubernetes master: %s' % kube_master) self._get_credential(kube_master) configurator = k8s_resources_configurator() antidote_hosts = configurator.get_k8s_resources_name( resource='node', label_selectors='service_g5k=antidote') elmerfs_hosts = antidote_hosts self.deploy_antidote(kube_namespace) self.deploy_elmerfs(kube_master, kube_namespace, elmerfs_hosts) if self.args.monitoring is not None: self.deploy_monitoring(kube_master, kube_namespace)
def config_host(self, kube_namespace): logger.info("Starting configuring an AntidoteDB cluster") antidote_services_ips = list() for cluster in self.clusters: kube_config = self._get_credential(cluster) logger.info('Creating namespace "%s" to deploy Antidote DC' % kube_namespace) configurator = k8s_resources_configurator() configurator.create_namespace(namespace=kube_namespace, kube_config=kube_config) self.config_antidote(kube_config, cluster, kube_namespace) service_name = 'antidote-exposer-%s' % cluster.name antidote_services_ips.append( '%s:8087' % configurator.get_k8s_endpoint_ip( service_name, kube_config, kube_namespace)) logger.info( "Starting connecting all AntidoteDB DCs to create a cluster") self.connect_antidote_DCs(antidote_services_ips, kube_config, kube_namespace) logger.info("Finish configuring an AntidoteDB cluster\n")
def perform_combination(self, kube_namespace, concurrent_clients): logger.info('-----------------------------------------------------------------') logger.info('5. Starting deploying fmke client to stress the Antidote database') fmke_client_k8s_dir = self.configs['exp_env']['fmke_yaml_path'] logger.debug('Delete old k8s yaml files if exists') for filename in os.listdir(fmke_client_k8s_dir): if filename.startswith('create_fmke_client_') or filename.startswith('fmke_client_'): if '.template' not in filename: try: os.remove(os.path.join(fmke_client_k8s_dir, filename)) except OSError: logger.debug("Error while deleting file") logger.debug('Create fmke_client folder on each fmke_client node') configurator = k8s_resources_configurator() exp_nodes = configurator.get_k8s_resources_name(resource='node', label_selectors='service_g5k=fmke_client') cmd = 'mkdir -p /tmp/fmke_client' execute_cmd(cmd, exp_nodes) logger.debug('Create fmke_client config files to stress database for each Antidote DC') file_path = os.path.join(fmke_client_k8s_dir, 'fmke_client.config.template') fmke_list = configurator.get_k8s_resources(resource='pod', label_selectors='app=fmke', kube_namespace=kube_namespace) for cluster in self.configs['exp_env']['clusters']: fmke_IPs = list() for fmke in fmke_list.items: if cluster in fmke.metadata.name: fmke_IPs.append(fmke.status.pod_ip) fmke_ports = [9090 for i in range(0, len(fmke_IPs))] # Modify fmke_client config files with new values with open(file_path) as f: doc = f.read().replace('["127.0.0.1"]', '%s' % fmke_IPs) doc = doc.replace("[9090]", '%s' % fmke_ports) doc = doc.replace("{concurrent, 16}.", '{concurrent, %s}.' % concurrent_clients) doc = doc.replace("'", '"') file_path2 = os.path.join(fmke_client_k8s_dir, 'fmke_client_%s.config' % cluster) with open(file_path2, 'w') as f: f.write(doc) logger.debug('Upload fmke_client config files to kube_master to be used by kubectl to run fmke_client pods') getput_file(hosts=exp_nodes, file_paths=[file_path2], dest_location='/tmp/fmke_client/', action='put') logger.debug('Create create_fmke_client.yaml files to run job stress for each Antidote DC') file_path = os.path.join(fmke_client_k8s_dir, 'create_fmke_client.yaml.template') with open(file_path) as f: doc = yaml.safe_load(f) fmke_client_files = list() for cluster in self.configs['exp_env']['clusters']: doc['spec']['parallelism'] = self.configs['exp_env']['n_fmke_client_per_dc'] doc['spec']['completions'] = self.configs['exp_env']['n_fmke_client_per_dc'] doc['metadata']['name'] = 'fmke-client-%s' % cluster doc['spec']['template']['spec']['containers'][0]['lifecycle']['postStart']['exec']['command'] = [ "cp", "/cluster_node/fmke_client_%s.config" % cluster, "/fmke_client/fmke_client.config"] doc['spec']['template']['spec']['nodeSelector'] = { 'service_g5k': 'fmke_client', 'cluster_g5k': '%s' % cluster} file_path = os.path.join(fmke_client_k8s_dir, 'create_fmke_client_%s.yaml' % cluster) with open(file_path, 'w') as f: yaml.safe_dump(doc, f) fmke_client_files.append(file_path) logger.info("Running fmke client instances on each DC") logger.debug("Init configurator: k8s_resources_configurator") configurator = k8s_resources_configurator() configurator.deploy_k8s_resources(files=fmke_client_files, namespace=kube_namespace) t = '0' with open(os.path.join(fmke_client_k8s_dir, 'fmke_client.config.template')) as search: for line in search: line = line.rstrip() # remove '\n' at end of line if "{duration" in line: t = line.split(',')[1].split('}')[0].strip() timeout = (int(t) + 5)*60 logger.info("Stressing database in %s minutes ....." % t) configurator.wait_k8s_resources(resource='job', label_selectors="app=fmke-client", timeout=timeout, kube_namespace=kube_namespace) logger.info("Finish stressing Antidote database")
def deploy_fmke_client(self, fmke_yaml_path, test_duration, concurrent_clients, n_total_fmke_clients, workload=None, kube_namespace='default'): """Deploy FMKe client on the given K8s cluster Parameters ---------- fmke_yaml_path: str a path to the K8s yaml deployment files test_duration: int the duration to perform the workload concurrent_clients: int the number of concurrent clients n_total_fmke_clients: int the total number of clients need to be deployed on the system workload: dict the workload ratio of FMKe benchmark kube_namespace: str the name of K8s namespace """ logger.debug('Delete old k8s yaml files if exists') for filename in os.listdir(fmke_yaml_path): if filename.startswith('create_fmke_client_') or filename.startswith('fmke_client_'): if '.template' not in filename: try: os.remove(os.path.join(fmke_yaml_path, filename)) except OSError: logger.debug('Error while deleting file') if workload: logger.debug('Create the new workload ratio') new_workload = ',\n'.join([' {%s, %s}' % (key, val) for key, val in workload.items()]) operations = '{operations,[\n%s\n]}.' % new_workload logger.debug('Init configurator: k8s_resources_configurator') configurator = k8s_resources_configurator() fmke_list = configurator.get_k8s_resources(resource='pod', label_selectors='app=fmke', kube_namespace=kube_namespace) fmke_client_files = list() config_file_path = os.path.join(fmke_yaml_path, 'fmke_client.config.template') create_file_path = os.path.join(fmke_yaml_path, 'create_fmke_client.yaml.template') for fmke in fmke_list.items: node = fmke.spec.node_name.split('.')[0] # Modify fmke_client config files with new values logger.debug('Create fmke_client config files to stress database for each AntidoteDB DC') with open(config_file_path) as f: doc = f.read() doc = doc.replace('127.0.0.1', '%s' % fmke.status.pod_ip) doc = doc.replace('{concurrent, 16}.', '{concurrent, %s}.' % concurrent_clients) doc = doc.replace('{duration, 3}.', '{duration, %s}.' % test_duration) doc = doc.replace("'", '"') if workload: doc = re.sub(r'{operations.*', operations, doc, flags=re.S) file_path = os.path.join(fmke_yaml_path, 'fmke_client_%s.config' % node) with open(file_path, 'w') as f: f.write(doc) logger.debug('Create fmke_client folder on each fmke_client node') cmd = 'mkdir -p /tmp/fmke_client' execute_cmd(cmd, fmke.status.host_ip) logger.debug('Upload fmke_client config files to kube_master to be used by kubectl to run fmke_client pods') getput_file(hosts=fmke.status.host_ip, file_paths=[file_path], dest_location='/tmp/fmke_client/', action='put') logger.debug('Create create_fmke_client.yaml files to deploy one FMKe client') with open(create_file_path) as f: doc = yaml.safe_load(f) doc['metadata']['name'] = 'fmke-client-%s' % node doc['spec']['template']['spec']['containers'][0]['lifecycle']['postStart']['exec']['command'] = [ 'cp', '/cluster_node/fmke_client_%s.config' % node, '/fmke_client/fmke_client.config'] doc['spec']['template']['spec']['nodeSelector'] = { 'service': 'fmke', 'kubernetes.io/hostname': '%s' % fmke.spec.node_name} file_path = os.path.join(fmke_yaml_path, 'create_fmke_client_%s.yaml' % node) with open(file_path, 'w') as f: yaml.safe_dump(doc, f) fmke_client_files.append(file_path) logger.info('Starting FMKe client instances on each AntidoteDB DC') configurator.deploy_k8s_resources(files=fmke_client_files, namespace=kube_namespace) sleep(20) logger.info('Checking if deploying enough the number of running FMKe_client or not') fmke_client_list = configurator.get_k8s_resources_name(resource='pod', label_selectors='app=fmke-client', kube_namespace=kube_namespace) if len(fmke_client_list) != n_total_fmke_clients: logger.info('n_fmke_client = %s, n_deployed_fmke_client = %s' %(n_total_fmke_clients, len(fmke_client_list))) raise CancelException('Cannot deploy enough FMKe_client') logger.info('Stressing database in %s minutes .....' % test_duration) deploy_ok = configurator.wait_k8s_resources(resource='job', label_selectors='app=fmke-client', timeout=(test_duration + 5)*60, kube_namespace=kube_namespace) if not deploy_ok: logger.error('Cannot wait until all FMKe client instances running completely') raise CancelException('Cannot wait until all FMKe client instances running completely') logger.info('Finish stressing AntidoteDB cluster')
def config_antidote(self, kube_namespace): logger.info('Starting deploying Antidote cluster') antidote_k8s_dir = self.args.yaml_path logger.info('Deleting all old resources on namspace %s' % kube_namespace) configurator = k8s_resources_configurator() configurator.delete_namespace(namespace=kube_namespace) configurator.create_namespace(namespace=kube_namespace) logger.debug('Delete old deployment files if exists') for filename in os.listdir(antidote_k8s_dir): if filename.startswith('createDC_') or filename.startswith( 'statefulSet_') or filename.startswith( 'exposer-service_') or filename.startswith( 'connectDCs_antidote'): if '.template' not in filename: try: os.remove(os.path.join(antidote_k8s_dir, filename)) except OSError: logger.debug("Error while deleting file") deploy_files = [os.path.join(antidote_k8s_dir, 'headlessService.yaml')] logger.debug('Modify the statefulSet file') file_path = os.path.join(antidote_k8s_dir, 'statefulSet.yaml.template') with open(file_path) as f: doc = yaml.safe_load(f) for cluster in self.configs['clusters']: doc['spec']['replicas'] = cluster['n_nodes'] doc['metadata']['name'] = 'antidote-%s' % cluster['cluster'] doc['spec']['template']['spec']['nodeSelector'] = { 'service_g5k': 'antidote', 'cluster_g5k': '%s' % cluster['cluster'] } file_path = os.path.join( antidote_k8s_dir, 'statefulSet_%s.yaml' % cluster['cluster']) with open(file_path, 'w') as f: yaml.safe_dump(doc, f) deploy_files.append(file_path) logger.info("Starting AntidoteDB instances") logger.debug("Init configurator: k8s_resources_configurator") configurator = k8s_resources_configurator() configurator.deploy_k8s_resources(files=deploy_files, namespace=kube_namespace) logger.info('Waiting until all Antidote instances are up') configurator.wait_k8s_resources(resource='pod', label_selectors='app=antidote', kube_namespace=kube_namespace) logger.debug('Creating createDc.yaml file for each Antidote DC') dcs = dict() for cluster in self.configs['clusters']: dcs[cluster['cluster']] = list() antidote_list = configurator.get_k8s_resources_name( resource='pod', label_selectors='app=antidote', kube_namespace=kube_namespace) for antidote in antidote_list: cluster = antidote.split('-')[1].strip() dcs[cluster].append(antidote.strip()) file_path = os.path.join(antidote_k8s_dir, 'createDC.yaml.template') with open(file_path) as f: doc = yaml.safe_load(f) antidote_masters = list() createdc_files = list() for cluster, pods in dcs.items(): doc['spec']['template']['spec']['containers'][0]['args'] = [ '--createDc', '%s.antidote:8087' % pods[0] ] + ['antidote@%s.antidote' % pod for pod in pods] doc['metadata']['name'] = 'createdc-%s' % cluster antidote_masters.append('%s.antidote:8087' % pods[0]) file_path = os.path.join(antidote_k8s_dir, 'createDC_%s.yaml' % cluster) with open(file_path, 'w') as f: yaml.safe_dump(doc, f) createdc_files.append(file_path) logger.debug('Creating exposer-service.yaml files') file_path = os.path.join(antidote_k8s_dir, 'exposer-service.yaml.template') with open(file_path) as f: doc = yaml.safe_load(f) for cluster, pods in dcs.items(): doc['spec']['selector'][ 'statefulset.kubernetes.io/pod-name'] = pods[0] doc['metadata']['name'] = 'antidote-exposer-%s' % cluster file_path = os.path.join(antidote_k8s_dir, 'exposer-service_%s.yaml' % cluster) with open(file_path, 'w') as f: yaml.safe_dump(doc, f) createdc_files.append(file_path) logger.info("Creating Antidote DCs and exposing services") configurator.deploy_k8s_resources(files=createdc_files, namespace=kube_namespace) logger.info('Waiting until all antidote DCs are created') configurator.wait_k8s_resources(resource='job', label_selectors='app=antidote', kube_namespace=kube_namespace) logger.debug( 'Creating connectDCs_antidote.yaml to connect all Antidote DCs') file_path = os.path.join(antidote_k8s_dir, 'connectDCs.yaml.template') with open(file_path) as f: doc = yaml.safe_load(f) doc['spec']['template']['spec']['containers'][0]['args'] = [ '--connectDcs' ] + antidote_masters file_path = os.path.join(antidote_k8s_dir, 'connectDCs_antidote.yaml') with open(file_path, 'w') as f: yaml.safe_dump(doc, f) logger.info("Connecting all Antidote DCs into a cluster") configurator.deploy_k8s_resources(files=[file_path], namespace=kube_namespace) logger.info('Waiting until connecting all Antidote DCs') configurator.wait_k8s_resources(resource='job', label_selectors='app=antidote', kube_namespace=kube_namespace) logger.info('Finish deploying an Antidote cluster')
def deploy_fmke_pop(self, fmke_yaml_path, dataset, n_fmke_pop_process, clusters, stabilizing_time, timeout=600, kube_namespace='default'): """Deploy FMKe populator on the given K8s cluster to generate the data in to an AntidoteDB cluster Parameters ---------- fmke_yaml_path: str a path to the K8s yaml deployment files dataset: str name of the dataset of FMKe benchmark to generate into the database n_fmke_pop_process: int the number of FMKe populator processes running simultaneously clusters: list list of cluster name stabilizing_time: int the time to wait for synchronize and replication process between datacenters kube_namespace: str the name of K8s namespace """ logger.debug('Modify the populate_data template file') configurator = k8s_resources_configurator() fmke_list = configurator.get_k8s_resources(resource='pod', label_selectors='app=fmke', kube_namespace=kube_namespace) fmke_IPs = list() for cluster in clusters: for fmke in fmke_list.items: if cluster.lower() in fmke.metadata.name: fmke_IPs.append('fmke@%s' % fmke.status.pod_ip) with open(os.path.join(fmke_yaml_path, 'populate_data.yaml.template')) as f: doc = yaml.safe_load(f) doc['metadata']['name'] = 'populate-data-without-prescriptions' doc['spec']['template']['spec']['containers'][0]['args'] = ['-f -d %s --noprescriptions -p %s' % (dataset, n_fmke_pop_process)] + fmke_IPs with open(os.path.join(fmke_yaml_path, 'populate_data.yaml'), 'w') as f: yaml.safe_dump(doc, f) logger.info('Populating the FMKe benchmark data without prescriptions') logger.debug('Init configurator: k8s_resources_configurator') configurator = k8s_resources_configurator() configurator.deploy_k8s_resources(files=[os.path.join(fmke_yaml_path, 'populate_data.yaml')], namespace=kube_namespace) logger.info('Waiting for populating data without prescriptions') deploy_ok = configurator.wait_k8s_resources(resource='job', label_selectors='app=fmke_pop', timeout=timeout, kube_namespace=kube_namespace) if not deploy_ok: raise CancelException('Cannot wait until finishing populating data') logger.info('Checking if the populating process finished successfully or not') fmke_pop_pods = configurator.get_k8s_resources_name(resource='pod', label_selectors='job-name=populate-data-without-prescriptions', kube_namespace=kube_namespace) logger.debug('FMKe pod name: %s' % fmke_pop_pods[0]) pop_result = dict() if len(fmke_pop_pods) > 0: log = configurator.get_k8s_pod_log(pod_name=fmke_pop_pods[0], kube_namespace=kube_namespace) last_line = log.strip().split('\n')[-1] logger.info('Last line of log: %s' % last_line) if 'Populated' in last_line and 'entities in' in last_line and 'avg' in last_line: result = log.strip().split('\n')[-1].split(' ') if len(result) == 8: pop_result = result[4] + '\n' + result[6] if len(result) == 9: pop_result = result[4] + '\n' + result[7] t = stabilizing_time logger.info('Waiting %s minutes for the replication and key distribution mechanisms between DCs' % t) sleep(t*60) else: raise CancelException('Populating process ERROR') logger.debug('FMKe populator result: \n%s' % pop_result) logger.debug('Modify the populate_data file to populate prescriptions') with open(os.path.join(fmke_yaml_path, 'populate_data.yaml.template')) as f: doc = yaml.safe_load(f) doc['metadata']['name'] = 'populate-data-with-onlyprescriptions' doc['spec']['template']['spec']['containers'][0]['args'] = [ '-f --onlyprescriptions -p 1'] + fmke_IPs with open(os.path.join(fmke_yaml_path, 'populate_data.yaml'), 'w') as f: yaml.safe_dump(doc, f) logger.info('Populating the FMKe benchmark data with prescriptions') configurator.deploy_k8s_resources(files=[os.path.join(fmke_yaml_path, 'populate_data.yaml')], namespace=kube_namespace) logger.info('Waiting for populating data with prescriptions') configurator.wait_k8s_resources(resource='job', label_selectors='app=fmke_pop', timeout=timeout, kube_namespace=kube_namespace) logger.info('Checking if the populating process finished successfully or not') fmke_pop_pods = configurator.get_k8s_resources_name(resource='pod', label_selectors='job-name=populate-data-with-onlyprescriptions', kube_namespace=kube_namespace) logger.info('FMKe pod: %s' % fmke_pop_pods[0]) if len(fmke_pop_pods) > 0: log = configurator.get_k8s_pod_log( pod_name=fmke_pop_pods[0], kube_namespace=kube_namespace) last_line = log.strip().split('\n')[-1] logger.info('Last line of log: %s' % last_line) if 'Populated' not in last_line: raise CancelException('Populating process ERROR') t = stabilizing_time logger.info('Waiting %s minutes for the replication and key distribution mechanisms between DCs' % t) sleep(t*60) logger.info('Finish populating data') return pop_result
def _set_kube_workers_label(self, kube_workers): configurator = k8s_resources_configurator() for host in kube_workers: cluster = host.split('-')[0] labels = 'cluster_g5k=%s,service_g5k=antidote' % cluster configurator.set_labels_node(nodename=host, labels=labels)