def get_results(comb, hosts, remote_result_files, local_result_dir):
"""Get all the results files from remote hosts to a local result directory
Parameters
----------
comb: dict
a dictionary that contains the set of parameters for a specific run
key: str, the name of the experiment parameter
value: object, the value of the experiment parameter in this combination
hosts: list
a list of hosts to get the results from
remote_result_files: list
a list of results files on the remote nodes
local_result_dir: str
the path to the directory to store the results on the local node
"""
logger.info('Create combination dir locally')
comb_dir = create_combination_dir(comb, local_result_dir)
logger.info('Download the result')
getput_file(hosts=hosts,
file_paths=remote_result_files,
dest_location=comb_dir,
action='get')
return comb_dir
def _get_credential(self, kube_master):
home = os.path.expanduser('~')
kube_dir = os.path.join(home, '.kube')
if not os.path.exists(kube_dir):
os.mkdir(kube_dir)
getput_file(hosts=[kube_master], file_paths=['~/.kube/config'], dest_location=kube_dir, action='get')
kube_config_file = os.path.join(kube_dir, 'config')
config.load_kube_config(config_file=kube_config_file)
logger.info('Kubernetes config file is stored at: %s' % kube_config_file)
def _get_credential(self, kube_master):
home = os.path.expanduser("~")
kube_dir = os.path.join(home, ".kube")
if not os.path.exists(kube_dir):
os.mkdir(kube_dir)
getput_file(
hosts=[kube_master],
file_paths=["~/.kube/config"],
dest_location=kube_dir,
action="get",
)
kube_config_file = os.path.join(kube_dir, "config")
config.load_kube_config(config_file=kube_config_file)
logger.info("Kubernetes config file is stored at: %s" %
kube_config_file)
def perform_combination(self, kube_namespace, concurrent_clients):
logger.info('-----------------------------------------------------------------')
logger.info('5. Starting deploying fmke client to stress the Antidote database')
fmke_client_k8s_dir = self.configs['exp_env']['fmke_yaml_path']
logger.debug('Delete old k8s yaml files if exists')
for filename in os.listdir(fmke_client_k8s_dir):
if filename.startswith('create_fmke_client_') or filename.startswith('fmke_client_'):
if '.template' not in filename:
try:
os.remove(os.path.join(fmke_client_k8s_dir, filename))
except OSError:
logger.debug("Error while deleting file")
logger.debug('Create fmke_client folder on each fmke_client node')
configurator = k8s_resources_configurator()
exp_nodes = configurator.get_k8s_resources_name(resource='node',
label_selectors='service_g5k=fmke_client')
cmd = 'mkdir -p /tmp/fmke_client'
execute_cmd(cmd, exp_nodes)
logger.debug('Create fmke_client config files to stress database for each Antidote DC')
file_path = os.path.join(fmke_client_k8s_dir, 'fmke_client.config.template')
fmke_list = configurator.get_k8s_resources(resource='pod',
label_selectors='app=fmke',
kube_namespace=kube_namespace)
for cluster in self.configs['exp_env']['clusters']:
fmke_IPs = list()
for fmke in fmke_list.items:
if cluster in fmke.metadata.name:
fmke_IPs.append(fmke.status.pod_ip)
fmke_ports = [9090 for i in range(0, len(fmke_IPs))]
# Modify fmke_client config files with new values
with open(file_path) as f:
doc = f.read().replace('["127.0.0.1"]', '%s' % fmke_IPs)
doc = doc.replace("[9090]", '%s' % fmke_ports)
doc = doc.replace("{concurrent, 16}.", '{concurrent, %s}.' % concurrent_clients)
doc = doc.replace("'", '"')
file_path2 = os.path.join(fmke_client_k8s_dir, 'fmke_client_%s.config' % cluster)
with open(file_path2, 'w') as f:
f.write(doc)
logger.debug('Upload fmke_client config files to kube_master to be used by kubectl to run fmke_client pods')
getput_file(hosts=exp_nodes, file_paths=[file_path2], dest_location='/tmp/fmke_client/', action='put')
logger.debug('Create create_fmke_client.yaml files to run job stress for each Antidote DC')
file_path = os.path.join(fmke_client_k8s_dir, 'create_fmke_client.yaml.template')
with open(file_path) as f:
doc = yaml.safe_load(f)
fmke_client_files = list()
for cluster in self.configs['exp_env']['clusters']:
doc['spec']['parallelism'] = self.configs['exp_env']['n_fmke_client_per_dc']
doc['spec']['completions'] = self.configs['exp_env']['n_fmke_client_per_dc']
doc['metadata']['name'] = 'fmke-client-%s' % cluster
doc['spec']['template']['spec']['containers'][0]['lifecycle']['postStart']['exec']['command'] = [
"cp", "/cluster_node/fmke_client_%s.config" % cluster, "/fmke_client/fmke_client.config"]
doc['spec']['template']['spec']['nodeSelector'] = {
'service_g5k': 'fmke_client', 'cluster_g5k': '%s' % cluster}
file_path = os.path.join(fmke_client_k8s_dir, 'create_fmke_client_%s.yaml' % cluster)
with open(file_path, 'w') as f:
yaml.safe_dump(doc, f)
fmke_client_files.append(file_path)
logger.info("Running fmke client instances on each DC")
logger.debug("Init configurator: k8s_resources_configurator")
configurator = k8s_resources_configurator()
configurator.deploy_k8s_resources(files=fmke_client_files, namespace=kube_namespace)
t = '0'
with open(os.path.join(fmke_client_k8s_dir, 'fmke_client.config.template')) as search:
for line in search:
line = line.rstrip() # remove '\n' at end of line
if "{duration" in line:
t = line.split(',')[1].split('}')[0].strip()
timeout = (int(t) + 5)*60
logger.info("Stressing database in %s minutes ....." % t)
configurator.wait_k8s_resources(resource='job',
label_selectors="app=fmke-client",
timeout=timeout,
kube_namespace=kube_namespace)
logger.info("Finish stressing Antidote database")
def deploy_fmke_client(self, fmke_yaml_path, test_duration, concurrent_clients, n_total_fmke_clients, workload=None, kube_namespace='default'):
"""Deploy FMKe client on the given K8s cluster
Parameters
----------
fmke_yaml_path: str
a path to the K8s yaml deployment files
test_duration: int
the duration to perform the workload
concurrent_clients: int
the number of concurrent clients
n_total_fmke_clients: int
the total number of clients need to be deployed on the system
workload: dict
the workload ratio of FMKe benchmark
kube_namespace: str
the name of K8s namespace
"""
logger.debug('Delete old k8s yaml files if exists')
for filename in os.listdir(fmke_yaml_path):
if filename.startswith('create_fmke_client_') or filename.startswith('fmke_client_'):
if '.template' not in filename:
try:
os.remove(os.path.join(fmke_yaml_path, filename))
except OSError:
logger.debug('Error while deleting file')
if workload:
logger.debug('Create the new workload ratio')
new_workload = ',\n'.join([' {%s, %s}' % (key, val) for key, val in workload.items()])
operations = '{operations,[\n%s\n]}.' % new_workload
logger.debug('Init configurator: k8s_resources_configurator')
configurator = k8s_resources_configurator()
fmke_list = configurator.get_k8s_resources(resource='pod',
label_selectors='app=fmke',
kube_namespace=kube_namespace)
fmke_client_files = list()
config_file_path = os.path.join(fmke_yaml_path, 'fmke_client.config.template')
create_file_path = os.path.join(fmke_yaml_path, 'create_fmke_client.yaml.template')
for fmke in fmke_list.items:
node = fmke.spec.node_name.split('.')[0]
# Modify fmke_client config files with new values
logger.debug('Create fmke_client config files to stress database for each AntidoteDB DC')
with open(config_file_path) as f:
doc = f.read()
doc = doc.replace('127.0.0.1', '%s' % fmke.status.pod_ip)
doc = doc.replace('{concurrent, 16}.', '{concurrent, %s}.' % concurrent_clients)
doc = doc.replace('{duration, 3}.', '{duration, %s}.' % test_duration)
doc = doc.replace("'", '"')
if workload:
doc = re.sub(r'{operations.*', operations, doc, flags=re.S)
file_path = os.path.join(fmke_yaml_path, 'fmke_client_%s.config' % node)
with open(file_path, 'w') as f:
f.write(doc)
logger.debug('Create fmke_client folder on each fmke_client node')
cmd = 'mkdir -p /tmp/fmke_client'
execute_cmd(cmd, fmke.status.host_ip)
logger.debug('Upload fmke_client config files to kube_master to be used by kubectl to run fmke_client pods')
getput_file(hosts=fmke.status.host_ip, file_paths=[file_path], dest_location='/tmp/fmke_client/', action='put')
logger.debug('Create create_fmke_client.yaml files to deploy one FMKe client')
with open(create_file_path) as f:
doc = yaml.safe_load(f)
doc['metadata']['name'] = 'fmke-client-%s' % node
doc['spec']['template']['spec']['containers'][0]['lifecycle']['postStart']['exec']['command'] = [
'cp', '/cluster_node/fmke_client_%s.config' % node, '/fmke_client/fmke_client.config']
doc['spec']['template']['spec']['nodeSelector'] = {
'service': 'fmke', 'kubernetes.io/hostname': '%s' % fmke.spec.node_name}
file_path = os.path.join(fmke_yaml_path, 'create_fmke_client_%s.yaml' % node)
with open(file_path, 'w') as f:
yaml.safe_dump(doc, f)
fmke_client_files.append(file_path)
logger.info('Starting FMKe client instances on each AntidoteDB DC')
configurator.deploy_k8s_resources(files=fmke_client_files, namespace=kube_namespace)
sleep(20)
logger.info('Checking if deploying enough the number of running FMKe_client or not')
fmke_client_list = configurator.get_k8s_resources_name(resource='pod',
label_selectors='app=fmke-client',
kube_namespace=kube_namespace)
if len(fmke_client_list) != n_total_fmke_clients:
logger.info('n_fmke_client = %s, n_deployed_fmke_client = %s' %(n_total_fmke_clients, len(fmke_client_list)))
raise CancelException('Cannot deploy enough FMKe_client')
logger.info('Stressing database in %s minutes .....' % test_duration)
deploy_ok = configurator.wait_k8s_resources(resource='job',
label_selectors='app=fmke-client',
timeout=(test_duration + 5)*60,
kube_namespace=kube_namespace)
if not deploy_ok:
logger.error('Cannot wait until all FMKe client instances running completely')
raise CancelException('Cannot wait until all FMKe client instances running completely')
logger.info('Finish stressing AntidoteDB cluster')
def deploy_elmerfs(self, kube_master, kube_namespace, elmerfs_hosts):
logger.info("Starting deploying elmerfs on hosts")
configurator = packages_configurator()
# configurator.install_packages(["libfuse2", "wget", "jq"], elmerfs_hosts)
elmerfs_repo = self.configs["exp_env"]["elmerfs_repo"]
elmerfs_version = self.configs["exp_env"]["elmerfs_version"]
elmerfs_file_path = self.configs["exp_env"]["elmerfs_path"]
if elmerfs_repo is None:
elmerfs_repo = "https://github.com/scality/elmerfs"
if elmerfs_version is None:
elmerfs_version = "latest"
logger.info("Killing elmerfs process if it is running")
for host in elmerfs_hosts:
cmd = "pidof elmerfs"
_, r = execute_cmd(cmd, host)
pids = r.processes[0].stdout.strip().split(" ")
if len(pids) >= 1 and pids[0] != '':
for pid in pids:
cmd = "kill %s" % pid.strip()
execute_cmd(cmd, host)
cmd = "umount /tmp/dc-$(hostname)"
execute_cmd(cmd, host)
cmd = "rm -rf /tmp/dc-$(hostname)"
execute_cmd(cmd, host)
logger.info("Delete elmerfs project folder on host (if existing)")
cmd = "rm -rf /tmp/elmerfs_repo"
execute_cmd(cmd, kube_master)
if elmerfs_file_path is None:
logger.info("Downloading elmerfs project from the repo")
cmd = """curl \
-H "Accept: application/vnd.github.v3+json" \
https://api.github.com/repos/scality/elmerfs/releases/%s | jq ".tag_name" \
| xargs -I tag_name git clone https://github.com/scality/elmerfs.git --branch tag_name --single-branch /tmp/elmerfs_repo """ % elmerfs_version
execute_cmd(cmd, kube_master)
cmd = "cd /tmp/elmerfs_repo \
&& git submodule update --init --recursive"
execute_cmd(cmd, kube_master)
cmd = """cat <<EOF | sudo tee /tmp/elmerfs_repo/Dockerfile
FROM rust:1.47
RUN mkdir /elmerfs
WORKDIR /elmerfs
COPY . .
RUN apt-get update \
&& apt-get -y install libfuse-dev
RUN cargo build --release
CMD ["/bin/bash"]
"""
execute_cmd(cmd, kube_master)
logger.info("Building elmerfs")
cmd = " cd /tmp/elmerfs_repo/ \
&& docker build -t elmerfs ."
execute_cmd(cmd, kube_master)
cmd = "docker run --name elmerfs elmerfs \
&& docker cp -L elmerfs:/elmerfs/target/release/main /tmp/elmerfs \
&& docker rm elmerfs"
execute_cmd(cmd, kube_master)
getput_file(
hosts=[kube_master],
file_paths=["/tmp/elmerfs"],
dest_location="/tmp",
action="get",
)
elmerfs_file_path = "/tmp/elmerfs"
logger.info(
"Uploading elmerfs binary file from local to %s elmerfs hosts" %
len(elmerfs_hosts))
getput_file(
hosts=elmerfs_hosts,
file_paths=[elmerfs_file_path],
dest_location="/tmp",
action="put",
)
cmd = "chmod +x /tmp/elmerfs \
&& mkdir -p /tmp/dc-$(hostname)"
execute_cmd(cmd, elmerfs_hosts)
logger.debug("Getting IP of antidoteDB instances on nodes")
antidote_ips = dict()
configurator = k8s_resources_configurator()
pod_list = configurator.get_k8s_resources(
resource="pod",
label_selectors="app=antidote",
kube_namespace=kube_namespace,
)
for pod in pod_list.items:
node = pod.spec.node_name
if node not in antidote_ips:
antidote_ips[node] = list()
antidote_ips[node].append(pod.status.pod_ip)
for host in elmerfs_hosts:
antidote_options = [
"--antidote=%s:8087" % ip for ip in antidote_ips[host]
]
elmerfs_cmd = "RUST_BACKTRACE=1 RUST_LOG=debug nohup /tmp/elmerfs %s --mount=/tmp/dc-$(hostname) --force-view=$ELMERFS_UID > /tmp/elmer.log 2>&1" % " ".join(
antidote_options)
logger.info("Starting elmerfs on %s with cmd: %s" %
(host, elmerfs_cmd))
execute_cmd(elmerfs_cmd, host, mode='start')
sleep(10)
for i in range(10):
cmd = "pidof elmerfs"
_, r = execute_cmd(cmd, host)
pid = r.processes[0].stdout.strip().split(" ")
if len(pid) >= 1 and pid[0].strip():
break
else:
execute_cmd(elmerfs_cmd, host, mode="start")
sleep(10)
else:
logger.info("Cannot deploy elmerfs on host %s" % host)
return False
logger.info("Finish deploying elmerfs\n")
return True
def install_elmerfs(self, kube_master, elmerfs_hosts, elmerfs_mountpoint, elmerfs_repo, elmerfs_version, elmerfs_path):
# Create folder to build the elmerfs from the repo
# cmd = 'rm -rf /tmp/elmerfs_repo && mkdir -p /tmp/elmerfs_repo'
# execute_cmd(cmd, elmerfs_hosts)
if elmerfs_repo is None:
elmerfs_repo = 'https://github.com/scality/elmerfs'
if elmerfs_version is None:
elmerfs_version = 'latest'
if elmerfs_path is None:
logger.info('Installing elmerfs')
configurator = packages_configurator()
configurator.install_packages(['libfuse2', 'wget', 'jq'], elmerfs_hosts)
logger.info('Create folder to build the elmerfs from the repo')
# Create folder to build the elmerfs from the repo
cmd = 'rm -rf /tmp/elmerfs_repo && mkdir -p /tmp/elmerfs_repo'
execute_cmd(cmd, kube_master)
logger.info('Downloading elmerfs project from the repo')
cmd = '''curl \
-H 'Accept: application/vnd.github.v3+json' \
https://api.github.com/repos/scality/elmerfs/releases/%s | jq '.tag_name' \
| xargs -I tag_name git clone https://github.com/scality/elmerfs.git --branch tag_name --single-branch /tmp/elmerfs_repo ''' % elmerfs_version
execute_cmd(cmd, kube_master)
logger.info('update the repo')
cmd = 'cd /tmp/elmerfs_repo \
&& git submodule update --init --recursive'
execute_cmd(cmd, kube_master)
logger.info('Creating the Docker file')
cmd = '''cat <<EOF | sudo tee /tmp/elmerfs_repo/Dockerfile
FROM rust:1.47
RUN mkdir /elmerfs
WORKDIR /elmerfs
COPY . .
RUN apt-get update \
&& apt-get -y install libfuse-dev
RUN cargo build --release
CMD ['/bin/bash']
'''
execute_cmd(cmd, kube_master)
logger.info('Building elmerfs Docker image')
cmd = 'cd /tmp/elmerfs_repo/ \
&& docker build -t elmerfs .'
execute_cmd(cmd, kube_master)
logger.info('Building elmerfs')
cmd = 'docker run --name elmerfs elmerfs \
&& docker cp -L elmerfs:/elmerfs/target/release/main /tmp/elmerfs \
&& docker rm elmerfs'
execute_cmd(cmd, kube_master)
getput_file(hosts=[kube_master],
file_paths=['/tmp/elmerfs'],
dest_location='/tmp',
action='get',)
elmerfs_path = '/tmp/elmerfs'
logger.info('Uploading elmerfs binary file from local to %s elmerfs hosts' % len(elmerfs_hosts))
getput_file(hosts=elmerfs_hosts,
file_paths=[elmerfs_path],
dest_location='/tmp',
action='put')
cmd = 'chmod +x /tmp/elmerfs'
execute_cmd(cmd, elmerfs_hosts)
logger.info('Create mountpoint and result folder')
cmd = 'rm -rf /tmp/results && mkdir -p /tmp/results && \
rm -rf %s && mkdir -p %s' % (elmerfs_mountpoint, elmerfs_mountpoint)
execute_cmd(cmd, elmerfs_hosts)
def deploy_elmerfs(self, kube_master, kube_namespace, elmerfs_hosts):
logger.info("Starting deploying elmerfs on hosts")
configurator = packages_configurator()
configurator.install_packages(['libfuse2', 'wget', 'jq'],
elmerfs_hosts)
elmerfs_repo = self.configs['exp_env']['elmerfs_repo']
elmerfs_version = self.configs['exp_env']['elmerfs_version']
if elmerfs_repo is None:
elmerfs_repo = 'https://github.com/scality/elmerfs'
if elmerfs_version is None:
elmerfs_version = 'latest'
logger.info('Killing elmerfs process if it is running')
for host in elmerfs_hosts:
cmd = "ps aux | grep elmerfs | awk '{print$2}'"
_, r = execute_cmd(cmd, host)
pids = r.processes[0].stdout.strip().split('\r\n')
if len(pids) >= 3:
cmd = "kill %s && umount /tmp/dc-$(hostname)" % pids[0]
execute_cmd(cmd, host)
logger.info("Downloading elmerfs project from the repo")
cmd = '''curl \
-H "Accept: application/vnd.github.v3+json" \
https://api.github.com/repos/scality/elmerfs/releases/%s | jq ".tag_name" \
| xargs -I tag_name git clone https://github.com/scality/elmerfs.git --branch tag_name --single-branch /tmp/elmerfs_repo ''' % elmerfs_version
execute_cmd(cmd, kube_master)
cmd = "cd /tmp/elmerfs_repo \
&& git submodule update --init --recursive"
execute_cmd(cmd, kube_master)
cmd = '''cat <<EOF | sudo tee /tmp/elmerfs_repo/Dockerfile
FROM rust:1.47
RUN mkdir /elmerfs
WORKDIR /elmerfs
COPY . .
RUN apt-get update \
&& apt-get -y install libfuse-dev
RUN cargo build --release
CMD ["/bin/bash"]
'''
execute_cmd(cmd, kube_master)
logger.info("Building elmerfs")
cmd = " cd /tmp/elmerfs_repo/ \
&& docker build -t elmerfs ."
execute_cmd(cmd, kube_master)
cmd = "docker run --name elmerfs elmerfs \
&& docker cp -L elmerfs:/elmerfs/target/release/main /tmp/elmerfs \
&& docker rm elmerfs"
execute_cmd(cmd, kube_master)
getput_file(hosts=[kube_master],
file_paths=['/tmp/elmerfs'],
dest_location='/tmp',
action='get')
elmerfs_file_path = '/tmp/elmerfs'
logger.info(
"Uploading elmerfs binary file from local to %s elmerfs hosts" %
len(elmerfs_hosts))
getput_file(hosts=elmerfs_hosts,
file_paths=[elmerfs_file_path],
dest_location='/tmp',
action='put')
cmd = "chmod +x /tmp/elmerfs \
&& mkdir -p /tmp/dc-$(hostname)"
execute_cmd(cmd, elmerfs_hosts)
logger.debug('Getting IP of antidoteDB instances on nodes')
antidote_ips = dict()
configurator = k8s_resources_configurator()
pod_list = configurator.get_k8s_resources(
resource='pod',
label_selectors='app=antidote',
kube_namespace=kube_namespace)
for pod in pod_list.items:
node = pod.spec.node_name
if node not in antidote_ips:
antidote_ips[node] = list()
antidote_ips[node].append(pod.status.pod_ip)
for host in elmerfs_hosts:
antidote_options = [
"--antidote=%s:8087" % ip for ip in antidote_ips[host]
]
cmd = "RUST_BACKTRACE=1 RUST_LOG=debug nohup /tmp/elmerfs %s --mount=/tmp/dc-$(hostname) --no-locks > /tmp/elmer.log 2>&1" % " ".join(
antidote_options)
logger.info("Starting elmerfs on %s with cmd: %s" % (host, cmd))
execute_cmd(cmd, host, mode='start')
sleep(5)
logger.info('Finish deploying elmerfs\n')