def _install_kubeadm(self):
logger.info('Starting installing kubeadm on %s nodes' % len(self.hosts))
logger.debug('Turning off Firewall on hosts')
cmd = '''cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1'''
execute_cmd(cmd, self.hosts)
cmd = 'sudo sysctl --system'
execute_cmd(cmd, self.hosts)
logger.debug('Turning off swap on hosts')
cmd = 'swapoff -a'
execute_cmd(cmd, self.hosts)
logger.debug('Installing kubeadm kubelet kubectl')
configurator = packages_configurator()
configurator.install_packages(['apt-transport-https', 'curl'], self.hosts)
cmd = 'curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -'
execute_cmd(cmd, self.hosts)
cmd = '''cat <<EOF | sudo tee /etc/apt/sources.list.d/kubernetes.list
deb https://apt.kubernetes.io/ kubernetes-xenial main'''
execute_cmd(cmd, self.hosts)
configurator.install_packages(['kubelet', 'kubeadm', 'kubectl'], self.hosts)
def config_host(self, kube_master, kube_namespace):
logger.info('Starting configuring nodes')
# configuring Kubernetes environment
kube_workers = [host for host in self.hosts if host != kube_master]
if self.args.kube_master is None:
kube_workers = self.deploy_k8s(kube_master)
self.setup_k8s_env(kube_master, kube_namespace, kube_workers)
elif self.args.setup_k8s_env:
logger.info('Kubernetes master: %s' % kube_master)
self.setup_k8s_env(kube_master, kube_namespace, kube_workers)
else:
self._get_credential(kube_master)
if self.args.attach_volume:
self._setup_ovh_kube_volumes(kube_workers, n_pv=3)
# Install elmerfs dependencies
configurator = packages_configurator()
configurator.install_packages(['libfuse2', 'jq'], kube_workers)
# Installing benchmark for running the experiments
if self.configs['parameters']['benchmarks'] in ['mailserver', 'videoserver']:
logger.info('Installing Filebench')
self.install_filebench(kube_workers)
logger.info('Finish configuring nodes')
return kube_master
def install_benchsoftware(self, hosts):
logger.info("Starting installing benchmark software")
logger.info("Installing tccommand")
cmd = "curl -sSL https://raw.githubusercontent.com/thombashi/tcconfig/master/scripts/installer.sh | bash"
execute_cmd(cmd, hosts)
configurator = packages_configurator()
configurator.install_packages(["bonnie++", "libfuse2", "wget", "jq"],
hosts)
logger.info("Installing crefi")
cmd = "pip install pyxattr"
execute_cmd(cmd, hosts)
cmd = "pip install crefi"
execute_cmd(cmd, hosts)
# create folder to store the results on all hosts
cmd = "mkdir -p /tmp/results"
execute_cmd(cmd, hosts)
# prepare data on all host for convergence experiment
cmd = "mkdir -p /tmp/convergence_files"
execute_cmd(cmd, hosts)
cmd = "wget https://raw.githubusercontent.com/ntlinh16/elmerfs-eval/main/resources_convergence_exp/timing_copy_file.sh -P /tmp/convergence_files/ -N"
execute_cmd(cmd, hosts)
cmd = "wget https://raw.githubusercontent.com/ntlinh16/elmerfs-eval/main/resources_convergence_exp/periodically_checksum.sh -P /tmp/convergence_files/ -N"
execute_cmd(cmd, hosts)
test_file = self.configs['exp_env']['convergence_test_file']
cmd = "wget %s -P /tmp/convergence_files/ -N" % test_file
execute_cmd(cmd, hosts)
cmd = "mv /tmp/convergence_files/%s /tmp/convergence_files/sample" % (
test_file.split('/')[-1])
execute_cmd(cmd, hosts)
# prepare data on all host for performance experiment
cmd = "mkdir -p /tmp/performance_files"
execute_cmd(cmd, hosts)
cmd = "wget https://raw.githubusercontent.com/ntlinh16/elmerfs-eval/main/resources_performance_exp/elmerfs_setup.sh -P /tmp/performance_files/ -N"
execute_cmd(cmd, hosts)
cmd = "wget https://raw.githubusercontent.com/ntlinh16/elmerfs-eval/main/resources_performance_exp/elmerfs_run.sh -P /tmp/performance_files/ -N"
execute_cmd(cmd, hosts)
cmd = "chmod +x /tmp/performance_files/*"
execute_cmd(cmd, hosts)
# create random UID and set it as ELMERFS_UID env variable on each hot for running elmerfs
uids = random.sample(range(10**4, 10**6), len(self.hosts))
for host, uid in zip(hosts, uids):
cmd = 'echo "export ELMERFS_UID=%s" >> ~/.bashrc' % uid
execute_cmd(cmd, host)
logger.info("Finish installing benchmark software on hosts\n")
def config_host(self, kube_master_site, kube_namespace):
kube_master = self.args.kube_master
if self.args.kube_master is None:
antidote_hosts = list()
for cluster in self.configs['clusters']:
cluster_name = cluster['cluster']
if cluster_name == self.configs['exp_env']['kube_master_site']:
antidote_hosts += [
host for host in self.hosts
if host.startswith(cluster_name)
][0:cluster['n_nodes'] + 1]
else:
antidote_hosts += [
host for host in self.hosts
if host.startswith(cluster_name)
][0:cluster['n_nodes']]
for host in antidote_hosts:
if host.startswith(kube_master_site):
kube_master = host
break
kube_workers = self.config_kube(kube_master, antidote_hosts,
kube_namespace)
else:
logger.info('Kubernetes master: %s' % kube_master)
self._get_credential(kube_master)
configurator = k8s_resources_configurator()
deployed_hosts = configurator.get_k8s_resources(resource='node')
kube_workers = [
host.metadata.name for host in deployed_hosts.items
]
kube_workers.remove(kube_master)
logger.info('Installing elmerfs dependencies')
configurator = packages_configurator()
configurator.install_packages(['libfuse2', 'wget', 'jq'], kube_workers)
# Create mount point on elmerfs hosts
cmd = 'mkdir -p /tmp/dc-$(hostname)'
execute_cmd(cmd, kube_workers)
# Installing filebench for running the experiments
logger.info('Installing Filebench')
self.install_filebench(kube_workers)
def install_filebench(self, hosts):
configurator = packages_configurator()
configurator.install_packages(['build-essential', 'bison', 'flex', 'libtool'], hosts)
cmd = 'wget https://github.com/filebench/filebench/archive/refs/tags/1.5-alpha3.tar.gz -P /tmp/ -N'
execute_cmd(cmd, hosts)
cmd = 'tar -xf /tmp/1.5-alpha3.tar.gz --directory /tmp/'
execute_cmd(cmd, hosts)
cmd = '''cd /tmp/filebench-1.5-alpha3/ &&
libtoolize &&
aclocal &&
autoheader &&
automake --add-missing &&
autoconf &&
./configure &&
make &&
make install'''
execute_cmd(cmd, hosts)
def config_docker(self):
"""Install Docker on the given hosts
Parameters
----------
hosts: str
a list of hosts
"""
logger.info('Starting installing Docker on %s hosts' % len(self.hosts))
configurator = packages_configurator()
configurator.install_packages(['wget'], self.hosts)
logger.info('Downloading the official get_docker script')
cmd = 'wget https://get.docker.com -O get-docker.sh'
self.error_hosts = execute_cmd(cmd, self.hosts)
logger.info('Installing Docker by using get_docker script')
cmd = 'sh get-docker.sh'
self.error_hosts = execute_cmd(cmd, self.hosts)
logger.info('Finish installing Docker on %s hosts' % len(self.hosts))
def install_glusterfs(self, hosts):
logger.info('Installing GlusterFS')
configurator = packages_configurator()
configurator.install_packages(["glusterfs-server"], hosts)
cmd = 'systemctl start glusterd'
execute_cmd(cmd, hosts)
gluster_configuration = list()
for index, host in enumerate(hosts):
cmd = "hostname -I | awk '{print $1}'"
_, r = execute_cmd(cmd, host)
host_ip = r.processes[0].stdout.strip()
gluster_configuration.append("%s gluster-%s.%s.local gluster-%s " % (host_ip, index, host, index))
gluster_configuration = "\n".join(gluster_configuration)
cmd = "echo '%s' >> /etc/hosts" % gluster_configuration
execute_cmd(cmd, hosts)
for index, _ in enumerate(hosts):
cmd = 'gluster peer probe gluster-%s' % index
execute_cmd(cmd, hosts[0])
def setup_env(self):
"""Setting the experiment environment base on the user's requirements
This function normally contains two steps:
1. Provisioning hosts on G5k if needed
(if you provided the OAR_JOB_ID of the already reserved hosts,
the provisioner will not make a reservation again)
2. Configuring all your necessary packages/services on those hosts.
"""
provisioner = g5k_provisioner(
config_file_path=self.args.config_file_path,
keep_alive=self.args.keep_alive,
out_of_chart=self.args.out_of_chart,
oar_job_ids=self.args.oar_job_ids,
no_deploy_os=self.args.no_deploy_os,
is_reservation=self.args.is_reservation,
job_name="cloudal")
provisioner.provisioning()
self.hosts = provisioner.hosts
oar_job_ids = provisioner.oar_result
self.oar_result = provisioner.oar_result
##################################################
# Configuring hosts with your applications here #
##################################################
# For example: install some dependencies
configurator = packages_configurator()
configurator.install_packages(['sysstat', 'htop'], self.hosts)
# or call the provided configurator (by cloudal) to deploy some well-known services
configurator = docker_configurator(self.hosts)
configurator.config_docker()
# or perform some commands on all of hosts
logger.info("Downloading cloudal")
cmd = "cd /tmp/ && git clone https://github.com/ntlinh16/cloudal.git"
execute_cmd(cmd, self.hosts)
return oar_job_ids
def deploy_elmerfs(self, kube_master, kube_namespace, elmerfs_hosts):
logger.info("Starting deploying elmerfs on hosts")
configurator = packages_configurator()
# configurator.install_packages(["libfuse2", "wget", "jq"], elmerfs_hosts)
elmerfs_repo = self.configs["exp_env"]["elmerfs_repo"]
elmerfs_version = self.configs["exp_env"]["elmerfs_version"]
elmerfs_file_path = self.configs["exp_env"]["elmerfs_path"]
if elmerfs_repo is None:
elmerfs_repo = "https://github.com/scality/elmerfs"
if elmerfs_version is None:
elmerfs_version = "latest"
logger.info("Killing elmerfs process if it is running")
for host in elmerfs_hosts:
cmd = "pidof elmerfs"
_, r = execute_cmd(cmd, host)
pids = r.processes[0].stdout.strip().split(" ")
if len(pids) >= 1 and pids[0] != '':
for pid in pids:
cmd = "kill %s" % pid.strip()
execute_cmd(cmd, host)
cmd = "umount /tmp/dc-$(hostname)"
execute_cmd(cmd, host)
cmd = "rm -rf /tmp/dc-$(hostname)"
execute_cmd(cmd, host)
logger.info("Delete elmerfs project folder on host (if existing)")
cmd = "rm -rf /tmp/elmerfs_repo"
execute_cmd(cmd, kube_master)
if elmerfs_file_path is None:
logger.info("Downloading elmerfs project from the repo")
cmd = """curl \
-H "Accept: application/vnd.github.v3+json" \
https://api.github.com/repos/scality/elmerfs/releases/%s | jq ".tag_name" \
| xargs -I tag_name git clone https://github.com/scality/elmerfs.git --branch tag_name --single-branch /tmp/elmerfs_repo """ % elmerfs_version
execute_cmd(cmd, kube_master)
cmd = "cd /tmp/elmerfs_repo \
&& git submodule update --init --recursive"
execute_cmd(cmd, kube_master)
cmd = """cat <<EOF | sudo tee /tmp/elmerfs_repo/Dockerfile
FROM rust:1.47
RUN mkdir /elmerfs
WORKDIR /elmerfs
COPY . .
RUN apt-get update \
&& apt-get -y install libfuse-dev
RUN cargo build --release
CMD ["/bin/bash"]
"""
execute_cmd(cmd, kube_master)
logger.info("Building elmerfs")
cmd = " cd /tmp/elmerfs_repo/ \
&& docker build -t elmerfs ."
execute_cmd(cmd, kube_master)
cmd = "docker run --name elmerfs elmerfs \
&& docker cp -L elmerfs:/elmerfs/target/release/main /tmp/elmerfs \
&& docker rm elmerfs"
execute_cmd(cmd, kube_master)
getput_file(
hosts=[kube_master],
file_paths=["/tmp/elmerfs"],
dest_location="/tmp",
action="get",
)
elmerfs_file_path = "/tmp/elmerfs"
logger.info(
"Uploading elmerfs binary file from local to %s elmerfs hosts" %
len(elmerfs_hosts))
getput_file(
hosts=elmerfs_hosts,
file_paths=[elmerfs_file_path],
dest_location="/tmp",
action="put",
)
cmd = "chmod +x /tmp/elmerfs \
&& mkdir -p /tmp/dc-$(hostname)"
execute_cmd(cmd, elmerfs_hosts)
logger.debug("Getting IP of antidoteDB instances on nodes")
antidote_ips = dict()
configurator = k8s_resources_configurator()
pod_list = configurator.get_k8s_resources(
resource="pod",
label_selectors="app=antidote",
kube_namespace=kube_namespace,
)
for pod in pod_list.items:
node = pod.spec.node_name
if node not in antidote_ips:
antidote_ips[node] = list()
antidote_ips[node].append(pod.status.pod_ip)
for host in elmerfs_hosts:
antidote_options = [
"--antidote=%s:8087" % ip for ip in antidote_ips[host]
]
elmerfs_cmd = "RUST_BACKTRACE=1 RUST_LOG=debug nohup /tmp/elmerfs %s --mount=/tmp/dc-$(hostname) --force-view=$ELMERFS_UID > /tmp/elmer.log 2>&1" % " ".join(
antidote_options)
logger.info("Starting elmerfs on %s with cmd: %s" %
(host, elmerfs_cmd))
execute_cmd(elmerfs_cmd, host, mode='start')
sleep(10)
for i in range(10):
cmd = "pidof elmerfs"
_, r = execute_cmd(cmd, host)
pid = r.processes[0].stdout.strip().split(" ")
if len(pid) >= 1 and pid[0].strip():
break
else:
execute_cmd(elmerfs_cmd, host, mode="start")
sleep(10)
else:
logger.info("Cannot deploy elmerfs on host %s" % host)
return False
logger.info("Finish deploying elmerfs\n")
return True
def install_elmerfs(self, kube_master, elmerfs_hosts, elmerfs_mountpoint, elmerfs_repo, elmerfs_version, elmerfs_path):
# Create folder to build the elmerfs from the repo
# cmd = 'rm -rf /tmp/elmerfs_repo && mkdir -p /tmp/elmerfs_repo'
# execute_cmd(cmd, elmerfs_hosts)
if elmerfs_repo is None:
elmerfs_repo = 'https://github.com/scality/elmerfs'
if elmerfs_version is None:
elmerfs_version = 'latest'
if elmerfs_path is None:
logger.info('Installing elmerfs')
configurator = packages_configurator()
configurator.install_packages(['libfuse2', 'wget', 'jq'], elmerfs_hosts)
logger.info('Create folder to build the elmerfs from the repo')
# Create folder to build the elmerfs from the repo
cmd = 'rm -rf /tmp/elmerfs_repo && mkdir -p /tmp/elmerfs_repo'
execute_cmd(cmd, kube_master)
logger.info('Downloading elmerfs project from the repo')
cmd = '''curl \
-H 'Accept: application/vnd.github.v3+json' \
https://api.github.com/repos/scality/elmerfs/releases/%s | jq '.tag_name' \
| xargs -I tag_name git clone https://github.com/scality/elmerfs.git --branch tag_name --single-branch /tmp/elmerfs_repo ''' % elmerfs_version
execute_cmd(cmd, kube_master)
logger.info('update the repo')
cmd = 'cd /tmp/elmerfs_repo \
&& git submodule update --init --recursive'
execute_cmd(cmd, kube_master)
logger.info('Creating the Docker file')
cmd = '''cat <<EOF | sudo tee /tmp/elmerfs_repo/Dockerfile
FROM rust:1.47
RUN mkdir /elmerfs
WORKDIR /elmerfs
COPY . .
RUN apt-get update \
&& apt-get -y install libfuse-dev
RUN cargo build --release
CMD ['/bin/bash']
'''
execute_cmd(cmd, kube_master)
logger.info('Building elmerfs Docker image')
cmd = 'cd /tmp/elmerfs_repo/ \
&& docker build -t elmerfs .'
execute_cmd(cmd, kube_master)
logger.info('Building elmerfs')
cmd = 'docker run --name elmerfs elmerfs \
&& docker cp -L elmerfs:/elmerfs/target/release/main /tmp/elmerfs \
&& docker rm elmerfs'
execute_cmd(cmd, kube_master)
getput_file(hosts=[kube_master],
file_paths=['/tmp/elmerfs'],
dest_location='/tmp',
action='get',)
elmerfs_path = '/tmp/elmerfs'
logger.info('Uploading elmerfs binary file from local to %s elmerfs hosts' % len(elmerfs_hosts))
getput_file(hosts=elmerfs_hosts,
file_paths=[elmerfs_path],
dest_location='/tmp',
action='put')
cmd = 'chmod +x /tmp/elmerfs'
execute_cmd(cmd, elmerfs_hosts)
logger.info('Create mountpoint and result folder')
cmd = 'rm -rf /tmp/results && mkdir -p /tmp/results && \
rm -rf %s && mkdir -p %s' % (elmerfs_mountpoint, elmerfs_mountpoint)
execute_cmd(cmd, elmerfs_hosts)
def deploy_elmerfs(self, kube_master, kube_namespace, elmerfs_hosts):
logger.info("Starting deploying elmerfs on hosts")
configurator = packages_configurator()
configurator.install_packages(['libfuse2', 'wget', 'jq'],
elmerfs_hosts)
elmerfs_repo = self.configs['exp_env']['elmerfs_repo']
elmerfs_version = self.configs['exp_env']['elmerfs_version']
if elmerfs_repo is None:
elmerfs_repo = 'https://github.com/scality/elmerfs'
if elmerfs_version is None:
elmerfs_version = 'latest'
logger.info('Killing elmerfs process if it is running')
for host in elmerfs_hosts:
cmd = "ps aux | grep elmerfs | awk '{print$2}'"
_, r = execute_cmd(cmd, host)
pids = r.processes[0].stdout.strip().split('\r\n')
if len(pids) >= 3:
cmd = "kill %s && umount /tmp/dc-$(hostname)" % pids[0]
execute_cmd(cmd, host)
logger.info("Downloading elmerfs project from the repo")
cmd = '''curl \
-H "Accept: application/vnd.github.v3+json" \
https://api.github.com/repos/scality/elmerfs/releases/%s | jq ".tag_name" \
| xargs -I tag_name git clone https://github.com/scality/elmerfs.git --branch tag_name --single-branch /tmp/elmerfs_repo ''' % elmerfs_version
execute_cmd(cmd, kube_master)
cmd = "cd /tmp/elmerfs_repo \
&& git submodule update --init --recursive"
execute_cmd(cmd, kube_master)
cmd = '''cat <<EOF | sudo tee /tmp/elmerfs_repo/Dockerfile
FROM rust:1.47
RUN mkdir /elmerfs
WORKDIR /elmerfs
COPY . .
RUN apt-get update \
&& apt-get -y install libfuse-dev
RUN cargo build --release
CMD ["/bin/bash"]
'''
execute_cmd(cmd, kube_master)
logger.info("Building elmerfs")
cmd = " cd /tmp/elmerfs_repo/ \
&& docker build -t elmerfs ."
execute_cmd(cmd, kube_master)
cmd = "docker run --name elmerfs elmerfs \
&& docker cp -L elmerfs:/elmerfs/target/release/main /tmp/elmerfs \
&& docker rm elmerfs"
execute_cmd(cmd, kube_master)
getput_file(hosts=[kube_master],
file_paths=['/tmp/elmerfs'],
dest_location='/tmp',
action='get')
elmerfs_file_path = '/tmp/elmerfs'
logger.info(
"Uploading elmerfs binary file from local to %s elmerfs hosts" %
len(elmerfs_hosts))
getput_file(hosts=elmerfs_hosts,
file_paths=[elmerfs_file_path],
dest_location='/tmp',
action='put')
cmd = "chmod +x /tmp/elmerfs \
&& mkdir -p /tmp/dc-$(hostname)"
execute_cmd(cmd, elmerfs_hosts)
logger.debug('Getting IP of antidoteDB instances on nodes')
antidote_ips = dict()
configurator = k8s_resources_configurator()
pod_list = configurator.get_k8s_resources(
resource='pod',
label_selectors='app=antidote',
kube_namespace=kube_namespace)
for pod in pod_list.items:
node = pod.spec.node_name
if node not in antidote_ips:
antidote_ips[node] = list()
antidote_ips[node].append(pod.status.pod_ip)
for host in elmerfs_hosts:
antidote_options = [
"--antidote=%s:8087" % ip for ip in antidote_ips[host]
]
cmd = "RUST_BACKTRACE=1 RUST_LOG=debug nohup /tmp/elmerfs %s --mount=/tmp/dc-$(hostname) --no-locks > /tmp/elmer.log 2>&1" % " ".join(
antidote_options)
logger.info("Starting elmerfs on %s with cmd: %s" % (host, cmd))
execute_cmd(cmd, host, mode='start')
sleep(5)
logger.info('Finish deploying elmerfs\n')
def deploy_monitoring(self,
node,
monitoring_yaml_path,
kube_namespace='default'):
"""Deploy monitoring system for AntidoteDB cluster on the given K8s cluster
Parameters
----------
node: str
the IP or hostname used to connect to the node to deploy monitoring system on
monitoring_yaml_path: str
a path to the K8s yaml deployment files
kube_namespace: str
the name of K8s namespace
"""
logger.info("Deleting old deployment")
cmd = "rm -rf /root/antidote_stats"
execute_cmd(cmd, node)
configurator = packages_configurator()
configurator.install_packages(['git'], [node])
cmd = "git clone https://github.com/AntidoteDB/antidote_stats.git"
execute_cmd(cmd, node)
logger.info("Setting to allow pods created on kube mater node")
cmd = "kubectl taint nodes --all node-role.kubernetes.io/master-"
execute_cmd(cmd, node, is_continue=True)
logger.debug("Init configurator: k8s_resources_configurator")
configurator = k8s_resources_configurator()
pods = configurator.get_k8s_resources_name(
resource='pod',
label_selectors='app=antidote',
kube_namespace=kube_namespace)
antidote_info = ["%s.antidote:3001" % pod for pod in pods]
logger.debug(
'Modify the prometheus.yml file with AntidoteDB instances info')
file_path = os.path.join(monitoring_yaml_path,
'prometheus.yml.template')
with open(file_path) as f:
doc = f.read().replace('antidotedc_info', '%s' % antidote_info)
prometheus_configmap_file = os.path.join(monitoring_yaml_path,
'prometheus.yml')
with open(prometheus_configmap_file, 'w') as f:
f.write(doc)
configurator.create_configmap(file=prometheus_configmap_file,
namespace=kube_namespace,
configmap_name='prometheus-configmap')
logger.debug('Modify the deploy_prometheus.yaml file with node info')
if not is_ip(node):
node_info = configurator.get_k8s_resources(
resource='node',
label_selectors='kubernetes.io/hostname=%s' % node)
for item in node_info.items[0].status.addresses:
if item.type == 'InternalIP':
node_ip = item.address
node_hostname = node
else:
node_ip = node
cmd = 'hostname'
_, r = execute_cmd(cmd, node)
node_hostname = r.processes[0].stdout.strip().lower()
file_path = os.path.join(monitoring_yaml_path,
'deploy_prometheus.yaml.template')
with open(file_path) as f:
doc = f.read().replace('node_ip', '%s' % node_ip)
doc = doc.replace("node_hostname", '%s' % node_hostname)
prometheus_deploy_file = os.path.join(monitoring_yaml_path,
'deploy_prometheus.yaml')
with open(prometheus_deploy_file, 'w') as f:
f.write(doc)
logger.info("Starting Prometheus service")
configurator.deploy_k8s_resources(files=[prometheus_deploy_file],
namespace=kube_namespace)
logger.info('Waiting until Prometheus instance is up')
configurator.wait_k8s_resources(resource='pod',
label_selectors="app=prometheus",
kube_namespace=kube_namespace)
logger.debug('Modify the deploy_grafana.yaml file with node info')
file_path = os.path.join(monitoring_yaml_path,
'deploy_grafana.yaml.template')
with open(file_path) as f:
doc = f.read().replace('node_ip', '%s' % node_ip)
doc = doc.replace("node_hostname", '%s' % node_hostname)
grafana_deploy_file = os.path.join(monitoring_yaml_path,
'deploy_grafana.yaml')
with open(grafana_deploy_file, 'w') as f:
f.write(doc)
file = '/root/antidote_stats/monitoring/grafana-config/provisioning/datasources/all.yml'
cmd = """ sed -i "s/localhost/%s/" %s """ % (node_ip, file)
execute_cmd(cmd, node)
logger.info("Starting Grafana service")
configurator.deploy_k8s_resources(files=[grafana_deploy_file],
namespace=kube_namespace)
logger.info('Waiting until Grafana instance is up')
configurator.wait_k8s_resources(resource='pod',
label_selectors="app=grafana",
kube_namespace=kube_namespace)
logger.info("Finish deploying monitoring system\n")
prometheus_url = "http://%s:9090" % node_ip
grafana_url = "http://%s:3000" % node_ip
logger.info("Connect to Grafana at: %s" % grafana_url)
logger.info("Connect to Prometheus at: %s" % prometheus_url)
return prometheus_url, grafana_url