def create_influxdb(app_id,
database_name="asperathos",
img="influxdb",
namespace="default",
visualizer_port=8086,
timeout=60):
kube.config.load_kube_config(api.k8s_conf_path)
influx_pod_spec = {
"apiVersion": "v1",
"kind": "Pod",
"metadata": {
"name": "influxdb-%s" % app_id,
"labels": {
"app": "influxdb-%s" % app_id
}
},
"spec": {
"containers": [{
"name": "influxdb-master",
"image": img,
"env": [{
"name": "MASTER",
"value": str(True)
}],
"ports": [{
"containerPort": visualizer_port
}]
}]
}
}
influx_svc_spec = {
"apiVersion": "v1",
"kind": "Service",
"metadata": {
"name": "influxdb-%s" % app_id,
"labels": {
"app": "influxdb-%s" % app_id
}
},
"spec": {
"ports": [{
"protocol": "TCP",
"port": visualizer_port,
"targetPort": visualizer_port
}],
"selector": {
"app": "influxdb-%s" % app_id
},
"type":
"NodePort"
}
}
CoreV1Api = kube.client.CoreV1Api()
node_port = None
# Gets the redis ip if the value is not explicitic in the config file
try:
redis_ip = api.redis_ip
except AttributeError:
redis_ip = api.get_node_cluster(api.k8s_conf_path)
try:
KUBEJOBS_LOG.log("Creating InfluxDB Pod...")
CoreV1Api.create_namespaced_pod(namespace=namespace,
body=influx_pod_spec)
KUBEJOBS_LOG.log("Creating InfluxDB Service...")
s = CoreV1Api.create_namespaced_service(namespace=namespace,
body=influx_svc_spec)
ready = False
attempts = timeout
while not ready:
read = CoreV1Api.read_namespaced_pod_status(name="influxdb-%s" %
app_id,
namespace=namespace)
node_port = s.spec.ports[0].node_port
if read.status.phase == "Running" and node_port is not None:
try:
# TODO change redis_ip to node_ip
client = InfluxDBClient(redis_ip, node_port, 'root',
'root', database_name)
client.create_database(database_name)
KUBEJOBS_LOG.log("InfluxDB is ready!!")
ready = True
except Exception:
KUBEJOBS_LOG.log("InfluxDB is not ready yet...")
else:
attempts -= 1
if attempts > 0:
time.sleep(1)
else:
raise Exception("InfluxDB cannot be started!"
"Time limite exceded...")
time.sleep(1)
influxdb_data = {"port": node_port, "name": database_name}
return influxdb_data
except kube.client.rest.ApiException as e:
KUBEJOBS_LOG.log(e)
def provision_redis_or_die(app_id,
namespace="default",
redis_port=6379,
timeout=60):
"""Provision a redis database for the workload being executed.
Create a redis-master Pod and expose it through a NodePort Service.
Once created this method waits ``timeout`` seconds until the
database is Ready, failing otherwise.
"""
# load kubernetes config
kube.config.load_kube_config(api.k8s_conf_path)
# name redis instance as ``redis-{app_id}``
name = "redis-%s" % app_id
# create the Pod object for redis
redis_pod_spec = {
"apiVersion": "v1",
"kind": "Pod",
"metadata": {
"name": name,
"labels": {
"app": name
}
},
"spec": {
"containers": [{
"name": "redis-master",
"image": "redis",
"env": [{
"name": "MASTER",
"value": str(True)
}],
"ports": [{
"containerPort": redis_port
}]
}]
}
}
# create the Service object for redis
redis_svc_spec = {
"apiVersion": "v1",
"kind": "Service",
"metadata": {
"name": name
},
"spec": {
"ports": [{
"port": redis_port,
"targetPort": redis_port
}],
"selector": {
"app": name
},
"type": "NodePort"
}
}
# create Pod and Service
CoreV1Api = kube.client.CoreV1Api()
node_port = None
try:
# TODO(clenimar): improve logging
KUBEJOBS_LOG.log("creating pod...")
CoreV1Api.create_namespaced_pod(namespace=namespace,
body=redis_pod_spec)
KUBEJOBS_LOG.log("creating service...")
s = CoreV1Api.create_namespaced_service(namespace=namespace,
body=redis_svc_spec)
node_port = s.spec.ports[0].node_port
except kube.client.rest.ApiException as e:
KUBEJOBS_LOG.log(e)
KUBEJOBS_LOG.log("created redis Pod and Service: %s" % name)
# Gets the redis ip if the value is not explicit in the config file
try:
redis_ip = api.redis_ip
except AttributeError:
redis_ip = api.get_node_cluster(api.k8s_conf_path)
# wait until the redis instance is Ready
# (ie. accessible via the Service)
# if it takes longer than ``timeout`` seconds, die
redis_ready = False
start = time.time()
while time.time() - start < timeout:
time.sleep(5)
KUBEJOBS_LOG.log("trying redis on %s:%s..." % (redis_ip, node_port))
try:
r = redis.StrictRedis(host=redis_ip, port=node_port)
if r.info()['loading'] == 0:
redis_ready = True
KUBEJOBS_LOG.log("connected to redis on %s:%s!" %
(redis_ip, node_port))
break
except redis.exceptions.ConnectionError:
KUBEJOBS_LOG.log("redis is not ready yet")
if redis_ready:
return redis_ip, node_port
else:
KUBEJOBS_LOG.log("timed out waiting for redis to be available.")
KUBEJOBS_LOG.log("redis address: %s:%d" % (name, node_port))
KUBEJOBS_LOG.log("clean resources and die!")
delete_redis_resources(app_id=app_id)
# die!
raise Exception("Could not provision redis")
def start_application(self, data):
try:
# Download files that contains the items
jobs = requests.get(data['redis_workload']).text.\
split('\n')[:-1]
# If the cluster name is informed in data, active the cluster
if('cluster_name' in data.keys()):
v10.activate_cluster(data['cluster_name'], data)
# Provision a redis database for the job. Die in case of error.
# TODO(clenimar): configure ``timeout`` via a request param,
# e.g. api.redis_creation_timeout.
redis_ip, redis_port = self.k8s.provision_redis_or_die(self.app_id)
# agent_port = k8s.create_cpu_agent(self.app_id)
# inject REDIS_HOST in the environment
data['env_vars']['REDIS_HOST'] = 'redis-%s' % self.app_id
# inject SCONE_CONFIG_ID in the environment
# FIXME: make SCONE_CONFIG_ID optional in submission
data['env_vars']['SCONE_CONFIG_ID'] = data['config_id']
# create a new Redis client and fill the work queue
if(self.rds is None):
self.rds = redis.StrictRedis(host=redis_ip, port=redis_port)
queue_size = len(jobs)
# Check if a visualizer will be created
self.enable_visualizer = data['enable_visualizer']
# Create all visualizer components
if self.enable_visualizer:
# Specify the datasource to be used in the visualization
datasource_type = data['visualizer_info']['datasource_type']
if datasource_type == "influxdb":
database_data = k8s.create_influxdb(self.app_id)
# Gets the redis ip if the value is not explicit in the
# config file
try:
redis_ip = api.redis_ip
except AttributeError:
redis_ip = api.get_node_cluster(api.k8s_conf_path)
database_data.update({"url": redis_ip})
data['monitor_info'].update(
{'database_data': database_data})
data['visualizer_info'].update(
{'database_data': database_data})
data['monitor_info'].update(
{'datasource_type': datasource_type})
KUBEJOBS_LOG.log("Creating Visualization platform")
data['visualizer_info'].update({
'enable_visualizer': data['enable_visualizer'],
'plugin': data['monitor_plugin'],
'visualizer_plugin': data['visualizer_plugin'],
'username': data['username'],
'password': data['password']})
visualizer.start_visualization(
api.visualizer_url, self.app_id, data['visualizer_info'])
self.visualizer_url = visualizer.get_visualizer_url(
api.visualizer_url, self.app_id)
KUBEJOBS_LOG.log(
"Dashboard of the job created on: %s" %
(self.visualizer_url))
KUBEJOBS_LOG.log("Creating Redis queue")
for job in jobs:
self.rds.rpush("job", job)
KUBEJOBS_LOG.log("Creating Job")
self.k8s.create_job(
self.app_id,
data['cmd'],
data['img'],
data['init_size'],
data['env_vars'],
config_id=data["config_id"])
self.starting_time = datetime.datetime.now()
# Starting monitor
data['monitor_info'].update(
{
'number_of_jobs': queue_size,
'submission_time': self.starting_time.
strftime('%Y-%m-%dT%H:%M:%S.%fGMT'),
'redis_ip': redis_ip,
'redis_port': redis_port,
'enable_visualizer': self.enable_visualizer}) # ,
# 'cpu_agent_port': agent_port})
monitor.start_monitor(api.monitor_url, self.app_id,
data['monitor_plugin'],
data['monitor_info'], 2)
# Starting controller
data.update({'redis_ip': redis_ip, 'redis_port': redis_port})
controller.start_controller_k8s(api.controller_url,
self.app_id, data)
while not self.job_completed and not self.terminated:
self.update_application_state("ongoing")
self.job_completed = self.k8s.completed(self.app_id)
time.sleep(1)
# Stop monitor, controller and visualizer
if(self.get_application_state() == "ongoing"):
self.update_application_state("completed")
KUBEJOBS_LOG.log("Job finished")
time.sleep(float(self.waiting_time))
if self.enable_visualizer:
visualizer.stop_visualization(
api.visualizer_url, self.app_id, data['visualizer_info'])
monitor.stop_monitor(api.monitor_url, self.app_id)
controller.stop_controller(api.controller_url, self.app_id)
self.visualizer_url = "Url is dead!"
KUBEJOBS_LOG.log("Stoped services")
# delete redis resources
if not self.get_application_state() == 'terminated':
self.k8s.terminate_job(self.app_id)
except Exception as ex:
self.update_application_state("error")
KUBEJOBS_LOG.log("ERROR: %s" % ex)
KUBEJOBS_LOG.log("Application finished.")