def create_influxdb(app_id, database_name="asperathos", img="influxdb", namespace="default", visualizer_port=8086, timeout=60): kube.config.load_kube_config(api.k8s_conf_path) influx_pod_spec = { "apiVersion": "v1", "kind": "Pod", "metadata": { "name": "influxdb-%s" % app_id, "labels": { "app": "influxdb-%s" % app_id } }, "spec": { "containers": [{ "name": "influxdb-master", "image": img, "env": [{ "name": "MASTER", "value": str(True) }], "ports": [{ "containerPort": visualizer_port }] }] } } influx_svc_spec = { "apiVersion": "v1", "kind": "Service", "metadata": { "name": "influxdb-%s" % app_id, "labels": { "app": "influxdb-%s" % app_id } }, "spec": { "ports": [{ "protocol": "TCP", "port": visualizer_port, "targetPort": visualizer_port }], "selector": { "app": "influxdb-%s" % app_id }, "type": "NodePort" } } CoreV1Api = kube.client.CoreV1Api() node_port = None # Gets the redis ip if the value is not explicitic in the config file try: redis_ip = api.redis_ip except AttributeError: redis_ip = api.get_node_cluster(api.k8s_conf_path) try: KUBEJOBS_LOG.log("Creating InfluxDB Pod...") CoreV1Api.create_namespaced_pod(namespace=namespace, body=influx_pod_spec) KUBEJOBS_LOG.log("Creating InfluxDB Service...") s = CoreV1Api.create_namespaced_service(namespace=namespace, body=influx_svc_spec) ready = False attempts = timeout while not ready: read = CoreV1Api.read_namespaced_pod_status(name="influxdb-%s" % app_id, namespace=namespace) node_port = s.spec.ports[0].node_port if read.status.phase == "Running" and node_port is not None: try: # TODO change redis_ip to node_ip client = InfluxDBClient(redis_ip, node_port, 'root', 'root', database_name) client.create_database(database_name) KUBEJOBS_LOG.log("InfluxDB is ready!!") ready = True except Exception: KUBEJOBS_LOG.log("InfluxDB is not ready yet...") else: attempts -= 1 if attempts > 0: time.sleep(1) else: raise Exception("InfluxDB cannot be started!" "Time limite exceded...") time.sleep(1) influxdb_data = {"port": node_port, "name": database_name} return influxdb_data except kube.client.rest.ApiException as e: KUBEJOBS_LOG.log(e)
def provision_redis_or_die(app_id, namespace="default", redis_port=6379, timeout=60): """Provision a redis database for the workload being executed. Create a redis-master Pod and expose it through a NodePort Service. Once created this method waits ``timeout`` seconds until the database is Ready, failing otherwise. """ # load kubernetes config kube.config.load_kube_config(api.k8s_conf_path) # name redis instance as ``redis-{app_id}`` name = "redis-%s" % app_id # create the Pod object for redis redis_pod_spec = { "apiVersion": "v1", "kind": "Pod", "metadata": { "name": name, "labels": { "app": name } }, "spec": { "containers": [{ "name": "redis-master", "image": "redis", "env": [{ "name": "MASTER", "value": str(True) }], "ports": [{ "containerPort": redis_port }] }] } } # create the Service object for redis redis_svc_spec = { "apiVersion": "v1", "kind": "Service", "metadata": { "name": name }, "spec": { "ports": [{ "port": redis_port, "targetPort": redis_port }], "selector": { "app": name }, "type": "NodePort" } } # create Pod and Service CoreV1Api = kube.client.CoreV1Api() node_port = None try: # TODO(clenimar): improve logging KUBEJOBS_LOG.log("creating pod...") CoreV1Api.create_namespaced_pod(namespace=namespace, body=redis_pod_spec) KUBEJOBS_LOG.log("creating service...") s = CoreV1Api.create_namespaced_service(namespace=namespace, body=redis_svc_spec) node_port = s.spec.ports[0].node_port except kube.client.rest.ApiException as e: KUBEJOBS_LOG.log(e) KUBEJOBS_LOG.log("created redis Pod and Service: %s" % name) # Gets the redis ip if the value is not explicit in the config file try: redis_ip = api.redis_ip except AttributeError: redis_ip = api.get_node_cluster(api.k8s_conf_path) # wait until the redis instance is Ready # (ie. accessible via the Service) # if it takes longer than ``timeout`` seconds, die redis_ready = False start = time.time() while time.time() - start < timeout: time.sleep(5) KUBEJOBS_LOG.log("trying redis on %s:%s..." % (redis_ip, node_port)) try: r = redis.StrictRedis(host=redis_ip, port=node_port) if r.info()['loading'] == 0: redis_ready = True KUBEJOBS_LOG.log("connected to redis on %s:%s!" % (redis_ip, node_port)) break except redis.exceptions.ConnectionError: KUBEJOBS_LOG.log("redis is not ready yet") if redis_ready: return redis_ip, node_port else: KUBEJOBS_LOG.log("timed out waiting for redis to be available.") KUBEJOBS_LOG.log("redis address: %s:%d" % (name, node_port)) KUBEJOBS_LOG.log("clean resources and die!") delete_redis_resources(app_id=app_id) # die! raise Exception("Could not provision redis")
def start_application(self, data): try: # Download files that contains the items jobs = requests.get(data['redis_workload']).text.\ split('\n')[:-1] # If the cluster name is informed in data, active the cluster if('cluster_name' in data.keys()): v10.activate_cluster(data['cluster_name'], data) # Provision a redis database for the job. Die in case of error. # TODO(clenimar): configure ``timeout`` via a request param, # e.g. api.redis_creation_timeout. redis_ip, redis_port = self.k8s.provision_redis_or_die(self.app_id) # agent_port = k8s.create_cpu_agent(self.app_id) # inject REDIS_HOST in the environment data['env_vars']['REDIS_HOST'] = 'redis-%s' % self.app_id # inject SCONE_CONFIG_ID in the environment # FIXME: make SCONE_CONFIG_ID optional in submission data['env_vars']['SCONE_CONFIG_ID'] = data['config_id'] # create a new Redis client and fill the work queue if(self.rds is None): self.rds = redis.StrictRedis(host=redis_ip, port=redis_port) queue_size = len(jobs) # Check if a visualizer will be created self.enable_visualizer = data['enable_visualizer'] # Create all visualizer components if self.enable_visualizer: # Specify the datasource to be used in the visualization datasource_type = data['visualizer_info']['datasource_type'] if datasource_type == "influxdb": database_data = k8s.create_influxdb(self.app_id) # Gets the redis ip if the value is not explicit in the # config file try: redis_ip = api.redis_ip except AttributeError: redis_ip = api.get_node_cluster(api.k8s_conf_path) database_data.update({"url": redis_ip}) data['monitor_info'].update( {'database_data': database_data}) data['visualizer_info'].update( {'database_data': database_data}) data['monitor_info'].update( {'datasource_type': datasource_type}) KUBEJOBS_LOG.log("Creating Visualization platform") data['visualizer_info'].update({ 'enable_visualizer': data['enable_visualizer'], 'plugin': data['monitor_plugin'], 'visualizer_plugin': data['visualizer_plugin'], 'username': data['username'], 'password': data['password']}) visualizer.start_visualization( api.visualizer_url, self.app_id, data['visualizer_info']) self.visualizer_url = visualizer.get_visualizer_url( api.visualizer_url, self.app_id) KUBEJOBS_LOG.log( "Dashboard of the job created on: %s" % (self.visualizer_url)) KUBEJOBS_LOG.log("Creating Redis queue") for job in jobs: self.rds.rpush("job", job) KUBEJOBS_LOG.log("Creating Job") self.k8s.create_job( self.app_id, data['cmd'], data['img'], data['init_size'], data['env_vars'], config_id=data["config_id"]) self.starting_time = datetime.datetime.now() # Starting monitor data['monitor_info'].update( { 'number_of_jobs': queue_size, 'submission_time': self.starting_time. strftime('%Y-%m-%dT%H:%M:%S.%fGMT'), 'redis_ip': redis_ip, 'redis_port': redis_port, 'enable_visualizer': self.enable_visualizer}) # , # 'cpu_agent_port': agent_port}) monitor.start_monitor(api.monitor_url, self.app_id, data['monitor_plugin'], data['monitor_info'], 2) # Starting controller data.update({'redis_ip': redis_ip, 'redis_port': redis_port}) controller.start_controller_k8s(api.controller_url, self.app_id, data) while not self.job_completed and not self.terminated: self.update_application_state("ongoing") self.job_completed = self.k8s.completed(self.app_id) time.sleep(1) # Stop monitor, controller and visualizer if(self.get_application_state() == "ongoing"): self.update_application_state("completed") KUBEJOBS_LOG.log("Job finished") time.sleep(float(self.waiting_time)) if self.enable_visualizer: visualizer.stop_visualization( api.visualizer_url, self.app_id, data['visualizer_info']) monitor.stop_monitor(api.monitor_url, self.app_id) controller.stop_controller(api.controller_url, self.app_id) self.visualizer_url = "Url is dead!" KUBEJOBS_LOG.log("Stoped services") # delete redis resources if not self.get_application_state() == 'terminated': self.k8s.terminate_job(self.app_id) except Exception as ex: self.update_application_state("error") KUBEJOBS_LOG.log("ERROR: %s" % ex) KUBEJOBS_LOG.log("Application finished.")