def deployment_yaml_generator(self, deployment): """ generates yaml for creating deployment in kubernetes Args: deployment: Service Deployment Returns: str: path of yaml generated """ self.logger.info('entering deployment_yaml_generator') # converting any special characters to '-' project_name = project_utils.modify_string_for_deployment( deployment.project_name) component = project_utils.modify_string_for_deployment( deployment.component_name) # this will be the name used in the deployment file deployment_name = '{}--{}'.format(project_name, component) # reading contents from the standard xpresso deployment yaml file with open("config/kubernetes-deployfile.yaml", "r") as f: content = f.read() yaml_content = self.populate_yaml_content(content, deployment, deployment_name) filename = "{}/deployfile--{}.yaml".format( self.deployment_files_folder, deployment_name) with open(filename, "w+") as f: yaml.safe_dump(yaml_content, f) self.logger.info('exiting deployment_yaml_generator') return filename
def cronjob_yaml_generator(self, project_name, component, schedule, image, environment, args): """ generates yaml file to create a cronjob :param environment: environment :param project_name: project name :param component: component name :param schedule: Cron Job schedule in standard Cron format :param image: docker image :param args: array of args to run :return: path of yaml generated """ self.logger.info('entering cronjob_yaml_generator') if not project_utils.validate_cronjob_format(schedule): self.logger.error('Invalid cron schedule provided. Exiting.') raise InvalidCronScheduleException # reading contents from the standard xpresso cronjob yaml file with open("config/kubernetes-cronjobfile.yaml", "r") as f: content = f.read() # converting any special characters to '-' project_name = project_utils.modify_string_for_deployment(project_name) component = project_utils.modify_string_for_deployment(component) # this will be the name used in the job file cronjob_name = '{}--{}'.format(project_name, component) content = content.format(cronjob_name, schedule, cronjob_name, image, environment, args) yaml_content = yaml.safe_load(content) filename = "{}/cronjobfile--{}.yaml".format( self.deployment_files_folder, cronjob_name) with open(filename, "w+") as f: yaml.safe_dump(yaml_content, f) self.logger.info('exiting cronjob_yaml_generator') return filename
def get_service_ip(self, deployment: ServiceDeployment): """ method to get the list of IP addresses for services of a component Args: deployment: Service Depoyment Object Returns: array of service IPs """ self.logger.info('Entering get_service_ip method') service_name = '{}--{}'.format( project_utils.modify_string_for_deployment( deployment.project_name), project_utils.modify_string_for_deployment( deployment.component_name)) k8s_beta = client.CoreV1Api() r = k8s_beta.read_namespaced_service( name=service_name, namespace=project_utils.modify_string_for_deployment( deployment.project_name)) service_ips = [] for port in r.spec.ports: service_ips.append('{}:{}'.format(deployment.master_node, port.node_port)) self.logger.info('Exiting get_service_ip method') return service_ips
def service_yaml_generator(self, project_name, component, port): """ generates yaml for creating service in kubernetes Args: project_name: project to be deployed component: component for which this yaml is generated port: array containing info of ports to be opened Returns: path of yaml generated """ self.logger.info('entering service_yaml_generator') # reading contents from the standard xpresso service yaml file with open("config/kubernetes-servicefile.yaml", "r") as f: content = f.read() # converting any special characters to '-' project_name = project_utils.modify_string_for_deployment(project_name) component = project_utils.modify_string_for_deployment(component) ports = [] for i in port: temp = str(i) fixed_port = project_utils.modify_string_for_deployment( temp).replace("'", '"') ports.append(json.loads(fixed_port)) # this will be the name used in the service file service_name = '{}--{}'.format(project_name, component) content = content.format(service_name, ports, service_name) yaml_content = yaml.safe_load(content) filename = "{}/servicefile--{}.yaml".format( self.deployment_files_folder, service_name) with open(filename, "w+") as f: yaml.safe_dump(yaml_content, f) self.logger.info('exiting service_yaml_generator') return filename
def job_yaml_generator(self, deployment): """ generates yaml file to create a job Args: deployment: Any Deployment Returns: str: path of yaml generated """ self.logger.info('entering job_yaml_generator') # reading contents from the standard xpresso job yaml file # converting any special characters to '-' project_name = project_utils.modify_string_for_deployment( deployment.project_name) component = project_utils.modify_string_for_deployment( deployment.component_name) # this will be the name used in the job file job_name = '{}--{}'.format(project_name, component) with open("config/kubernetes-jobfile.yaml", "r") as f: content = f.read() content = content.replace("K8_XPRESSO_COMPONENT_NAME", str(job_name)) content = content.replace("K8_XPRESSO_COMPONENT_IMAGE_NAME", str(deployment.docker_image)) content = content.replace("K8_XPRESSO_COMPONENT_ENVIRONMENT_LIST", str(deployment.environment)) content = content.replace("K8_XPRESSO_COMPONENT_COMMAND", str(deployment.commands)) content = content.replace("K8_XPRESSO_COMPONENT_REPLICAS", str(deployment.replicas)) if deployment.need_persistence(): content = content.replace("K8_XPRESSO_COMPONENT_VOLUME_MOUNT_PATH", str(deployment.volume_mount_path)) # content = content.format(job_name, job_name, image, environment, # command, parallelism) yaml_content = yaml.safe_load(content) # Remove persistence if not required if not deployment.need_persistence(): try: del yaml_content["spec"]["template"]["spec"]["volumes"] del yaml_content["spec"]["template"]["spec"]["containers"][0][ "volumeMounts"] except (IndexError, KeyError): self.logger.warning("spec.template.spec.volumes not found") filename = "{}/jobfile--{}.yaml".format(self.deployment_files_folder, job_name) with open(filename, "w+") as f: yaml.safe_dump(yaml_content, f) self.logger.info('exiting job_yaml_generator') return filename
def patch_service_client(self, path, project_name): """ helper function to patch service for project as a given yaml file on Kubernetes via the Kubernetes API Args: path: path of the yaml to be deployed project_name: project to be deployed (needed for namespace) Returns: status of service patching (True/Error code) """ self.logger.info('entering patch_service_client') try: with open(path) as f: dep = yaml.safe_load(f) k8s_beta = client.CoreV1Api() # collecting response from API r = k8s_beta.patch_namespaced_service( namespace=project_utils.modify_string_for_deployment( project_name), body=dep, name=dep['metadata']['name']) self.logger.debug("Service patched. Details : {}".format( str(r))) self.logger.info('exiting patch_service_client') return True except ApiException as e: self.logger.error('Patching service failed. Error details : ' '{}'.format(e)) if e.status == 422: # Unprocessable Entity self.logger.error("Can't patch service port.") raise PortPatchingAttemptedException raise ServiceCreationFailedException
def deploy_client(self, path, project_name): """ helper function to create deployment for a given yaml file on Kubernetes via the Kubernetes API Args: path: path of the yaml to be deployed project_name: project to be deployed (needed for namespace) Returns: status of deployment (True/Error Code) """ self.logger.info('entering deploy_client') try: with open(path) as f: dep = yaml.safe_load(f) k8s_beta = client.ExtensionsV1beta1Api() # collecting response from API r = k8s_beta.create_namespaced_deployment( body=dep, namespace=project_utils.modify_string_for_deployment( project_name)) self.logger.debug("Deployment created. Details : {}".format( str(r))) self.logger.info('exiting deploy_client') return True except ApiException as e: if e.status == 409: # in case of conflict, patch the deployment self.patch_deployment_client(path, project_name) return True self.logger.error('Creation of deployment failed. Exiting.') raise DeploymentCreationFailedException
def patch_deployment_client(self, path, project_name): """ helper function to patch deployment for project as a given yaml file on Kubernetes via the Kubernetes API Args: path: path of the yaml to be deployed project_name: project to be deployed (needed for namespace) :return: status of patching (True/Error Code) """ self.logger.info('entering patch_deploy_client') try: with open(path) as f: dep = yaml.safe_load(f) k8s_beta = client.ExtensionsV1beta1Api() # collecting response from API r = k8s_beta.patch_namespaced_deployment( name=dep['metadata']['name'], body=dep, namespace=project_utils.modify_string_for_deployment( project_name)) self.logger.debug("Deployment patched. Details : {}".format( str(r))) self.logger.info('exiting patch_deploy_client') return True except ApiException as e: self.logger.error('Patching deployment failed. ' 'Error info : {}.'.format(e)) raise DeploymentCreationFailedException
def create_service_client(self, path, project_name): """ helper function to create service for a given yaml file on Kubernetes via the Kubernetes API Args: path: path of the yaml to be deployed project_name: project to be deployed (needed for namespace) Returns: status of service creation (True/Error code) """ self.logger.info('entering create_service_client') try: with open(path) as f: dep = yaml.safe_load(f) k8s_beta = client.CoreV1Api() # collecting response from API r = k8s_beta.create_namespaced_service( namespace=project_utils.modify_string_for_deployment( project_name), body=dep) self.logger.debug("Service created. Details : {}".format( str(r))) self.logger.info('exiting create_service_client') return True except ApiException as e: if e.status == 409: self.patch_service_client(path, project_name) return True self.logger.error('Creation of service failed. Exiting.') raise ServiceCreationFailedException
def patch_persistence_volume_claim(self, pv, pvc, project_name): """ Helper function to patch persistence volume claim Args: pv: persistence volume yaml file pvc: persistence volumet claim yaml fil project_name: project to be deployed (needed for namespace) :return: status of patching (True/Error Code) """ self.logger.info('entering persistence') try: with open(pvc) as f: dep = yaml.safe_load(f) k8s_beta = client.CoreV1Api() # collecting response from API r = k8s_beta.patch_namespaced_persistent_volume_claim( name=dep['metadata']['name'], body=dep, namespace=project_utils.modify_string_for_deployment( project_name)) self.logger.debug( "Persistence volume patched. Details : {}".format(str(r))) self.logger.info('exiting patch_deploy_client') return True except ApiException as e: self.logger.error('Patching PVC failed. ' 'Error info : {}.'.format(e)) raise DeploymentCreationFailedException
def create_cronjob_client(self, path, project_name): """ method to create a cronjob in kubernetes :param path: path of the yaml file :param project_name: project name of which the cronjob is a part :return: status (True/Error code) """ self.logger.info('Entering create_cronjob_client') try: with open(path) as f: dep = yaml.safe_load(f) k8s_beta = client.BatchV1beta1Api() # collecting response from API r = k8s_beta.create_namespaced_cron_job( namespace=project_utils.modify_string_for_deployment( project_name), body=dep) self.logger.debug("Cron Job created. Details : {}".format( str(r))) self.logger.info('exiting create_cronjob_client') return True except ApiException as e: if e.status == 409: # in case of conflict, patch the cronjob self.patch_cronjob_client(path, project_name) return True self.logger.error('Creation of cron job failed. Exiting.') raise CronjobCreationFailedException
def persistent_volume_yaml_generator(self, deployment, persstence_type): """ generates yaml for creating persistent volumne Args: deployment: Any Deployment Returns: str: path of yaml generated """ self.logger.info('entering persistent_yaml_generator') # converting any special characters to '-' project_name = project_utils.modify_string_for_deployment( deployment.project_name) component = project_utils.modify_string_for_deployment( deployment.component_name) # this will be the name used in the deployment file deployment_name = '{}--{}'.format(project_name, component) # reading contents from the standard xpresso deployment yaml file with open(f"config/kubernetes-persistent-{persstence_type}.yaml", "r") as f: content = f.read() content = content.replace("K8_XPRESSO_COMPONENT_NAME", str(deployment_name)) content = content.replace("K8_XPRESSO_PERSISTENT_STORAGE_SIZE", str(deployment.volume_size)) content = content.replace("K8_XPRESSO_PROJECT_NAME", str(project_name)) yaml_content = yaml.safe_load(content) filename = ( f"{self.deployment_files_folder}" f"/persistent-{persstence_type}-file--{deployment_name}.yaml") with open(filename, "w+") as f: yaml.safe_dump(yaml_content, f) self.logger.info('exiting persistent_yaml_generator') return filename
def create_persistence_if_required(self, deployment): """ Check if persistence is required, If yes then create one""" self.logger.debug("Checking for persistence") if not deployment.need_persistence(): self.logger.debug("Persistence not needed.") return False self.logger.info("Persistence is needed") pv = self.persistent_volume_yaml_generator(deployment, persstence_type="volume") pvc = self.persistent_volume_yaml_generator( deployment, persstence_type="volume-claim") try: with open(pv) as f: dep = yaml.safe_load(f) k8s_beta = client.CoreV1Api() # collecting response from API r = k8s_beta.create_persistent_volume(body=dep) self.logger.debug( f"Persistence Volume created. Details : {str(r)}") except ApiException as e: if e.status == 409: # in case of conflict, patch the deployment self.patch_persistence_volume(pv) return True self.logger.error('Creation of PV failed. Exiting.') raise DeploymentCreationFailedException try: with open(pvc) as f: dep = yaml.safe_load(f) k8s_beta = client.CoreV1Api() # collecting response from API r = k8s_beta.create_namespaced_persistent_volume_claim( body=dep, namespace=project_utils.modify_string_for_deployment( deployment.project_name)) self.logger.debug( f"Persistence Volume Claim created. Details : {str(r)}") self.logger.info('exiting deploy_client') return True except ApiException as e: if e.status == 409: # in case of conflict, patch the deployment self.patch_persistence_volume_claim(pv, pvc, deployment.project_name) return True self.logger.error('Creation of PVC failed. Exiting.') raise DeploymentCreationFailedException
def kube_undeploy(self, project_name): """ Undeploys a project :param project_name: project to be undeployed :return: nothing """ try: self.logger.debug('Deleting namespace to undeploy project') k8s_beta = client.CoreV1Api() resp = k8s_beta.delete_namespace( project_utils.modify_string_for_deployment(project_name)) self.logger.debug("Namespace deleted. Details : {}".format( str(resp))) except ApiException as e: if e.status == 404: # Not found self.logger.error('Project is not deployed currently.') raise CurrentlyNotDeployedException
def patch_job_client(self, path, project_name): self.logger.info('entering patch_job_client') try: with open(path) as f: dep = yaml.safe_load(f) k8s_beta = client.BatchV1Api() # collecting response from API r = k8s_beta.patch_namespaced_job( name=dep['metadata']['name'], body=dep, namespace=project_utils.modify_string_for_deployment( project_name)) self.logger.debug("Job patched. Details : {}".format(str(r))) self.logger.info('exiting patch_job_client') return True except ApiException as e: self.logger.error('Patching job failed. ' 'Error info : {}.'.format(e)) raise JobCreationFailedException
def check_for_namespace(self, project): """ Check if a namespace exists for the given project. If not, creates it. :param project: project to be deployed :return: nothing """ # check if namespace exists for the project self.logger.debug('checking for existing namespace') namespaces = client.CoreV1Api().list_namespace() flag = False project_name = project['name'] for ns in namespaces.items: if ns.metadata.name == \ project_utils.modify_string_for_deployment(project_name): flag = True self.logger.debug('Namespace for project already exists.') if not flag: # if project_name not in namespaces self.logger.debug('creating namespace for the project') ns_path = self.kubernetes_deploy.namespace_yaml_generator( project_name) # create namespace for project self.kubernetes_deploy.create_namespace_client(ns_path)
def namespace_yaml_generator(self, project_name): """ generates yaml file to create a new namespace Args: project_name: name of the project to be deployed Returns: path of the yaml generated """ self.logger.info('entering namespace_yaml_generator') with open("config/kubernetes-namespacefile.yaml", "r") as f: content = f.read() # converting any special characters to '-' project_name = project_utils.modify_string_for_deployment(project_name) content = content.format(project_name) yaml_content = yaml.safe_load(content) filename = "{}/namespacefile--{}.yaml".format( self.deployment_files_folder, project_name) with open(filename, "w+") as f: yaml.safe_dump(yaml_content, f) self.logger.info('exiting namespace_yaml_generator') return filename