Esempio n. 1
0
    def _create_build_resources(self):
        logger.debug(
            "Creating Tekton build resources: PipelineResource and Task")
        git_res = yaml.safe_load(kconfig.git_res)
        git_res_name = git_res['metadata']['name']

        task_def = yaml.safe_load(kconfig.task_def)
        task_name = task_def['metadata']['name']

        git_url_param = {
            'name': 'url',
            'value': self.knative_config['git_url']
        }
        git_rev_param = {
            'name': 'revision',
            'value': self.knative_config['git_rev']
        }
        params = [git_url_param, git_rev_param]
        git_res['spec']['params'] = params

        logger.debug('Setting git url to: {}'.format(
            self.knative_config['git_url']))
        logger.debug('Setting git rev to: {}'.format(
            self.knative_config['git_rev']))

        try:
            self.api.delete_namespaced_custom_object(
                group="tekton.dev",
                version="v1alpha1",
                name=task_name,
                namespace=self.namespace,
                plural="tasks",
                body=client.V1DeleteOptions())
        except Exception:
            # ksvc resource Not Found  - Not deleted
            pass

        try:
            self.api.delete_namespaced_custom_object(
                group="tekton.dev",
                version="v1alpha1",
                name=git_res_name,
                namespace=self.namespace,
                plural="pipelineresources",
                body=client.V1DeleteOptions())
        except Exception:
            # ksvc resource Not Found - Not deleted
            pass

        self.api.create_namespaced_custom_object(group="tekton.dev",
                                                 version="v1alpha1",
                                                 namespace=self.namespace,
                                                 plural="pipelineresources",
                                                 body=git_res)

        self.api.create_namespaced_custom_object(group="tekton.dev",
                                                 version="v1alpha1",
                                                 namespace=self.namespace,
                                                 plural="tasks",
                                                 body=task_def)
Esempio n. 2
0
    def _create_build_resources(self):
        logger.debug("Creating Build resources: PipelineResource and Task")
        git_res = yaml.safe_load(kconfig.git_res)
        git_res_name = git_res['metadata']['name']

        task_def = yaml.safe_load(kconfig.task_def)
        task_name = task_def['metadata']['name']

        git_url_param = {'name': 'url', 'value': kconfig.GIT_URL_DEFAULT}
        git_rev_param = {'name': 'revision', 'value': kconfig.GIT_REV_DEFAULT}
        params = [git_url_param, git_rev_param]

        git_res['spec']['params'] = params

        try:
            self.api.delete_namespaced_custom_object(
                    group="tekton.dev",
                    version="v1alpha1",
                    name=task_name,
                    namespace="default",
                    plural="tasks",
                    body=client.V1DeleteOptions()
                )
        except Exception:
            # ksvc resource Not Found  - Not deleted
            pass

        try:
            self.api.delete_namespaced_custom_object(
                    group="tekton.dev",
                    version="v1alpha1",
                    name=git_res_name,
                    namespace="default",
                    plural="pipelineresources",
                    body=client.V1DeleteOptions()
                )
        except Exception:
            # ksvc resource Not Found - Not deleted
            pass

        self.api.create_namespaced_custom_object(
                group="tekton.dev",
                version="v1alpha1",
                namespace="default",
                plural="pipelineresources",
                body=git_res
            )

        self.api.create_namespaced_custom_object(
                group="tekton.dev",
                version="v1alpha1",
                namespace="default",
                plural="tasks",
                body=task_def
            )
Esempio n. 3
0
    def _create_build_resources(self):

        git_res = yaml.load(kconfig.git_res, Loader=yaml.FullLoader)
        task = yaml.load(kconfig.task_def, Loader=yaml.FullLoader)
        task_name = task['metadata']['name']
        git_res_name = git_res['metadata']['name']

        git_url_param = {'name': 'url', 'value': kconfig.GIT_URL_DEFAULT}
        git_rev_param = {'name': 'revision', 'value': kconfig.GIT_REV_DEFAULT}
        params = [git_url_param, git_rev_param]

        git_res['spec']['params'] = params

        try:
            self.api.delete_namespaced_custom_object(
                group="tekton.dev",
                version="v1alpha1",
                name=task_name,
                namespace="default",
                plural="tasks",
                body=client.V1DeleteOptions())
        except Exception as e:
            if json.loads(e.body)['code'] == 404:
                log_msg = 'ksvc resource: {} Not Found'.format(task_name)
                logger.debug(log_msg)

        try:
            self.api.delete_namespaced_custom_object(
                group="tekton.dev",
                version="v1alpha1",
                name=git_res_name,
                namespace="default",
                plural="pipelineresources",
                body=client.V1DeleteOptions())
        except Exception as e:
            if json.loads(e.body)['code'] == 404:
                log_msg = 'ksvc resource: {} Not Found'.format(git_res_name)
                logger.debug(log_msg)

        git_res_obj = self.api.create_namespaced_custom_object(
            group="tekton.dev",
            version="v1alpha1",
            namespace="default",
            plural="pipelineresources",
            body=git_res)

        task_obj = self.api.create_namespaced_custom_object(
            group="tekton.dev",
            version="v1alpha1",
            namespace="default",
            plural="tasks",
            body=task)
Esempio n. 4
0
    def create_service(self, docker_image_name):
        # custom resource as Dict
        svc_res = yaml.load(kconfig.service_res, Loader=yaml.FullLoader)
        svc_res['spec']['runLatest']['configuration']['revisionTemplate']['spec']['container']['image'] = \
            self.knative_config['docker_repo'] + '/' + docker_image_name
        service_name = self._format_action_name(docker_image_name)
        svc_res['metadata']['name'] = service_name

        try:
            self.api.delete_namespaced_custom_object(
                group="serving.knative.dev",
                version="v1alpha1",
                name=service_name,
                namespace="default",
                plural="services",
                body=client.V1DeleteOptions())
        except Exception as e:
            if json.loads(e.body)['code'] == 404:
                log_msg = 'ksvc resource: {} Not Found'.format(service_name)
                logger.debug(log_msg)
        # create the resource
        ksvcobj = self.api.create_namespaced_custom_object(
            group="serving.knative.dev",
            version="v1alpha1",
            namespace="default",
            plural="services",
            body=svc_res)
        lnk = ksvcobj['metadata']['selfLink']

        idx = lnk.rfind('/')

        ksvcname = lnk[idx + 1:]

        w = watch.Watch()
        for event in w.stream(
                self.api.list_namespaced_custom_object,
                namespace='default',
                group="serving.knative.dev",
                version="v1alpha1",
                plural="services",
                field_selector="metadata.name={0}".format(ksvcname),
                _request_timeout=60):
            #print("Event: %s %s" % (event['type'], event['object'].get('status')))
            #print("\n-----------------------------------")
            conditions = None
            if event['object'].get('status') is not None:
                conditions = event['object']['status']['conditions']
                if event['object']['status'].get('url') is not None:
                    url = event['object']['status']['url']
            if conditions and conditions[0]['status'] == 'True' and conditions[
                    1]['status'] == 'True' and conditions[2][
                        'status'] == 'True':
                w.stop()

        log_msg = 'Service resource created - URL: {}'.format(url)
        logger.info(log_msg)
        if not self.log_level:
            print(log_msg)
        return url
Esempio n. 5
0
    def _create_service(self, docker_image_name, runtime_memory, timeout):
        """
        Creates a service in knative based on the docker_image_name and the memory provided
        """
        logger.debug("Creating PyWren runtime service resource in k8s")
        svc_res = yaml.safe_load(kconfig.service_res)

        service_name = self._format_service_name(docker_image_name, runtime_memory)
        svc_res['metadata']['name'] = service_name

        svc_res['spec']['template']['spec']['timeoutSeconds'] = timeout

        docker_image = '/'.join([self.knative_config['docker_repo'], docker_image_name])
        svc_res['spec']['template']['spec']['container']['image'] = docker_image

        svc_res['spec']['template']['spec']['container']['resources']['limits']['memory'] = '{}Mi'.format(runtime_memory)

        try:
            # delete the service resource if exists
            self.api.delete_namespaced_custom_object(
                    group="serving.knative.dev",
                    version="v1alpha1",
                    name=service_name,
                    namespace="default",
                    plural="services",
                    body=client.V1DeleteOptions()
                )
        except Exception:
            pass

        # create the service resource
        self.api.create_namespaced_custom_object(
                group="serving.knative.dev",
                version="v1alpha1",
                namespace="default",
                plural="services",
                body=svc_res
            )

        w = watch.Watch()
        for event in w.stream(self.api.list_namespaced_custom_object,
                              namespace='default', group="serving.knative.dev",
                              version="v1alpha1", plural="services",
                              field_selector="metadata.name={0}".format(service_name)):
            conditions = None
            if event['object'].get('status') is not None:
                conditions = event['object']['status']['conditions']
                if event['object']['status'].get('url') is not None:
                    service_url = event['object']['status']['url']
            if conditions and conditions[0]['status'] == 'True' and \
               conditions[1]['status'] == 'True' and conditions[2]['status'] == 'True':
                w.stop()

        log_msg = 'Runtime Service resource created - URL: {}'.format(service_url)
        logger.debug(log_msg)

        self.service_hosts[service_name] = service_url[7:]

        return service_url
Esempio n. 6
0
 def delete_runtime(self, docker_image_name, memory):
     service_name = self._format_service_name(docker_image_name, memory)
     logger.info('Deleting runtime: {}'.format(service_name))
     try:
         self.api.delete_namespaced_custom_object(
             group="serving.knative.dev",
             version="v1",
             name=service_name,
             namespace=self.namespace,
             plural="services",
             body=client.V1DeleteOptions())
     except Exception:
         pass
Esempio n. 7
0
 def delete_runtime(self, docker_image_name, memory):
     service_name = self._format_service_name(docker_image_name, memory)
     logger.info('Deleting runtime: {}'.format(service_name))
     try:
         self.api.delete_namespaced_custom_object(
             group=kconfig.DEFAULT_GROUP,
             version=kconfig.DEFAULT_VERSION,
             name=service_name,
             namespace=self.namespace,
             plural="services",
             body=client.V1DeleteOptions())
     except Exception:
         pass
Esempio n. 8
0
 def delete_runtime(self, docker_image_name, memory):
     service_name = self._format_action_name(docker_image_name)
     try:
         self.api.delete_namespaced_custom_object(
             group="serving.knative.dev",
             version="v1alpha1",
             name=service_name,
             namespace="default",
             plural="services",
             body=client.V1DeleteOptions())
         print("Resource deleted")
     except Exception as e:
         if json.loads(e.body)['code'] == 404:
             log_msg = 'ksvc resource: {} Not Found'.format(service_name)
             logger.debug(log_msg)
Esempio n. 9
0
    def _create_service(self, docker_image_name, runtime_memory, timeout):
        """
        Creates a service in knative based on the docker_image_name and the memory provided
        """
        logger.debug("Creating Lithops runtime service in Knative")
        svc_res = yaml.safe_load(kconfig.service_res)

        service_name = self._format_service_name(docker_image_name,
                                                 runtime_memory)
        svc_res['metadata']['name'] = service_name
        svc_res['metadata']['namespace'] = self.namespace

        logger.debug("Service name: {}".format(service_name))
        logger.debug("Namespace: {}".format(self.namespace))

        svc_res['spec']['template']['spec']['timeoutSeconds'] = timeout
        full_docker_image_name = '/'.join(
            [self.knative_config['docker_repo'], docker_image_name])
        svc_res['spec']['template']['spec']['containers'][0][
            'image'] = full_docker_image_name
        svc_res['spec']['template']['spec']['containers'][0]['resources'][
            'limits']['memory'] = '{}Mi'.format(runtime_memory)
        svc_res['spec']['template']['spec']['containers'][0]['resources'][
            'limits']['cpu'] = '{}m'.format(self.knative_config['cpu'])
        svc_res['spec']['template']['spec']['containers'][0]['resources'][
            'requests']['memory'] = '{}Mi'.format(runtime_memory)
        svc_res['spec']['template']['spec']['containers'][0]['resources'][
            'requests']['cpu'] = '{}m'.format(self.knative_config['cpu'])

        try:
            # delete the service resource if exists
            self.api.delete_namespaced_custom_object(
                group="serving.knative.dev",
                version="v1",
                name=service_name,
                namespace=self.namespace,
                plural="services",
                body=client.V1DeleteOptions())
            time.sleep(2)
        except Exception:
            pass

        # create the service resource
        self.api.create_namespaced_custom_object(group="serving.knative.dev",
                                                 version="v1",
                                                 namespace=self.namespace,
                                                 plural="services",
                                                 body=svc_res)

        w = watch.Watch()
        for event in w.stream(
                self.api.list_namespaced_custom_object,
                namespace=self.namespace,
                group="serving.knative.dev",
                version="v1",
                plural="services",
                field_selector="metadata.name={0}".format(service_name),
                timeout_seconds=300):
            if event['object'].get('status'):
                service_url = event['object']['status'].get('url')
                conditions = event['object']['status']['conditions']
                if conditions[0]['status'] == 'True' and \
                   conditions[1]['status'] == 'True' and \
                   conditions[2]['status'] == 'True':
                    w.stop()
                    time.sleep(2)

        log_msg = 'Runtime Service created - URL: {}'.format(service_url)
        logger.debug(log_msg)

        self.service_host_suffix = service_url[7:].replace(service_name, '')
        # Store service host suffix in local cache
        serice_host_data = {}
        serice_host_data['service_host_suffix'] = self.service_host_suffix
        dump_yaml_config(self.serice_host_filename, serice_host_data)
        self.knative_config['service_host_suffix'] = self.service_host_suffix

        return service_url
Esempio n. 10
0
    def _build_default_runtime_from_git(self, docker_image_name):
        """
        Builds the default runtime and pushes it to the docker container registry
        """
        image_name, revision = docker_image_name.split(':')

        if self.knative_config[
                'docker_repo'] == 'docker.io' and revision != 'latest':
            resp = requests.get(
                'https://index.docker.io/v1/repositories/{}/tags/{}'.format(
                    docker_image_name, revision))
            if resp.status_code == 200:
                logger.debug(
                    'Docker image docker.io/{}:{} already exists in Dockerhub. '
                    'Skipping build process.'.format(docker_image_name,
                                                     revision))
                return

        logger.debug("Building default Lithops runtime from git with Tekton")

        if not {"docker_user", "docker_token"} <= set(self.knative_config):
            raise Exception("You must provide 'docker_user' and 'docker_token'"
                            " to build the default runtime")

        task_run = yaml.safe_load(kconfig.task_run)
        task_run['spec']['inputs']['params'] = []
        python_version = version_str(sys.version_info).replace('.', '')
        path_to_dockerfile = {
            'name':
            'pathToDockerFile',
            'value':
            'lithops/compute/backends/knative/tekton/Dockerfile.python{}'.
            format(python_version)
        }
        task_run['spec']['inputs']['params'].append(path_to_dockerfile)
        image_url = {
            'name': 'imageUrl',
            'value': '/'.join([self.knative_config['docker_repo'], image_name])
        }
        task_run['spec']['inputs']['params'].append(image_url)
        image_tag = {'name': 'imageTag', 'value': revision}
        task_run['spec']['inputs']['params'].append(image_tag)

        self._create_account_resources()
        self._create_build_resources()

        task_run_name = task_run['metadata']['name']
        try:
            self.api.delete_namespaced_custom_object(
                group="tekton.dev",
                version="v1alpha1",
                name=task_run_name,
                namespace=self.namespace,
                plural="taskruns",
                body=client.V1DeleteOptions())
        except Exception:
            pass

        self.api.create_namespaced_custom_object(group="tekton.dev",
                                                 version="v1alpha1",
                                                 namespace=self.namespace,
                                                 plural="taskruns",
                                                 body=task_run)

        logger.debug("Building runtime...")
        pod_name = None
        w = watch.Watch()
        for event in w.stream(
                self.api.list_namespaced_custom_object,
                namespace=self.namespace,
                group="tekton.dev",
                version="v1alpha1",
                plural="taskruns",
                field_selector="metadata.name={0}".format(task_run_name)):
            if event['object'].get('status'):
                pod_name = event['object']['status']['podName']
                w.stop()

        if pod_name is None:
            raise Exception(
                'Unable to get the pod name from the task that is building the runtime'
            )

        w = watch.Watch()
        for event in w.stream(
                self.v1.list_namespaced_pod,
                namespace=self.namespace,
                field_selector="metadata.name={0}".format(pod_name)):
            if event['object'].status.phase == "Succeeded":
                w.stop()
            if event['object'].status.phase == "Failed":
                w.stop()
                logger.debug(
                    'Something went wrong building the default Lithops runtime with Tekton'
                )
                for container in event['object'].status.container_statuses:
                    if container.state.terminated.reason == 'Error':
                        logs = self.v1.read_namespaced_pod_log(
                            name=pod_name,
                            container=container.name,
                            namespace=self.namespace)
                        logger.debug("Tekton container '{}' failed: {}".format(
                            container.name, logs.strip()))

                raise Exception(
                    'Unable to build the default Lithops runtime with Tekton')

        self.api.delete_namespaced_custom_object(group="tekton.dev",
                                                 version="v1alpha1",
                                                 name=task_run_name,
                                                 namespace=self.namespace,
                                                 plural="taskruns",
                                                 body=client.V1DeleteOptions())

        logger.debug(
            'Default Lithops runtime built from git and uploaded to Dockerhub')
Esempio n. 11
0
    def _build_docker_image_from_git(self, docker_image_name):
        """
        Builds the docker image and pushes it to the docker container registry
        """
        logger.debug("Building default docker image from git")

        revision = 'latest' if 'SNAPSHOT' in __version__ else __version__

        if self.knative_config[
                'docker_repo'] == 'docker.io' and revision != 'latest':
            resp = requests.get(
                'https://index.docker.io/v1/repositories/{}/tags/{}'.format(
                    docker_image_name, revision))
            if resp.status_code == 200:
                logger.debug(
                    'Docker image docker.io/{}:{} already created in Dockerhub. '
                    'Skipping build process.'.format(docker_image_name,
                                                     revision))
                return

        task_run = yaml.safe_load(kconfig.task_run)
        image_url = {
            'name':
            'imageUrl',
            'value':
            '/'.join([self.knative_config['docker_repo'], docker_image_name])
        }
        task_run['spec']['inputs']['params'].append(image_url)
        image_tag = {'name': 'imageTag', 'value': revision}
        task_run['spec']['inputs']['params'].append(image_tag)

        self._create_account_resources()
        self._create_build_resources()

        task_run_name = task_run['metadata']['name']
        try:
            self.api.delete_namespaced_custom_object(
                group="tekton.dev",
                version="v1alpha1",
                name=task_run_name,
                namespace=self.namespace,
                plural="taskruns",
                body=client.V1DeleteOptions())
        except Exception:
            pass

        self.api.create_namespaced_custom_object(group="tekton.dev",
                                                 version="v1alpha1",
                                                 namespace=self.namespace,
                                                 plural="taskruns",
                                                 body=task_run)

        logger.debug("Building image...")
        pod_name = None
        w = watch.Watch()
        for event in w.stream(
                self.api.list_namespaced_custom_object,
                namespace=self.namespace,
                group="tekton.dev",
                version="v1alpha1",
                plural="taskruns",
                field_selector="metadata.name={0}".format(task_run_name)):
            if event['object'].get('status') is not None:
                pod_name = event['object']['status']['podName']
                w.stop()

        if pod_name is None:
            raise Exception(
                'Unable to get the pod name from the task that is building the runtime'
            )

        w = watch.Watch()
        for event in w.stream(
                self.v1.list_namespaced_pod,
                namespace=self.namespace,
                field_selector="metadata.name={0}".format(pod_name)):
            if event['object'].status.phase == "Succeeded":
                w.stop()
            if event['object'].status.phase == "Failed":
                w.stop()
                raise Exception(
                    'Unable to create the Docker image from the git repository'
                )

        self.api.delete_namespaced_custom_object(group="tekton.dev",
                                                 version="v1alpha1",
                                                 name=task_run_name,
                                                 namespace=self.namespace,
                                                 plural="taskruns",
                                                 body=client.V1DeleteOptions())

        logger.debug('Docker image created from git and uploaded to Dockerhub')
Esempio n. 12
0
    def _create_service(self, docker_image_name, runtime_memory, timeout):
        """
        Creates a service in knative based on the docker_image_name and the memory provided
        """
        logger.debug("Creating PyWren runtime service resource in k8s")
        svc_res = yaml.safe_load(kconfig.service_res)

        revision = 'latest' if 'SNAPSHOT' in __version__ else __version__
        # TODO: Take into account revision in service name
        service_name = self._format_service_name(docker_image_name, runtime_memory)
        svc_res['metadata']['name'] = service_name
        svc_res['metadata']['namespace'] = self.namespace

        svc_res['spec']['template']['spec']['timeoutSeconds'] = timeout
        docker_image = '/'.join([self.knative_config['docker_repo'], docker_image_name])

        svc_res['spec']['template']['spec']['containers'][0]['image'] = '{}:{}'.format(docker_image, revision)
        svc_res['spec']['template']['spec']['containers'][0]['resources']['limits']['memory'] = '{}Mi'.format(runtime_memory)

        try:
            # delete the service resource if exists
            self.api.delete_namespaced_custom_object(
                    group="serving.knative.dev",
                    version="v1alpha1",
                    name=service_name,
                    namespace=self.namespace,
                    plural="services",
                    body=client.V1DeleteOptions()
                )
            time.sleep(2)
        except Exception:
            pass

        # create the service resource
        self.api.create_namespaced_custom_object(
                group="serving.knative.dev",
                version="v1alpha1",
                namespace=self.namespace,
                plural="services",
                body=svc_res
            )

        w = watch.Watch()
        for event in w.stream(self.api.list_namespaced_custom_object,
                              namespace=self.namespace, group="serving.knative.dev",
                              version="v1alpha1", plural="services",
                              field_selector="metadata.name={0}".format(service_name)):
            conditions = None
            if event['object'].get('status') is not None:
                conditions = event['object']['status']['conditions']
                if event['object']['status'].get('url') is not None:
                    service_url = event['object']['status']['url']
            if conditions and conditions[0]['status'] == 'True' and \
               conditions[1]['status'] == 'True' and conditions[2]['status'] == 'True':
                # Workaround to prevent invoking the service immediately after creation.
                # TODO: Open issue.
                time.sleep(2)
                w.stop()

        log_msg = 'Runtime Service resource created - URL: {}'.format(service_url)
        logger.debug(log_msg)

        self.service_host_suffix = service_url[7:].replace(service_name, '')
        # Store service host suffix in local cache
        serice_host_data = {}
        serice_host_data['service_host_suffix'] = self.service_host_suffix
        dump_yaml_config(self.serice_host_filename, serice_host_data)

        return service_url
Esempio n. 13
0
    def _build_docker_image_from_git(self, docker_image_name):
        """
        Builds the docker image and pushes it to the docker container registry
        """
        # TODO: Test if the image already exists
        self._create_account_resources()
        self._create_build_resources()

        logger.debug("Building default docker image from git")
        task_run = yaml.safe_load(kconfig.task_run)
        image_url = {'name': 'imageUrl', 'value': '/'.join([self.knative_config['docker_repo'], docker_image_name])}
        task_run['spec']['inputs']['params'].append(image_url)
        #image_tag = {'name': 'imageTag', 'value':  __version__}
        #task_run['spec']['inputs']['params'].append(image_tag)

        task_run_name = task_run['metadata']['name']
        try:
            self.api.delete_namespaced_custom_object(
                    group="tekton.dev",
                    version="v1alpha1",
                    name=task_run_name,
                    namespace="default",
                    plural="taskruns",
                    body=client.V1DeleteOptions()
                )
        except Exception:
            pass

        self.api.create_namespaced_custom_object(
                    group="tekton.dev",
                    version="v1alpha1",
                    namespace="default",
                    plural="taskruns",
                    body=task_run
                )

        pod_name = None
        w = watch.Watch()
        for event in w.stream(self.api.list_namespaced_custom_object, namespace='default',
                              group="tekton.dev", version="v1alpha1", plural="taskruns",
                              field_selector="metadata.name={0}".format(task_run_name)):
            if event['object'].get('status') is not None:
                pod_name = event['object']['status']['podName']
                w.stop()

        if pod_name is None:
            raise Exception('Unable to get the pod name from the task that is building the runtime')

        w = watch.Watch()
        for event in w.stream(self.v1.list_namespaced_pod, namespace='default',
                              field_selector="metadata.name={0}".format(pod_name)):
            if event['object'].status.phase == "Succeeded":
                w.stop()
            if event['object'].status.phase == "Failed":
                w.stop()
                raise Exception('Unable to create the Docker image from the git repository')

        self.api.delete_namespaced_custom_object(
                    group="tekton.dev",
                    version="v1alpha1",
                    name=task_run_name,
                    namespace="default",
                    plural="taskruns",
                    body=client.V1DeleteOptions()
                )

        logger.debug('Docker image created from git and uploaded to Dockerhub')
Esempio n. 14
0
    def _create_service(self, docker_image_name, runtime_memory, timeout):
        """
        Creates a service in knative based on the docker_image_name and the memory provided
        """
        logger.debug("Creating Lithops runtime service in Knative")
        svc_res = yaml.safe_load(kconfig.service_res)

        service_name = self._format_service_name(docker_image_name, runtime_memory)
        svc_res['metadata']['name'] = service_name
        svc_res['metadata']['namespace'] = self.namespace

        logger.debug("Service name: {}".format(service_name))
        logger.debug("Namespace: {}".format(self.namespace))

        svc_res['spec']['template']['spec']['timeoutSeconds'] = timeout
        svc_res['spec']['template']['spec']['containerConcurrency'] = self.knative_config['runtime_concurrency']

        container = svc_res['spec']['template']['spec']['containers'][0]
        container['image'] = docker_image_name
        container['env'][0] = {'name': 'CONCURRENCY', 'value': str(self.knative_config['runtime_concurrency'])}
        container['env'][1] = {'name': 'TIMEOUT', 'value': str(timeout)}
        container['resources']['limits']['memory'] = '{}Mi'.format(runtime_memory)
        container['resources']['limits']['cpu'] = str(self.knative_config['runtime_cpu'])
        container['resources']['requests']['memory'] = '{}Mi'.format(runtime_memory)
        container['resources']['requests']['cpu'] = str(self.knative_config['runtime_cpu'])

        svc_res['spec']['template']['metadata']['annotations']['autoscaling.knative.dev/minScale'] = str(self.knative_config['runtime_min_instances'])
        svc_res['spec']['template']['metadata']['annotations']['autoscaling.knative.dev/maxScale'] = str(self.knative_config['runtime_max_instances'])
        svc_res['spec']['template']['metadata']['annotations']['autoscaling.knative.dev/target'] = str(self.knative_config['runtime_concurrency'])

        try:
            # delete the service resource if exists
            self.custom_api.delete_namespaced_custom_object(
                    group=kconfig.DEFAULT_GROUP,
                    version=kconfig.DEFAULT_VERSION,
                    name=service_name,
                    namespace=self.namespace,
                    plural="services",
                    body=client.V1DeleteOptions()
                )
            time.sleep(2)
        except Exception:
            pass

        # create the service resource
        self.custom_api.create_namespaced_custom_object(
                group=kconfig.DEFAULT_GROUP,
                version=kconfig.DEFAULT_VERSION,
                namespace=self.namespace,
                plural="services",
                body=svc_res
            )

        w = watch.Watch()
        for event in w.stream(self.custom_api.list_namespaced_custom_object,
                              namespace=self.namespace, group=kconfig.DEFAULT_GROUP,
                              version=kconfig.DEFAULT_VERSION, plural="services",
                              field_selector="metadata.name={0}".format(service_name),
                              timeout_seconds=300):
            if event['object'].get('status'):
                service_url = event['object']['status'].get('url')
                conditions = event['object']['status']['conditions']
                if conditions[0]['status'] == 'True' and \
                   conditions[1]['status'] == 'True' and \
                   conditions[2]['status'] == 'True':
                    w.stop()
                    time.sleep(2)

        log_msg = 'Runtime Service created - URL: {}'.format(service_url)
        logger.debug(log_msg)

        self.service_host_suffix = service_url[7:].replace(service_name, '')
        # Store service host suffix in local cache
        serice_host_data = {}
        serice_host_data['service_host_suffix'] = self.service_host_suffix
        dump_yaml_config(self.serice_host_filename, serice_host_data)
        self.knative_config['service_host_suffix'] = self.service_host_suffix

        return service_url