class Job(DeployerInterface): #pylint:disable=too-many-instance-attributes """Handle all the k8s' template building for a training""" def __init__(self, namespace=None, runs=1, output=None, cleanup=True, labels=None, job_name=None, stream_log=True, deployer_type=constants.JOB_DEPLOPYER_TYPE, pod_spec_mutators=None, annotations=None, config_file=None, context=None, client_configuration=None, persist_config=True, verify_ssl=True): """ :param namespace: k8s namespace where the training's components will be deployed. :param runs: Number of training(s) to be deployed. Hyperparameter search will generate multiple jobs. :param output: output :param cleanup: clean up deletes components after job finished :param labels: labels to be assigned to the training job :param job_name: name of the job :param stream_log: stream the log? :param deployer_type: type of deployer :param pod_spec_mutators: pod spec mutators (Default value = None) :param config_file: kubeconfig file, defaults to ~/.kube/config. Note that for the case that the SDK is running in cluster and you want to operate in another remote cluster, user must set config_file to load kube-config file explicitly. :param context: kubernetes context :param client_configuration: The kubernetes.client.Configuration to set configs to. :param persist_config: If True, config file will be updated when changed :param verify_ssl: use ssl verify or not, set in the client config """ if namespace is None: self.namespace = utils.get_default_target_namespace() else: self.namespace = namespace # Used as pod and job name self.job_name = job_name self.deployer_type = deployer_type self.deployment_spec = None self.runs = runs self.output = output self.backend = KubeManager(config_file=config_file, context=context, client_configuration=client_configuration, persist_config=persist_config, verify_ssl=verify_ssl) self.cleanup = cleanup self.stream_log = stream_log self.set_labels(labels, deployer_type) self.set_anotations(annotations) self.pod_spec_mutators = pod_spec_mutators or [] self.verify_ssl = verify_ssl def set_anotations(self, annotations): self.annotations = {} if annotations: self.annotations.update(annotations) def set_labels(self, labels, deployer_type): """set labels for the pods of a deployed job :param labels: dictionary of labels {label_name:label_value} :param deployer_type: deployer type name """ self.labels = {'fairing-deployer': deployer_type} if labels: self.labels.update(labels) def deploy(self, pod_spec): #pylint:disable=arguments-differ """deploy the training job using k8s client lib :param pod_spec: pod spec of deployed training job """ self.job_id = str(uuid.uuid1()) self.labels['fairing-id'] = self.job_id for fn in self.pod_spec_mutators: fn(self.backend, pod_spec, self.namespace) pod_template_spec = self.generate_pod_template_spec(pod_spec) pod_template_spec.spec.restart_policy = 'Never' pod_template_spec.spec.containers[0].name = 'fairing-job' self.deployment_spec = self.generate_deployment_spec(pod_template_spec) if self.output: api = k8s_client.ApiClient() job_output = api.sanitize_for_serialization(self.deployment_spec) print(json.dumps(job_output)) name = self.create_resource() logger.warning("The {} {} launched.".format(self.deployer_type, name)) if self.stream_log: self.get_logs() return name def create_resource(self): """ create job""" self._created_job = self.backend.create_job(self.namespace, self.deployment_spec) return self._created_job.metadata.name def generate_pod_template_spec(self, pod_spec): """Generate a V1PodTemplateSpec initiazlied with correct metadata and with the provided pod_spec :param pod_spec: pod spec """ if not isinstance(pod_spec, k8s_client.V1PodSpec): raise TypeError('pod_spec must be a V1PodSpec, but got %s' % type(pod_spec)) if not self.annotations: self.annotations = {'sidecar.istio.io/inject': 'false'} else: self.annotations['sidecar.istio.io/inject'] = 'false' return k8s_client.V1PodTemplateSpec(metadata=k8s_client.V1ObjectMeta( name="fairing-deployer", annotations=self.annotations, labels=self.labels), spec=pod_spec) def generate_deployment_spec(self, pod_template_spec): """Generate a V1Job initialized with correct completion and parallelism (for HP search) and with the provided V1PodTemplateSpec :param pod_template_spec: V1PodTemplateSpec """ if not isinstance(pod_template_spec, k8s_client.V1PodTemplateSpec): raise TypeError("""pod_template_spec must be a V1PodTemplateSpec, but got %s""" % type(pod_template_spec)) job_spec = k8s_client.V1JobSpec( template=pod_template_spec, parallelism=self.runs, completions=self.runs, backoff_limit=0, ) return k8s_client.V1Job(api_version="batch/v1", kind="Job", metadata=k8s_client.V1ObjectMeta( name=self.job_name, generate_name=constants.JOB_DEFAULT_NAME, labels=self.labels), spec=job_spec) def get_logs(self): """ get logs from the deployed job""" self.backend.log(self._created_job.metadata.name, self._created_job.metadata.namespace, self.labels, container="fairing-job") if self.cleanup: self.do_cleanup() def do_cleanup(self): """ clean up the pods after job finished""" logger.warning("Cleaning up job {}...".format( self._created_job.metadata.name)) client_config = k8s_client.Configuration() client_config.verify_ssl = self.verify_ssl api_client = k8s_client.ApiClient(configuration=client_config) k8s_client.BatchV1Api(api_client=api_client).delete_namespaced_job( self._created_job.metadata.name, self._created_job.metadata.namespace, body=k8s_client.V1DeleteOptions(propagation_policy='Foreground'))
class KFServing(DeployerInterface): """Serves a prediction endpoint using Kubeflow KFServing.""" def __init__(self, framework, default_storage_uri=None, canary_storage_uri=None, canary_traffic_percent=0, namespace=None, labels=None, annotations=None, custom_default_container=None, custom_canary_container=None, isvc_name=None, stream_log=False, cleanup=False, config_file=None, context=None, client_configuration=None, persist_config=True): """ :param framework: The framework for the InferenceService, such as Tensorflow, XGBoost and ScikitLearn etc. :param default_storage_uri: URI pointing to Saved Model assets for default service. :param canary_storage_uri: URI pointing to Saved Model assets for canary service. :param canary_traffic_percent: The amount of traffic to sent to the canary, defaults to 0. :param namespace: The k8s namespace where the InferenceService will be deployed. :param labels: Labels for the InferenceService, separate with commas if have more than one. :param annotations: Annotations for the InferenceService, separate with commas if have more than one. :param custom_default_container: A flexible custom default container for arbitrary customer provided containers. :param custom_canary_container: A flexible custom canary container for arbitrary customer provided containers. :param isvc_name: The InferenceService name. :param stream_log: Show log or not when InferenceService started, defaults to True. :param cleanup: Delete the kfserving or not, defaults to False. :param config_file: kubeconfig file, defaults to ~/.kube/config. Note that for the case that the SDK is running in cluster and you want to operate in another remote cluster, user must set config_file to load kube-config file explicitly. :param context: kubernetes context :param client_configuration: The kubernetes.client.Configuration to set configs to. :param persist_config: If True, config file will be updated when changed """ self.framework = framework self.isvc_name = isvc_name self.default_storage_uri = default_storage_uri self.canary_storage_uri = canary_storage_uri self.canary_traffic_percent = canary_traffic_percent self.annotations = annotations self.set_labels(labels) self.cleanup = cleanup self.custom_default_container = custom_default_container self.custom_canary_container = custom_canary_container self.stream_log = stream_log self.backend = KubeManager(config_file=config_file, context=context, client_configuration=client_configuration, persist_config=persist_config) if namespace is None: self.namespace = utils.get_default_target_namespace() else: self.namespace = namespace if self.framework != 'custom' and self.default_storage_uri is None: raise RuntimeError("The default_storage_uri must be specified for " "{} framework.".format(self.framework)) if self.framework == 'custom' and self.custom_default_container is None: raise RuntimeError( "The custom_default_container must be specified " "for custom framework.") def set_labels(self, labels): """set label for deployed prediction :param labels: dictionary of labels {label_name:label_value} """ self.fairing_id = str(uuid.uuid1()) self.labels = {'fairing-id': self.fairing_id} if labels: self.labels.update(labels) def deploy(self, isvc): # pylint:disable=arguments-differ,unused-argument """deploy kfserving endpoint :param isvc: InferenceService for deploying. """ self.created_isvc = self.backend.create_isvc(self.namespace, self.generate_isvc()) if self.stream_log: self.get_logs() isvc_name = self.created_isvc['metadata']['name'] logger.info( "Deployed the InferenceService {} successfully.".format(isvc_name)) if self.cleanup: logger.warning( "Cleaning up InferenceService {}...".format(isvc_name)) self.backend.delete_isvc(isvc_name, self.namespace) return isvc_name def generate_isvc(self): """ generate InferenceService """ api_version = constants.KFSERVING_GROUP + '/' + constants.KFSERVING_VERSION default_predictor, canary_predictor = None, None if self.framework == 'custom': default_predictor = self.generate_predictor_spec( self.framework, container=self.custom_default_container) else: default_predictor = self.generate_predictor_spec( self.framework, storage_uri=self.default_storage_uri) if self.framework != 'custom' and self.canary_storage_uri is not None: canary_predictor = self.generate_predictor_spec( self.framework, storage_uri=self.canary_storage_uri) if self.framework == 'custom' and self.custom_canary_container is not None: canary_predictor = self.generate_predictor_spec( self.framework, container=self.custom_canary_container) if canary_predictor: isvc_spec = V1alpha2InferenceServiceSpec( default=V1alpha2EndpointSpec(predictor=default_predictor), canary=V1alpha2EndpointSpec(predictor=canary_predictor), canary_traffic_percent=self.canary_traffic_percent) else: isvc_spec = V1alpha2InferenceServiceSpec( default=V1alpha2EndpointSpec(predictor=default_predictor), canary_traffic_percent=self.canary_traffic_percent) return V1alpha2InferenceService( api_version=api_version, kind=constants.KFSERVING_KIND, metadata=k8s_client.V1ObjectMeta( name=self.isvc_name, generate_name=constants.KFSERVING_DEFAULT_NAME, namespace=self.namespace), spec=isvc_spec) def generate_predictor_spec(self, framework, storage_uri=None, container=None): '''Generate predictor spec according to framework and default_storage_uri or custom container. ''' if self.framework == 'tensorflow': predictor = V1alpha2PredictorSpec( tensorflow=V1alpha2TensorflowSpec(storage_uri=storage_uri)) elif self.framework == 'onnx': predictor = V1alpha2PredictorSpec(onnx=V1alpha2ONNXSpec( storage_uri=storage_uri)) elif self.framework == 'pytorch': predictor = V1alpha2PredictorSpec(pytorch=V1alpha2PyTorchSpec( storage_uri=storage_uri)) elif self.framework == 'sklearn': predictor = V1alpha2PredictorSpec(sklearn=V1alpha2SKLearnSpec( storage_uri=storage_uri)) elif self.framework == 'triton': predictor = V1alpha2PredictorSpec(triton=V1alpha2TritonSpec( storage_uri=storage_uri)) elif self.framework == 'xgboost': predictor = V1alpha2PredictorSpec(xgboost=V1alpha2XGBoostSpec( storage_uri=storage_uri)) elif self.framework == 'custom': predictor = V1alpha2PredictorSpec(custom=V1alpha2CustomSpec( container=container)) else: raise RuntimeError("Unsupported framework {}".format(framework)) return predictor def get_logs(self): """ get log from prediction pod""" name = self.created_isvc['metadata']['name'] namespace = self.created_isvc['metadata']['namespace'] self.backend.log(name, namespace, self.labels, container=constants.KFSERVING_CONTAINER_NAME, follow=False)
class ClusterBuilder(BaseBuilder): """Builds a docker image in a Kubernetes cluster. """ def __init__(self, registry=None, image_name=constants.DEFAULT_IMAGE_NAME, context_source=None, preprocessor=None, push=True, base_image=constants.DEFAULT_BASE_IMAGE, pod_spec_mutators=None, namespace=None, dockerfile_path=None, cleanup=False, executable_path_prefix=None): super().__init__(registry=registry, image_name=image_name, push=push, preprocessor=preprocessor, base_image=base_image, dockerfile_path=dockerfile_path) self.manager = KubeManager() if context_source is None: raise RuntimeError("context_source is not specified") self.context_source = context_source self.pod_spec_mutators = pod_spec_mutators or [] self.namespace = namespace or utils.get_default_target_namespace() self.cleanup = cleanup self.executable_path_prefix = executable_path_prefix def build(self): logging.info("Building image using cluster builder.") install_reqs_before_copy = self.preprocessor.is_requirements_txt_file_present( ) if self.dockerfile_path: dockerfile_path = self.dockerfile_path else: dockerfile_path = dockerfile.write_dockerfile( path_prefix=self.preprocessor.path_prefix, base_image=self.base_image, install_reqs_before_copy=install_reqs_before_copy, executable_path_prefix=self.executable_path_prefix) self.preprocessor.output_map[dockerfile_path] = 'Dockerfile' context_path, context_hash = self.preprocessor.context_tar_gz() self.image_tag = self.full_image_name(context_hash) self.context_source.prepare(context_path) labels = {'fairing-builder': 'kaniko'} labels['fairing-build-id'] = str(uuid.uuid1()) pod_spec = self.context_source.generate_pod_spec( self.image_tag, self.push) for fn in self.pod_spec_mutators: fn(self.manager, pod_spec, self.namespace) pod_spec_template = client.V1PodTemplateSpec( metadata=client.V1ObjectMeta( generate_name="fairing-builder-", labels=labels, namespace=self.namespace, annotations={"sidecar.istio.io/inject": "false"}, ), spec=pod_spec) job_spec = client.V1JobSpec( template=pod_spec_template, parallelism=1, completions=1, backoff_limit=0, ) build_job = client.V1Job(api_version="batch/v1", kind="Job", metadata=client.V1ObjectMeta( generate_name="fairing-builder-", labels=labels, ), spec=job_spec) created_job = client. \ BatchV1Api(). \ create_namespaced_job(self.namespace, build_job) self.manager.log(name=created_job.metadata.name, namespace=created_job.metadata.namespace, selectors=labels, container="kaniko") # Invoke upstream clean ups self.context_source.cleanup() # Cleanup build_job if requested by user # Otherwise build_job will be cleaned up by Kubernetes GC if self.cleanup: logging.warning("Cleaning up job {}...".format( created_job.metadata.name)) client. \ BatchV1Api(). \ delete_namespaced_job( created_job.metadata.name, created_job.metadata.namespace, body=client.V1DeleteOptions(propagation_policy='Foreground') )
class KFServing(DeployerInterface): """Serves a prediction endpoint using Kubeflow KFServing.""" def __init__(self, framework, default_model_uri=None, canary_model_uri=None, canary_traffic_percent=0, namespace=None, labels=None, annotations=None, custom_default_spec=None, custom_canary_spec=None, stream_log=True, cleanup=False): """ :param framework: The framework for the kfservice, such as Tensorflow, XGBoost and ScikitLearn etc. :param default_model_uri: URI pointing to Saved Model assets for default service. :param canary_model_uri: URI pointing to Saved Model assets for canary service. :param canary_traffic_percent: The amount of traffic to sent to the canary, defaults to 0. :param namespace: The k8s namespace where the kfservice will be deployed. :param labels: Labels for the kfservice, separate with commas if have more than one. :param annotations: Annotations for the kfservice, separate with commas if have more than one. :param custom_default_spec: A flexible custom default specification for arbitrary customer provided containers. :param custom_canary_spec: A flexible custom canary specification for arbitrary customer provided containers. :param stream_log: Show log or not when kfservice started, defaults to True. :param cleanup: Delete the kfserving or not, defaults to False. """ self.framework = framework self.default_model_uri = default_model_uri self.canary_model_uri = canary_model_uri self.canary_traffic_percent = canary_traffic_percent self.annotations = annotations self.set_labels(labels) self.cleanup = cleanup self.custom_default_spec = custom_default_spec self.custom_canary_spec = custom_canary_spec self.stream_log = stream_log self.backend = KubeManager() if namespace is None: self.namespace = utils.get_default_target_namespace() else: self.namespace = namespace def set_labels(self, labels): """set label for deployed prediction :param labels: dictionary of labels {label_name:label_value} """ self.fairing_id = str(uuid.uuid1()) self.labels = {'fairing-id': self.fairing_id} if labels: self.labels.update(labels) def deploy(self, template_spec): # pylint:disable=arguments-differ,unused-argument """deploy kfserving endpoint :param template_spec: template spec """ self.kfservice = self.generate_kfservice() self.created_kfserving = self.backend.create_kfserving( self.namespace, self.kfservice) if self.stream_log: self.get_logs() kfservice_name = self.created_kfserving['metadata']['name'] logger.warning( "Deployed the kfservice {} successfully.".format(kfservice_name)) if self.cleanup: logger.warning("Cleaning up kfservice {}...".format(kfservice_name)) self.backend.delete_kfserving(kfservice_name, self.namespace) return kfservice_name def generate_kfservice(self): """ generate kfserving template""" spec = {} spec['default'] = {} if self.framework is not 'custom': # pylint:disable=literal-comparison if self.default_model_uri is not None: spec['default'][self.framework] = {} spec['default'][self.framework]['modelUri'] = self.default_model_uri else: raise RuntimeError( "The default_model_uri must be defined if the framework is not custom.") else: if self.custom_default_spec is not None: # TBD @jinchi Need to validate the custom_default_spec before executing. spec['default'][self.framework] = self.custom_default_spec else: raise RuntimeError( "The custom_default_spec must be defined if the framework is custom.") if self.framework != 'custom': if self.canary_model_uri is not None: spec['canary'] = {} spec['canary'][self.framework] = {} spec['canary'][self.framework]['modelUri'] = self.canary_model_uri spec['canaryTrafficPercent'] = self.canary_traffic_percent else: if self.custom_default_spec is not None: spec['canary'] = {} spec['canary'][self.framework] = self.custom_canary_spec spec['canaryTrafficPercent'] = self.canary_traffic_percent metadata = k8s_client.V1ObjectMeta( generate_name=constants.KFSERVING_DEFAULT_NAME, namespace=self.namespace, labels=self.labels, annotations=self.annotations ) kfservice = {} kfservice['kind'] = constants.KFSERVING_KIND kfservice['apiVersion'] = constants.KFSERVING_GROUP + \ '/' + constants.KFSERVING_VERSION kfservice['metadata'] = metadata kfservice['spec'] = spec return kfservice def get_logs(self): """ get log from prediction pod""" name = self.created_kfserving['metadata']['name'] namespace = self.created_kfserving['metadata']['namespace'] self.backend.log(name, namespace, self.labels, container=constants.KFSERVING_CONTAINER_NAME, follow=False)