def __init__(self,
registry=None,
image_name=constants.DEFAULT_IMAGE_NAME,
context_source=None,
preprocessor=None,
push=True,
base_image=constants.DEFAULT_BASE_IMAGE,
pod_spec_mutators=None,
namespace=None,
dockerfile_path=None,
cleanup=False,
executable_path_prefix=None):
super().__init__(registry=registry,
image_name=image_name,
push=push,
preprocessor=preprocessor,
base_image=base_image,
dockerfile_path=dockerfile_path)
self.manager = KubeManager()
if context_source is None:
raise RuntimeError("context_source is not specified")
self.context_source = context_source
self.pod_spec_mutators = pod_spec_mutators or []
self.namespace = namespace or utils.get_default_target_namespace()
self.cleanup = cleanup
self.executable_path_prefix = executable_path_prefix
def __init__(self, namespace=None, runs=1, output=None,
cleanup=True, labels=None, job_name=constants.JOB_DEFAULT_NAME,
stream_log=True, deployer_type=constants.JOB_DEPLOPYER_TYPE,
pod_spec_mutators=None, annotations=None):
"""
:param namespace: k8s namespace where the training's components will be deployed.
:param runs: Number of training(s) to be deployed. Hyperparameter search
will generate multiple jobs.
:param output: output
:param cleanup: clean up deletes components after job finished
:param labels: labels to be assigned to the training job
:param job_name: name of the job
:param stream_log: stream the log?
:param deployer_type: type of deployer
:param pod_spec_mutators: pod spec mutators (Default value = None)
"""
if namespace is None:
self.namespace = utils.get_default_target_namespace()
else:
self.namespace = namespace
# Used as pod and job name
self.job_name = job_name
self.deployer_type = deployer_type
self.deployment_spec = None
self.runs = runs
self.output = output
self.backend = KubeManager()
self.cleanup = cleanup
self.stream_log = stream_log
self.set_labels(labels, deployer_type)
self.set_anotations(annotations)
self.pod_spec_mutators = pod_spec_mutators or []
def __init__(self, namespace=None, build_context_source=None):
if not namespace and not utils.is_running_in_k8s():
logger.warning("Can't determine namespace automatically. "
"Using 'default' namespace but recomend to provide namespace explicitly"
". Using 'default' namespace might result in unable to mount some "
"required secrets in cloud backends.")
self._namespace = namespace or utils.get_default_target_namespace()
self._build_context_source = build_context_source
def __init__(self,
namespace=None,
runs=1,
output=None,
cleanup=True,
labels=None,
job_name=None,
stream_log=True,
deployer_type=constants.JOB_DEPLOPYER_TYPE,
pod_spec_mutators=None,
annotations=None,
config_file=None,
context=None,
client_configuration=None,
persist_config=True,
verify_ssl=True):
"""
:param namespace: k8s namespace where the training's components will be deployed.
:param runs: Number of training(s) to be deployed. Hyperparameter search
will generate multiple jobs.
:param output: output
:param cleanup: clean up deletes components after job finished
:param labels: labels to be assigned to the training job
:param job_name: name of the job
:param stream_log: stream the log?
:param deployer_type: type of deployer
:param pod_spec_mutators: pod spec mutators (Default value = None)
:param config_file: kubeconfig file, defaults to ~/.kube/config. Note that for the case
that the SDK is running in cluster and you want to operate in another remote
cluster, user must set config_file to load kube-config file explicitly.
:param context: kubernetes context
:param client_configuration: The kubernetes.client.Configuration to set configs to.
:param persist_config: If True, config file will be updated when changed
:param verify_ssl: use ssl verify or not, set in the client config
"""
if namespace is None:
self.namespace = utils.get_default_target_namespace()
else:
self.namespace = namespace
# Used as pod and job name
self.job_name = job_name
self.deployer_type = deployer_type
self.deployment_spec = None
self.runs = runs
self.output = output
self.backend = KubeManager(config_file=config_file,
context=context,
client_configuration=client_configuration,
persist_config=persist_config,
verify_ssl=verify_ssl)
self.cleanup = cleanup
self.stream_log = stream_log
self.set_labels(labels, deployer_type)
self.set_anotations(annotations)
self.pod_spec_mutators = pod_spec_mutators or []
self.verify_ssl = verify_ssl
def __init__(self,
namespace=None,
region='us-geo',
cos_endpoint_url=constants.IBM_COS_DEFAULT_ENDPOINT):
self.cos_endpoint_url = cos_endpoint_url
self.region = region
self.namespace = namespace or utils.get_default_target_namespace()
self.aws_access_key_id, self.aws_secret_access_key =\
ibm_cloud.get_ibm_cos_credentials(namespace)
def __init__(self,
framework,
default_storage_uri=None,
canary_storage_uri=None,
canary_traffic_percent=0,
namespace=None,
labels=None,
annotations=None,
custom_default_container=None,
custom_canary_container=None,
isvc_name=None,
stream_log=False,
cleanup=False):
"""
:param framework: The framework for the InferenceService, such as Tensorflow,
XGBoost and ScikitLearn etc.
:param default_storage_uri: URI pointing to Saved Model assets for default service.
:param canary_storage_uri: URI pointing to Saved Model assets for canary service.
:param canary_traffic_percent: The amount of traffic to sent to the canary, defaults to 0.
:param namespace: The k8s namespace where the InferenceService will be deployed.
:param labels: Labels for the InferenceService, separate with commas if have more than one.
:param annotations: Annotations for the InferenceService,
separate with commas if have more than one.
:param custom_default_container: A flexible custom default container for arbitrary customer
provided containers.
:param custom_canary_container: A flexible custom canary container for arbitrary customer
provided containers.
:param isvc_name: The InferenceService name.
:param stream_log: Show log or not when InferenceService started, defaults to True.
:param cleanup: Delete the kfserving or not, defaults to False.
"""
self.framework = framework
self.isvc_name = isvc_name
self.default_storage_uri = default_storage_uri
self.canary_storage_uri = canary_storage_uri
self.canary_traffic_percent = canary_traffic_percent
self.annotations = annotations
self.set_labels(labels)
self.cleanup = cleanup
self.custom_default_container = custom_default_container
self.custom_canary_container = custom_canary_container
self.stream_log = stream_log
self.backend = KubeManager()
if namespace is None:
self.namespace = utils.get_default_target_namespace()
else:
self.namespace = namespace
if self.framework != 'custom' and self.default_storage_uri is None:
raise RuntimeError("The default_storage_uri must be specified for "
"{} framework.".format(self.framework))
if self.framework == 'custom' and self.custom_default_container is None:
raise RuntimeError(
"The custom_default_container must be specified "
"for custom framework.")
def __init__(self, namespace=None, region=None,
resource_group_name=None, storage_account_name=None):
self.namespace = namespace or utils.get_default_target_namespace()
self.region = region or "NorthEurope"
self.resource_group_name = resource_group_name or "fairing"
self.storage_account_name = storage_account_name or "fairing{}".format(
uuid.uuid4().hex[:17]
)
self.share_name = constants.AZURE_FILES_SHARED_FOLDER
self.context_hash = None
self.context_path = None
def __init__(self,
namespace=None,
cos_endpoint_url=constants.IBM_COS_DEFAULT_ENDPOINT):
self.namespace = namespace or utils.get_default_target_namespace()
aws_access_key_id, aws_secret_accesss_key = get_ibm_cos_credentials(
self.namespace)
self.client = ibm_boto3.client(
"s3",
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_accesss_key,
endpoint_url=cos_endpoint_url)
def get_builder(
self,
preprocessor,
base_image,
registry,
needs_deps_installation=True, # pylint:disable=arguments-differ
pod_spec_mutators=None):
"""Creates a builder instance with right config for GCP
:param preprocessor: Preprocessor to use to modify inputs
:param base_image: Base image to use for this job
:param registry: Registry to push image to. Example: gcr.io/kubeflow-images
:param needs_deps_installation: need depends on installation(Default value = True)
:param pod_spec_mutators: list of functions that is used to mutate the podsspec.
e.g. fairing.cloud.gcp.add_gcp_credentials_if_exists
This can used to set things like volumes and security context.
(Default value =None)
"""
pod_spec_mutators = pod_spec_mutators or []
pod_spec_mutators.append(gcp.add_gcp_credentials_if_exists)
pod_spec_mutators.append(docker.add_docker_credentials_if_exists)
# TODO (karthikv2k): Add cloud build as the deafult
# once https://github.com/kubeflow/fairing/issues/145 is fixed
if not needs_deps_installation:
return AppendBuilder(preprocessor=preprocessor,
base_image=base_image,
registry=registry)
elif utils.is_running_in_k8s():
return ClusterBuilder(
preprocessor=preprocessor,
base_image=base_image,
registry=registry,
pod_spec_mutators=pod_spec_mutators,
context_source=gcs_context.GCSContextSource(
namespace=utils.get_default_target_namespace()))
elif ml_tasks_utils.is_docker_daemon_exists():
return DockerBuilder(preprocessor=preprocessor,
base_image=base_image,
registry=registry)
else:
# TODO (karthikv2k): Add more info on how to reolve this issue
raise RuntimeError(
"Not able to guess the right builder for this job!")
def __init__(self, framework, default_model_uri=None, canary_model_uri=None,
canary_traffic_percent=0, namespace=None, labels=None, annotations=None,
custom_default_spec=None, custom_canary_spec=None, stream_log=True,
cleanup=False):
"""
:param framework: The framework for the kfservice, such as Tensorflow,
XGBoost and ScikitLearn etc.
:param default_model_uri: URI pointing to Saved Model assets for default service.
:param canary_model_uri: URI pointing to Saved Model assets for canary service.
:param canary_traffic_percent: The amount of traffic to sent to the canary, defaults to 0.
:param namespace: The k8s namespace where the kfservice will be deployed.
:param labels: Labels for the kfservice, separate with commas if have more than one.
:param annotations: Annotations for the kfservice,
separate with commas if have more than one.
:param custom_default_spec: A flexible custom default specification for arbitrary customer
provided containers.
:param custom_canary_spec: A flexible custom canary specification for arbitrary customer
provided containers.
:param stream_log: Show log or not when kfservice started, defaults to True.
:param cleanup: Delete the kfserving or not, defaults to False.
"""
self.framework = framework
self.default_model_uri = default_model_uri
self.canary_model_uri = canary_model_uri
self.canary_traffic_percent = canary_traffic_percent
self.annotations = annotations
self.set_labels(labels)
self.cleanup = cleanup
self.custom_default_spec = custom_default_spec
self.custom_canary_spec = custom_canary_spec
self.stream_log = stream_log
self.backend = KubeManager()
if namespace is None:
self.namespace = utils.get_default_target_namespace()
else:
self.namespace = namespace
def apply_namespaced_object(self, spec, mode='create'): #pylint:disable=too-many-branches
"""Run apply on the provided Kubernetes specs.
:param specs: The YAML specs to apply.
:param mode: 4 valid modes: create, patch, replace and delete.
:returns: applied resources.
"""
if mode not in ['create', 'patch', 'replace', 'delete']:
raise ValueError(
"Unknown mode %s, "
"valid modes: create, patch, replace and delete." % mode)
if not isinstance(spec, dict):
spec = yaml.load(spec)
try:
namespace = spec["metadata"]["namespace"]
except KeyError:
namespace = utils.get_default_target_namespace()
kind = spec["kind"]
kind_snake = camel_to_snake(kind)
plural = spec["kind"].lower() + "s"
if mode in ['patch', 'replace', 'delete']:
try:
name = spec["metadata"]["name"]
except Exception:
raise RuntimeError(
"Cannot get the name in the spec for the operation %s." %
mode)
if not "/" in spec["apiVersion"]:
group = None
else:
group, version = spec["apiVersion"].split("/", 1)
if group is None or group.lower() == "apps":
if group is None:
api = client.CoreV1Api()
else:
api = client.AppsV1Api()
method_name = mode + "_namespaced_" + kind_snake
if mode == 'create':
method_args = [namespace, spec]
elif mode == 'delete':
method_args = [name, namespace]
else:
method_args = [name, namespace, spec]
else:
api = client.CustomObjectsApi()
method_name = mode + "_namespaced_custom_object"
if mode == 'create':
method_args = [group, version, namespace, plural, spec]
elif mode == 'delete':
method_args = [
group, version, namespace, plural, name,
client.V1DeleteOptions()
]
else:
method_args = [group, version, namespace, plural, name, spec]
apply_method = getattr(api, method_name)
try:
result = apply_method(*method_args)
except client.rest.ApiException as e:
raise RuntimeError("Exception when calling %s->%s: %s\n" % api,
apply_method, e)
return result
def __init__(self,
framework,
default_storage_uri=None,
canary_storage_uri=None,
canary_traffic_percent=0,
namespace=None,
labels=None,
annotations=None,
custom_default_container=None,
custom_canary_container=None,
isvc_name=None,
stream_log=False,
cleanup=False,
config_file=None,
context=None,
client_configuration=None,
persist_config=True):
"""
:param framework: The framework for the InferenceService, such as Tensorflow,
XGBoost and ScikitLearn etc.
:param default_storage_uri: URI pointing to Saved Model assets for default service.
:param canary_storage_uri: URI pointing to Saved Model assets for canary service.
:param canary_traffic_percent: The amount of traffic to sent to the canary, defaults to 0.
:param namespace: The k8s namespace where the InferenceService will be deployed.
:param labels: Labels for the InferenceService, separate with commas if have more than one.
:param annotations: Annotations for the InferenceService,
separate with commas if have more than one.
:param custom_default_container: A flexible custom default container for arbitrary customer
provided containers.
:param custom_canary_container: A flexible custom canary container for arbitrary customer
provided containers.
:param isvc_name: The InferenceService name.
:param stream_log: Show log or not when InferenceService started, defaults to True.
:param cleanup: Delete the kfserving or not, defaults to False.
:param config_file: kubeconfig file, defaults to ~/.kube/config. Note that for the case
that the SDK is running in cluster and you want to operate in another remote
cluster, user must set config_file to load kube-config file explicitly.
:param context: kubernetes context
:param client_configuration: The kubernetes.client.Configuration to set configs to.
:param persist_config: If True, config file will be updated when changed
"""
self.framework = framework
self.isvc_name = isvc_name
self.default_storage_uri = default_storage_uri
self.canary_storage_uri = canary_storage_uri
self.canary_traffic_percent = canary_traffic_percent
self.annotations = annotations
self.set_labels(labels)
self.cleanup = cleanup
self.custom_default_container = custom_default_container
self.custom_canary_container = custom_canary_container
self.stream_log = stream_log
self.backend = KubeManager(config_file=config_file,
context=context,
client_configuration=client_configuration,
persist_config=persist_config)
if namespace is None:
self.namespace = utils.get_default_target_namespace()
else:
self.namespace = namespace
if self.framework != 'custom' and self.default_storage_uri is None:
raise RuntimeError("The default_storage_uri must be specified for "
"{} framework.".format(self.framework))
if self.framework == 'custom' and self.custom_default_container is None:
raise RuntimeError(
"The custom_default_container must be specified "
"for custom framework.")
def execute(config,
docker_registry,
base_image="gcr.io/kubeflow-fairing/lightgbm:latest",
namespace=None,
stream_log=True,
cores_per_worker=None,
memory_per_worker=None,
pod_spec_mutators=None):
"""Runs the LightGBM CLI in a single pod in user's Kubeflow cluster.
Users can configure it to be a train, predict, and other supported tasks
by using the right config.
Please refere https://github.com/microsoft/LightGBM/blob/master/docs/Parameters.rst
for more information on config options.
:param config: config entries
:param docker_registry: docker registry name
:param base_image: base image (Default value = "gcr.io/kubeflow-fairing/lightgbm:latest")
:param namespace: k8s namespace (Default value = None)
:param stream_log: should that stream log? (Default value = True)
:param cores_per_worker: number of cores per worker (Default value = None)
:param memory_per_worker: memory value per worker (Default value = None)
:param pod_spec_mutators: pod spec mutators (Default value = None)
"""
if not namespace and not fairing_utils.is_running_in_k8s():
namespace = "kubeflow"
namespace = namespace or fairing_utils.get_default_target_namespace()
config_file_name = None
if isinstance(config, str):
config_file_name = config
config = utils.load_properties_config_file(config)
elif isinstance(config, dict):
config_file_name = utils.save_properties_config_file(config)
else:
raise RuntimeError("config should be of type dict or string(filepath) "
"but got {}".format(type(dict)))
utils.scrub_fields(config, BLACKLISTED_FIELDS)
_, num_machines = utils.get_config_value(config, NUM_MACHINES_FILEDS)
num_machines = num_machines or 1
if num_machines:
try:
num_machines = int(num_machines)
except ValueError:
raise ValueError(
"num_machines value in config should be an int >= 1 "
"but got {}".format(config.get('num_machines')))
if num_machines < 1:
raise ValueError(
"num_machines value in config should >= 1 but got {}".format(
num_machines))
if num_machines > 1:
config['machine_list_file'] = "mlist.txt"
output_map = generate_context_files(config, config_file_name, num_machines)
preprocessor = BasePreProcessor(command=[ENTRYPOINT],
output_map=output_map)
builder = AppendBuilder(registry=docker_registry,
base_image=base_image,
preprocessor=preprocessor)
builder.build()
pod_spec = builder.generate_pod_spec()
pod_spec_mutators = pod_spec_mutators or []
pod_spec_mutators.append(gcp.add_gcp_credentials_if_exists)
pod_spec_mutators.append(
k8s_utils.get_resource_mutator(cores_per_worker, memory_per_worker))
if num_machines == 1:
# non-distributed mode
deployer = Job(namespace=namespace,
pod_spec_mutators=pod_spec_mutators,
stream_log=stream_log)
else:
# distributed mode
deployer = TfJob(namespace=namespace,
pod_spec_mutators=pod_spec_mutators,
chief_count=1,
worker_count=num_machines - 1,
stream_log=stream_log)
deployer.deploy(pod_spec)
return deployer
