def _create_instance_manager(self, args): instance_manager = None container_command = ["/bin/bash"] if args.num_workers: assert args.worker_image, "Worker image cannot be empty" worker_client_command = ( BashCommandTemplate.SET_PIPEFAIL + " python -m elasticdl.python.worker.main" ) worker_args = [ "--master_addr", self.master_addr, "--job_type", self.job_type, ] worker_args.extend( build_arguments_from_parsed_result(args, filter_args=["envs"]) ) worker_args = wrap_python_args_with_string(worker_args) worker_args.insert(0, worker_client_command) if args.use_go_ps: opt_type, opt_args = get_optimizer_info(self.optimizer) ps_command = "elasticdl_ps" ps_command_args = [ "-job_name=" + args.job_name, "-namespace=" + args.namespace, "-master_addr=" + self.master_addr, "-port=2222", "-use_async=" + ("true" if args.use_async else "false"), "-grads_to_wait=" + str(args.grads_to_wait), "-lr_staleness_modulation=" + ("true" if args.lr_staleness_modulation else "false"), "-sync_version_tolerance=" + str(args.sync_version_tolerance), "-evaluation_steps=" + str(args.evaluation_steps), "-num_ps_pods=" + str(args.num_ps_pods), "-num_workers=" + str(args.num_workers), "-checkpoint_dir=" + str(args.checkpoint_dir), "-checkpoint_steps=" + str(args.checkpoint_steps), "-keep_checkpoint_max=" + str(args.keep_checkpoint_max), "-checkpoint_dir_for_init=" + str(args.checkpoint_dir_for_init), "-opt_type=" + opt_type, "-opt_args=" + opt_args, ] ps_command_args = wrap_go_args_with_string(ps_command_args) # Execute source /root/.bashrc to add the file path # of `elasticdl_ps` into the PATH environment variable. ps_args = [ "source", "/root/.bashrc_elasticdl", "&&", ps_command, ] ps_args.extend(ps_command_args) else: ps_command = ( BashCommandTemplate.SET_PIPEFAIL + " python -m elasticdl.python.ps.main" ) ps_command_args = [ "--grads_to_wait", str(args.grads_to_wait), "--lr_staleness_modulation", str(args.lr_staleness_modulation), "--sync_version_tolerance", str(args.sync_version_tolerance), "--use_async", str(args.use_async), "--model_zoo", args.model_zoo, "--model_def", args.model_def, "--job_name", args.job_name, "--port", "2222", "--master_addr", self.master_addr, "--namespace", args.namespace, "--evaluation_steps", str(args.evaluation_steps), "--checkpoint_dir", str(args.checkpoint_dir), "--checkpoint_steps", str(args.checkpoint_steps), "--keep_checkpoint_max", str(args.keep_checkpoint_max), "--num_ps_pods", str(args.num_ps_pods), "--checkpoint_dir_for_init", str(args.checkpoint_dir_for_init), "--num_workers", str(args.num_workers), "--log_level", str(args.log_level), "--minibatch_size", str(args.minibatch_size), "--num_minibatches_per_task", str(args.num_minibatches_per_task), ] ps_args = wrap_python_args_with_string(ps_command_args) ps_args.insert(0, ps_command) worker_args = ["-c", " ".join(worker_args)] ps_args = ["-c", " ".join(ps_args)] env_dict = parse_envs(args.envs) env = [] for key in env_dict: env.append(V1EnvVar(name=key, value=env_dict[key])) kwargs = get_dict_from_params_str(args.aux_params) disable_relaunch = kwargs.get("disable_relaunch", False) cluster_spec = self._get_image_cluster_spec(args.cluster_spec) instance_manager = InstanceManager( self.task_d, job_name=args.job_name, image_name=args.worker_image, worker_command=container_command, worker_args=worker_args, namespace=args.namespace, num_workers=args.num_workers, worker_resource_request=args.worker_resource_request, worker_resource_limit=args.worker_resource_limit, worker_pod_priority=args.worker_pod_priority, num_ps=args.num_ps_pods, ps_command=container_command, ps_args=ps_args, ps_resource_request=args.ps_resource_request, ps_resource_limit=args.ps_resource_limit, ps_pod_priority=args.ps_pod_priority, volume=args.volume, image_pull_policy=args.image_pull_policy, restart_policy=args.restart_policy, cluster_spec=cluster_spec, envs=env, expose_ports=self.distribution_strategy == DistributionStrategy.ALLREDUCE, disable_relaunch=disable_relaunch, log_file_path=args.log_file_path, ) return instance_manager
def test_transformer(): service_name = 'isvc-transformer' predictor = V1beta1PredictorSpec( min_replicas=1, pytorch=V1beta1TorchServeSpec( storage_uri= "gs://kfserving-examples/models/torchserve/image_classifier/v1", protocol_version="v1", resources=V1ResourceRequirements( requests={ "cpu": "100m", "memory": "1Gi" }, limits={ "cpu": "1", "memory": "1Gi" }, ), ), ) transformer = V1beta1TransformerSpec( min_replicas=1, containers=[ V1Container( image= '809251082950.dkr.ecr.us-west-2.amazonaws.com/kserve/image-transformer:' + os.environ.get("PULL_BASE_SHA"), name='kserve-container', resources=V1ResourceRequirements(requests={ 'cpu': '100m', 'memory': '1Gi' }, limits={ 'cpu': '100m', 'memory': '1Gi' }), args=["--model_name", "mnist"], env=[ V1EnvVar( name="STORAGE_URI", value= "gs://kfserving-examples/models/torchserve/image_classifier/v1" ) ]) ]) isvc = V1beta1InferenceService( api_version=constants.KSERVE_V1BETA1, kind=constants.KSERVE_KIND, metadata=client.V1ObjectMeta(name=service_name, namespace=KSERVE_TEST_NAMESPACE), spec=V1beta1InferenceServiceSpec(predictor=predictor, transformer=transformer)) kserve_client = KServeClient( config_file=os.environ.get("KUBECONFIG", "~/.kube/config")) kserve_client.create(isvc) try: kserve_client.wait_isvc_ready(service_name, namespace=KSERVE_TEST_NAMESPACE) except RuntimeError as e: print( kserve_client.api_instance.get_namespaced_custom_object( "serving.knative.dev", "v1", KSERVE_TEST_NAMESPACE, "services", service_name + "-predictor-default")) pods = kserve_client.core_api.list_namespaced_pod( KSERVE_TEST_NAMESPACE, label_selector='serving.kserve.io/inferenceservice={}'.format( service_name)) for pod in pods.items: print(pod) raise e res = predict(service_name, "./data/transformer.json", model_name="mnist") assert (res.get("predictions")[0] == 2) kserve_client.delete(service_name, KSERVE_TEST_NAMESPACE)
def get_container_env(self) -> Sequence[V1EnvVar]: user_env = [ V1EnvVar(name=name, value=value) for name, value in self.get_env().items() ] return user_env + self.get_kubernetes_environment()
def create_simple_job_spec( self, experiment: Experiment, name: str, image: str, min_cpu: int, min_mem: int, command: Optional[List[str]] = None, env: Dict[str, str] = {}, accelerator: Optional[Accelerator] = None, accelerator_count: int = 1, namespace: str = k.DEFAULT_NAMESPACE, machine_type: Optional[MachineType] = None, preemptible: bool = True, preemptible_tpu: bool = True, tpu_driver: str = k.DEFAULT_TPU_DRIVER) -> Optional[JobSpec]: """creates a simple kubernetes job (1 container, 1 pod) JobSpec for this cluster Args: name: job name image: container image url (gcr.io/...) min_cpu: minimum cpu needed, in milli-cpu min_mem: minimum memory needed, in MB command: command to execute, None = container entrypoint args: args to pass to command env: environment vars for container accelerator: accelerator type, None=cpu only accelerator_count: accelerator count namespace: kubernetes namespace machine_type: machine type, None=default for mode (cpu/gpu) preemptible: use preemptible instance preemptible_tpu: use preemptible tpus tpu_driver: tpu driver to use Returns: JobSpec on success, None otherwise """ args = conf.experiment_to_args(experiment.kwargs, experiment.args) # ------------------------------------------------------------------------ # container # tpu/gpu resources container_resources = V1ResourceRequirements( requests=Cluster.container_requests(min_cpu, min_mem), limits=Cluster.container_limits( accelerator, accelerator_count, preemptible_tpu, ), ) container_env = [V1EnvVar(name=k, value=v) for k, v in env.items()] # this is a simple 1-container, 1-pod job, so we just name the # container the same thing (minus the generated suffix) as the job itself container = V1Container( name=name, image=image, command=command, args=args, resources=container_resources, env=container_env, image_pull_policy='Always', ) # ------------------------------------------------------------------------ # template # todo: should we support anything other than a 'never' restart policy? # see this for discussion # https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/#pod-backoff-failure-policy tolerations = Cluster.tolerations(preemptible=preemptible) # backoff count plus 'OnFailure' may be correct here template_spec = V1PodSpec( restart_policy='Never', containers=[container], tolerations=tolerations, node_selector=Cluster.node_selector( preemptible=preemptible, machine_type=machine_type, accelerator=accelerator, ), host_ipc=True, ) template = V1PodTemplateSpec( metadata=Cluster.template_metadata( accelerator=accelerator, tpu_driver=tpu_driver, ), spec=template_spec, ) # ------------------------------------------------------------------------ # job job_spec = V1JobSpec(template=template, backoff_limit=4) return JobSpec.get_or_create( experiment=experiment, spec=ApiClient().sanitize_for_serialization(job_spec), platform=Platform.GKE, )
JOB_AND_PVC_PREFIX = "flush-" ALREADY_EXISTS = "AlreadyExists" CONFLICT = "Conflict" NOT_FOUND = "Not Found" DEFAULT_IMAGE_VERSION = "latest" try: with open("version.json") as fp: DEFAULT_IMAGE_VERSION = str( json.load(fp).get("version") or DEFAULT_IMAGE_VERSION) except (FileNotFoundError, json.decoder.JSONDecodeError): # pragma: no cover pass # use default DEFAULT_ENV = [ V1EnvVar(name=name, value=os.environ[name]) for name in get_config_dict() if name in os.environ ] parser = ArgumentParser(description=__doc__) parser.add_argument( "--command", default=["python", "-m", "ingestion_edge.flush"], type=json.loads, help= "Docker command for flush jobs; must drain queue until empty or exit non-zero", ) parser.add_argument( "--env", default=DEFAULT_ENV, type=json.loads,
def main(): args = parse_args() logger = get_logger("master", level=args.log_level.upper()) # Master addr master_ip = os.getenv("MY_POD_IP", "localhost") master_addr = "%s:%d" % (master_ip, args.port) # Start tensorboard service if required if args.tensorboard_log_dir: logger.info( "Starting tensorboard service with log directory %s", args.tensorboard_log_dir, ) tb_service = TensorboardService(args.tensorboard_log_dir, master_ip) tb_service.start() else: tb_service = None # Start task queue logger.debug( "Starting task queue with training data directory %s, " "evaluation data directory %s, " "and prediction data directory %s", args.training_data_dir, args.evaluation_data_dir, args.prediction_data_dir, ) task_d = _make_task_dispatcher( args.training_data_dir, args.evaluation_data_dir, args.prediction_data_dir, args.records_per_task, args.num_epochs, ) model_module = load_module( get_module_file_path(args.model_zoo, args.model_def)).__dict__ model_inst = load_model_from_module(args.model_def, model_module, args.model_params) optimizer = model_module[args.optimizer]() if all(( args.training_data_dir, args.evaluation_data_dir, args.evaluation_throttle_secs or args.evaluation_steps, )): job_type = JobType.TRAINING_WITH_EVALUATION elif all(( args.evaluation_data_dir, not args.training_data_dir, not args.prediction_data_dir, )): job_type = JobType.EVALUATION_ONLY elif all(( args.prediction_data_dir, not args.evaluation_data_dir, not args.training_data_dir, )): job_type = JobType.PREDICTION_ONLY else: job_type = JobType.TRAINING_ONLY # Initialize checkpoint service if args.checkpoint_steps or job_type == JobType.TRAINING_WITH_EVALUATION: logger.info("Starting checkpoint service") checkpoint_service = CheckpointService( args.checkpoint_dir, args.checkpoint_steps, args.keep_checkpoint_max, job_type == JobType.TRAINING_WITH_EVALUATION, ) else: checkpoint_service = None # Initialize evaluation service evaluation_service = None if (job_type == JobType.TRAINING_WITH_EVALUATION or job_type == JobType.EVALUATION_ONLY): logger.info( "Starting evaluation service with throttle seconds %d " " and evaluation steps %d", args.evaluation_throttle_secs, args.evaluation_steps, ) evaluation_service = EvaluationService( checkpoint_service, tb_service, task_d, args.evaluation_start_delay_secs, args.evaluation_throttle_secs, args.evaluation_steps, job_type == JobType.EVALUATION_ONLY, ) evaluation_service.start() task_d.set_evaluation_service(evaluation_service) embedding_service_endpoint = None # Search for embedding layers in the model, # if found, initialize embedding service layers = find_layer(model_inst, Embedding) if layers: embedding_service = EmbeddingService() embedding_service_endpoint = embedding_service.start_embedding_service( job_name=args.job_name, image_name=args.worker_image, namespace=args.namespace, resource_request=args.master_resource_request, resource_limit=args.master_resource_limit, pod_priority=args.worker_pod_priority, volume=args.volume, image_pull_policy=args.image_pull_policy, restart_policy=args.restart_policy, cluster_spec=args.cluster_spec, ) logger.info("Embedding service start succeeded. The endpoint is %s." % str(embedding_service_endpoint)) # The master service logger.info("Starting master service") server = grpc.server( futures.ThreadPoolExecutor(max_workers=64), options=[ ("grpc.max_send_message_length", GRPC.MAX_SEND_MESSAGE_LENGTH), ( "grpc.max_receive_message_length", GRPC.MAX_RECEIVE_MESSAGE_LENGTH, ), ], ) master_servicer = MasterServicer( args.grads_to_wait, args.minibatch_size, optimizer, task_d, init_var=model_inst.trainable_variables if model_inst.built else [], checkpoint_filename_for_init=args.checkpoint_filename_for_init, checkpoint_service=checkpoint_service, evaluation_service=evaluation_service, embedding_service_endpoint=embedding_service_endpoint, ) elasticdl_pb2_grpc.add_MasterServicer_to_server(master_servicer, server) server.add_insecure_port("[::]:{}".format(args.port)) server.start() logger.info("Server started at port: %d", args.port) worker_manager = None if args.num_workers: assert args.worker_image, "Worker image cannot be empty" worker_command = ["python"] worker_args = [ "-m", "elasticdl.python.worker.main", "--model_zoo", args.model_zoo, "--master_addr", master_addr, "--log_level", args.log_level, "--dataset_fn", args.dataset_fn, "--loss", args.loss, "--optimizer", args.optimizer, "--eval_metrics_fn", args.eval_metrics_fn, "--model_def", args.model_def, "--job_type", job_type, "--minibatch_size", str(args.minibatch_size), "--embedding_service_endpoint", str(embedding_service_endpoint), ] logger.info(">>> master pod envs argument is %s" % args.envs) env_dict = parse_envs(args.envs) env = [] for key in env_dict: env.append(V1EnvVar(name=key, value=env_dict[key])) worker_manager = WorkerManager( task_d, job_name=args.job_name, image_name=args.worker_image, command=worker_command, args=worker_args, namespace=args.namespace, num_workers=args.num_workers, worker_resource_request=args.worker_resource_request, worker_resource_limit=args.worker_resource_limit, pod_priority=args.worker_pod_priority, volume=args.volume, image_pull_policy=args.image_pull_policy, restart_policy=args.restart_policy, cluster_spec=args.cluster_spec, envs=env, ) worker_manager.update_status(WorkerManagerStatus.PENDING) logger.info("Launching %d workers", args.num_workers) worker_manager.start_workers() worker_manager.update_status(WorkerManagerStatus.RUNNING) if tb_service: worker_manager.start_tensorboard_service() try: while True: if task_d.finished(): if worker_manager: worker_manager.update_status(WorkerManagerStatus.FINISHED) if args.output: master_servicer.save_latest_checkpoint(args.output) break time.sleep(30) except KeyboardInterrupt: logger.warning("Server stopping") if evaluation_service: logger.info("Stopping evaluation service") evaluation_service.stop() logger.info("Stopping RPC server") server.stop(0) # Keep TensorBoard running when all the tasks are finished if tb_service: logger.info( "All tasks finished. Keeping TensorBoard service running...") while True: if tb_service.is_active(): time.sleep(10) else: logger.warning("Unable to keep TensorBoard running. " "It has already terminated") break logger.info("Master stopped")
def ensure_traefik(api_core_v1, api_ext_v1_beta1, api_apps_v1, api_custom, api_rbac_auth_v1_b1, admin_email, domain, static_ip, oauth_client_id, oauth_client_secret, oauth_domain, oauth_secret): ensure_crd(api=api_ext_v1_beta1, name='ingressroutes.traefik.containo.us', group='traefik.containo.us', kind='IngressRoute', plural='ingressroutes', singular='ingressroute', scope='Namespaced') ensure_crd(api=api_ext_v1_beta1, name='ingressroutetcps.traefik.containo.us', group='traefik.containo.us', kind='IngressRouteTCP', plural='ingressroutetcps', singular='ingressroutetcp', scope='Namespaced') ensure_crd(api=api_ext_v1_beta1, name='middlewares.traefik.containo.us', group='traefik.containo.us', kind='Middleware', plural='middlewares', singular='middleware', scope='Namespaced') ensure_crd(api=api_ext_v1_beta1, name='tlsoptions.traefik.containo.us', group='traefik.containo.us', kind='TLSOption', plural='tlsoptions', singular='tlsoption', scope='Namespaced') ensure_role(api=api_rbac_auth_v1_b1, role=V1ClusterRole( api_version='rbac.authorization.k8s.io/v1beta1', kind='ClusterRole', metadata=V1ObjectMeta(name='traefik-ingress-controller'), rules=[ V1PolicyRule( api_groups=[''], resources=['services', 'endpoints', 'secrets'], verbs=['get', 'list', 'watch']), V1PolicyRule(api_groups=['extensions'], resources=['ingresses'], verbs=['get', 'list', 'watch']), V1PolicyRule(api_groups=['extensions'], resources=['ingresses/status'], verbs=['update']), V1PolicyRule(api_groups=['traefik.containo.us'], resources=['middlewares'], verbs=['get', 'list', 'watch']), V1PolicyRule(api_groups=['traefik.containo.us'], resources=['ingressroutes'], verbs=['get', 'list', 'watch']), V1PolicyRule(api_groups=['traefik.containo.us'], resources=['ingressroutetcps'], verbs=['get', 'list', 'watch']), V1PolicyRule(api_groups=['traefik.containo.us'], resources=['tlsoptions'], verbs=['get', 'list', 'watch']) ]), name='traefik-ingress-controller') ensure_role_binding( api=api_rbac_auth_v1_b1, role_binding=V1ClusterRoleBinding( api_version='rbac.authorization.k8s.io/v1beta1', kind='ClusterRoleBinding', metadata=V1ObjectMeta(name='traefik-ingress-controller'), role_ref=V1RoleRef(api_group='rbac.authorization.k8s.io', kind='ClusterRole', name='traefik-ingress-controller'), subjects=[ V1Subject(kind='ServiceAccount', name='traefik-ingress-controller', namespace='default') ]), name='traefik-ingress-controller') ensure_service( api=api_core_v1, service=V1Service( api_version="v1", metadata=V1ObjectMeta(name='traefik'), spec=V1ServiceSpec( type='LoadBalancer', load_balancer_ip=static_ip, ports=[ # V1ServicePort( # protocol='TCP', # port=80, # name='web' # ), V1ServicePort(protocol='TCP', port=443, name='websecure'), ], selector={'app': 'traefik'})), name='traefik', namespace='default') ensure_service_account( api=api_core_v1, account=V1ServiceAccount( api_version="v1", metadata=V1ObjectMeta(name='traefik-ingress-controller'), ), name='traefik-ingress-controller', namespace='default') ensure_deployment( api=api_apps_v1, deployment=V1Deployment( api_version="apps/v1", metadata=V1ObjectMeta(name='traefik', labels={'app': 'traefik'}), spec=V1DeploymentSpec( replicas=1, selector=V1LabelSelector(match_labels={'app': 'traefik'}), template=V1PodTemplateSpec( metadata=V1ObjectMeta(name='traefik', labels={'app': 'traefik'}), spec=V1PodSpec( service_account_name='traefik-ingress-controller', containers=[ V1Container( name='traefik', image='traefik:v2.0', args=[ '--api.insecure', '--accesslog', '--entrypoints.web.Address=:80', '--entrypoints.websecure.Address=:443', '--providers.kubernetescrd', '--certificatesresolvers.default.acme.tlschallenge', f'--certificatesresolvers.default.acme.email={admin_email}', '--certificatesresolvers.default.acme.storage=acme.json', # '--certificatesresolvers.default.acme.caserver=https://acme-staging-v02.api.letsencrypt.org/directory', ], ports=[ V1ContainerPort(name='web', container_port=8000), V1ContainerPort(name='websecure', container_port=4443), V1ContainerPort(name='admin', container_port=8080), ]) ])))), name='traefik', namespace='default') ensure_deployment( api=api_apps_v1, deployment=V1Deployment( api_version="apps/v1", metadata=V1ObjectMeta(name='traefik-forward-auth', labels={'app': 'traefik-forward-auth'}), spec=V1DeploymentSpec( replicas=1, selector=V1LabelSelector( match_labels={'app': 'traefik-forward-auth'}), template=V1PodTemplateSpec( metadata=V1ObjectMeta( name='traefik-forward-auth', labels={'app': 'traefik-forward-auth'}), spec=V1PodSpec(containers=[ V1Container( name='traefik-forward-auth', image='thomseddon/traefik-forward-auth:2', ports=[ V1ContainerPort(name='auth', container_port=4181), ], env=[ V1EnvVar(name='PROVIDERS_GOOGLE_CLIENT_ID', value=oauth_client_id), # V1EnvVar(name='LOG_LEVEL', value='trace'), V1EnvVar(name='PROVIDERS_GOOGLE_CLIENT_SECRET', value=oauth_client_secret), V1EnvVar(name='SECRET', value=oauth_secret), V1EnvVar(name='DOMAIN', value=oauth_domain), V1EnvVar(name='COOKIE_DOMAIN', value=domain), V1EnvVar(name='AUTH_HOST', value=f'auth.{domain}'), ]) ])))), name='traefik-forward-auth', namespace='default') ensure_custom_object(api=api_custom, custom_object={ 'apiVersion': 'traefik.containo.us/v1alpha1', 'kind': 'IngressRoute', 'metadata': { 'name': 'traefik-forward-auth', }, 'spec': { 'entryPoints': ['websecure'], 'routes': [{ 'match': f'Host(`auth.{domain}`)', 'kind': 'Rule', 'services': [{ 'name': 'traefik-forward-auth', 'port': 4181 }], 'middlewares': [{ 'name': 'traefik-forward-auth' }] }], 'tls': { 'certResolver': 'default' } } }, group='traefik.containo.us', plural='ingressroutes', version='v1alpha1', name='traefik-forward-auth', namespace='default') ensure_custom_object(api=api_custom, custom_object={ 'apiVersion': 'traefik.containo.us/v1alpha1', 'kind': 'Middleware', 'metadata': { 'name': 'traefik-forward-auth', }, 'spec': { 'forwardAuth': { 'address': 'http://traefik-forward-auth:4181', 'authResponseHeaders': ['X-Forwarded-User'], } } }, group='traefik.containo.us', plural='middlewares', version='v1alpha1', name='traefik-forward-auth', namespace='default') ensure_service(api=api_core_v1, service=V1Service( api_version="v1", metadata=V1ObjectMeta(name='traefik-forward-auth'), spec=V1ServiceSpec( type='ClusterIP', ports=[ V1ServicePort(protocol='TCP', port=4181, name='auth'), ], selector={'app': 'traefik-forward-auth'})), name='traefik-forward-auth', namespace='default') ensure_whoami(api_apps_v1, api_core_v1, api_custom, domain)
def test_transformer(): service_name = 'raw' predictor = V1beta1PredictorSpec( min_replicas=1, pytorch=V1beta1TorchServeSpec( storage_uri= 'gs://kfserving-examples/models/torchserve/image_classifier', resources=V1ResourceRequirements(requests={ 'cpu': '100m', 'memory': '2Gi' }, limits={ 'cpu': '100m', 'memory': '2Gi' })), ) transformer = V1beta1TransformerSpec( min_replicas=1, containers=[ V1Container( image='kfserving/torchserve-image-transformer:latest', name='kfserving-container', resources=V1ResourceRequirements(requests={ 'cpu': '100m', 'memory': '2Gi' }, limits={ 'cpu': '100m', 'memory': '2Gi' }), env=[ V1EnvVar( name="STORAGE_URI", value= "gs://kfserving-examples/models/torchserve/image_classifier" ) ]) ]) annotations = dict() annotations['serving.kubeflow.org/raw'] = 'true' annotations['kubernetes.io/ingress.class'] = 'istio' isvc = V1beta1InferenceService( api_version=constants.KFSERVING_V1BETA1, kind=constants.KFSERVING_KIND, metadata=client.V1ObjectMeta(name=service_name, namespace=KFSERVING_TEST_NAMESPACE, annotations=annotations), spec=V1beta1InferenceServiceSpec(predictor=predictor, transformer=transformer)) KFServing.create(isvc) try: KFServing.wait_isvc_ready(service_name, namespace=KFSERVING_TEST_NAMESPACE) except RuntimeError as e: raise e time.sleep(30) isvc = KFServing.get( service_name, namespace=KFSERVING_TEST_NAMESPACE, ) cluster_ip = get_cluster_ip() logging.info("clusterip = %s", cluster_ip) host = isvc["status"]["url"] host = host[host.rfind('/') + 1:] url = 'http://{}/v1/models/mnist:predict'.format(cluster_ip) logging.info("url = %s ", url) headers = {"Host": host} data_str = '{"instances": [{"data": "iVBORw0KGgoAAAANSUhEUgAAABwAAAAcCAAAAABXZoBIAAAAw0lE\ QVR4nGNgGFggVVj4/y8Q2GOR83n+58/fP0DwcSqmpNN7oOTJw6f+/H2pjUU2JCSEk0EWqN0cl828e/FIxvz9/9cCh1\ zS5z9/G9mwyzl/+PNnKQ45nyNAr9ThMHQ/UG4tDofuB4bQIhz6fIBenMWJQ+7Vn7+zeLCbKXv6z59NOPQVgsIcW\ 4QA9YFi6wNQLrKwsBebW/68DJ388Nun5XFocrqvIFH59+XhBAxThTfeB0r+vP/QHbuDCgr2JmOXoSsAAKK7b\ U3vISS4AAAAAElFTkSuQmCC", "target": 0}]}' res = requests.post(url, data_str, headers=headers) logging.info("res.text = %s", res.text) preds = json.loads(res.content.decode("utf-8")) assert (preds["predictions"] == [2]) KFServing.delete(service_name, KFSERVING_TEST_NAMESPACE)
def apply_pod_profile(self, username, pod, profile, default_mount_path, gpu_mode=None): api_client = kubernetes.client.ApiClient() pod.metadata.labels['jupyterhub.opendatahub.io/user'] = escape(username) profile_volumes = profile.get('volumes') if profile_volumes: for volume in profile_volumes: volume_name = re.sub('[^a-zA-Z0-9\.]', '-', volume['name']).lower() read_only = volume['persistentVolumeClaim'].get('readOnly') pvc = V1PersistentVolumeClaimVolumeSource(volume['persistentVolumeClaim']['claimName'], read_only=read_only) mount_path = self.generate_volume_path(volume.get('mountPath'), default_mount_path, volume_name) pod.spec.volumes.append(V1Volume(name=volume_name, persistent_volume_claim=pvc)) pod.spec.containers[0].volume_mounts.append(V1VolumeMount(name=volume_name, mount_path=mount_path)) profile_environment = profile.get('env') if profile_environment: # Kept for backwards compatibility with simplified env var definitions if isinstance(profile_environment, dict): for k, v in profile['env'].items(): update = False for e in pod.spec.containers[0].env: if e.name == k: e.value = v update = True break if not update: pod.spec.containers[0].env.append(V1EnvVar(k, v)) elif isinstance(profile_environment, list): for i in profile_environment: r = type("Response", (), {}) r.data = json.dumps(i) env_var = api_client.deserialize(r, V1EnvVar) pod.spec.containers[0].env.append(env_var) resource_var = None resource_json = type("Response", (), {}) resource_json.data = json.dumps(profile.get('resources')) resource_var = api_client.deserialize(resource_json, V1ResourceRequirements) if resource_var: pod.spec.containers[0].resources = resource_var mem_limit = resource_var.limits.get('memory', '') if mem_limit: pod.spec.containers[0].env.append(V1EnvVar(name='MEM_LIMIT', value=self.get_mem_limit(mem_limit))) if profile.get('node_tolerations'): pod.spec.tolerations = profile.get('node_tolerations') if profile.get('node_affinity'): if not pod.spec.affinity: pod.spec.affinity = {} pod.spec.affinity['nodeAffinity'] = profile.get('node_affinity') for c in pod.spec.containers: update = False if type(c) is dict: env = c['env'] else: env = c.env for e in env: if type(e) is dict: if e['name'] == _JUPYTERHUB_USER_NAME_ENV: e['value'] = username update = True break else: if e.name == _JUPYTERHUB_USER_NAME_ENV: e.value = username update = True break if not update: env.append(V1EnvVar(_JUPYTERHUB_USER_NAME_ENV, username)) self.apply_gpu_config(gpu_mode, profile.get('gpu', 0), pod) return pod
def _get_container(self, command: str, config_map_volume_names: list = None) -> V1Container: """Get the K8s container object to be used by by a PodSpec. :param command: The command to run in the container. :return: The V1Container object definition to run the command. """ if self.use_https: protocol = "https" else: protocol = "http" config_map_mounts = [] if config_map_volume_names: for map_number, map_volume in enumerate(config_map_volume_names): mount_dir = self.config_map_path + f"/{map_number}" config_map_mounts.append( V1VolumeMount(name=map_volume, mount_path=mount_dir) ) env_setup_ca1 = "mkdir -p /root/.mc/certs/CAs; " # The find command doesn't exist by default in the minio container env_setup_ca2 = f"cp {self.config_map_path}/*/* /root/.mc/certs/CAs/; " env_setup_ca = env_setup_ca1 + env_setup_ca2 else: env_setup_ca = "" env_setup1 = f"export MC_HOST_{self.s3_alias}={protocol}://${{S3_ACCESSKEY}}:${{S3_SECRETKEY}}@" env_setup2 = f"{self.s3_host}:{self.s3_port}; " container_setup = env_setup_ca + env_setup1 + env_setup2 container = V1Container( name="netapp-dataops-s3mc-container", image=self.image, env=[ V1EnvVar( name="S3_ACCESSKEY", value_from=V1EnvVarSource( secret_key_ref=V1SecretKeySelector( name=self.credentials_secret_name, key="access_key" ) ) ), V1EnvVar( name="S3_SECRETKEY", value_from=V1EnvVarSource( secret_key_ref=V1SecretKeySelector( name=self.credentials_secret_name, key="secret_key" ) ) ), ], command=["/bin/bash"], args=["-c", container_setup + command], volume_mounts=[ V1VolumeMount(mount_path=self.data_volume_path, name=self.data_volume_name) ] + config_map_mounts, resources=self._get_resource_requirements() ) return container
def _create_instance_manager(self, args): instance_manager = None if args.num_workers: assert args.worker_image, "Worker image cannot be empty" worker_command = ["python"] worker_args = [ "-m", "elasticdl.python.worker.main", "--master_addr", self.master_addr, "--job_type", self.job_type, ] worker_args.extend(build_arguments_from_parsed_result(args)) ps_command = ["python"] ps_args = [ "-m", "elasticdl.python.ps.main", "--grads_to_wait", str(args.grads_to_wait), "--lr_staleness_modulation", str(args.lr_staleness_modulation), "--use_async", str(args.use_async), "--minibatch_size", str(args.minibatch_size), "--model_zoo", args.model_zoo, "--model_def", args.model_def, "--job_name", args.job_name, "--num_minibatches_per_task", str(args.num_minibatches_per_task), "--port", "2222", "--master_addr", self.master_addr, "--namespace", args.namespace, "--evaluation_steps", str(args.evaluation_steps), "--checkpoint_dir", str(args.checkpoint_dir), "--checkpoint_steps", str(args.checkpoint_steps), "--keep_checkpoint_max", str(args.keep_checkpoint_max), "--num_ps_pods", str(args.num_ps_pods), "--checkpoint_dir_for_init", str(args.checkpoint_dir_for_init), "--num_workers", str(args.num_workers), "--log_level", str(args.log_level), ] env_dict = parse_envs(args.envs) env = [] for key in env_dict: env.append(V1EnvVar(name=key, value=env_dict[key])) instance_manager = InstanceManager( self.task_d, job_name=args.job_name, image_name=args.worker_image, worker_command=worker_command, worker_args=worker_args, namespace=args.namespace, num_workers=args.num_workers, worker_resource_request=args.worker_resource_request, worker_resource_limit=args.worker_resource_limit, worker_pod_priority=args.worker_pod_priority, num_ps=args.num_ps_pods, ps_command=ps_command, ps_args=ps_args, ps_resource_request=args.ps_resource_request, ps_resource_limit=args.ps_resource_limit, ps_pod_priority=args.ps_pod_priority, volume=args.volume, image_pull_policy=args.image_pull_policy, restart_policy=args.restart_policy, cluster_spec=args.cluster_spec, envs=env, ) return instance_manager
def apply_pod_profile(self, spawner, pod, profile): api_client = kubernetes.client.ApiClient() pod.metadata.labels['jupyterhub.opendatahub.io/user'] = escape( spawner.user.name) profile_environment = profile.get('env') if profile_environment: # Kept for backwards compatibility with simplified env var definitions if isinstance(profile_environment, dict): for k, v in profile['env'].items(): update = False for e in pod.spec.containers[0].env: if e.name == k: e.value = v update = True break if not update: pod.spec.containers[0].env.append(V1EnvVar(k, v)) elif isinstance(profile_environment, list): for i in profile_environment: r = type("Response", (), {}) r.data = json.dumps(i) env_var = api_client.deserialize(r, V1EnvVar) pod.spec.containers[0].env.append(env_var) resource_var = None resource_json = type("Response", (), {}) resource_json.data = json.dumps(profile.get('resources')) resource_var = api_client.deserialize(resource_json, V1ResourceRequirements) if resource_var: pod.spec.containers[0].resources = resource_var if profile.get('node_tolerations'): pod.spec.tolerations = profile.get('node_tolerations') if profile.get('node_affinity'): if not pod.spec.affinity: pod.spec.affinity = {} pod.spec.affinity['nodeAffinity'] = profile.get('node_affinity') for c in pod.spec.containers: update = False if type(c) is dict: env = c['env'] else: env = c.env for e in env: if type(e) is dict: if e['name'] == _JUPYTERHUB_USER_NAME_ENV: e['value'] = spawner.user.name update = True break else: if e.name == _JUPYTERHUB_USER_NAME_ENV: e.value = spawner.user.name update = True break if not update: env.append( V1EnvVar(_JUPYTERHUB_USER_NAME_ENV, spawner.user.name)) self.apply_gpu_config(spawner.gpu_mode, spawner.gpu_count, pod) return pod
def get_reference_object(self) -> V1Deployment: """Get deployment object for outpost""" # Generate V1ContainerPort objects container_ports = [] for port in self.controller.deployment_ports: container_ports.append( V1ContainerPort( container_port=port.inner_port or port.port, name=port.name, protocol=port.protocol.upper(), ) ) meta = self.get_object_meta(name=self.name) image_name = self.controller.get_container_image() image_pull_secrets = self.outpost.config.kubernetes_image_pull_secrets version = get_full_version() return V1Deployment( metadata=meta, spec=V1DeploymentSpec( replicas=self.outpost.config.kubernetes_replicas, selector=V1LabelSelector(match_labels=self.get_pod_meta()), template=V1PodTemplateSpec( metadata=V1ObjectMeta( labels=self.get_pod_meta( **{ # Support istio-specific labels, but also use the standard k8s # recommendations "app.kubernetes.io/version": version, "app": "authentik-outpost", "version": version, } ) ), spec=V1PodSpec( image_pull_secrets=[ V1ObjectReference(name=secret) for secret in image_pull_secrets ], containers=[ V1Container( name=str(self.outpost.type), image=image_name, ports=container_ports, env=[ V1EnvVar( name="AUTHENTIK_HOST", value_from=V1EnvVarSource( secret_key_ref=V1SecretKeySelector( name=self.name, key="authentik_host", ) ), ), V1EnvVar( name="AUTHENTIK_HOST_BROWSER", value_from=V1EnvVarSource( secret_key_ref=V1SecretKeySelector( name=self.name, key="authentik_host_browser", ) ), ), V1EnvVar( name="AUTHENTIK_TOKEN", value_from=V1EnvVarSource( secret_key_ref=V1SecretKeySelector( name=self.name, key="token", ) ), ), V1EnvVar( name="AUTHENTIK_INSECURE", value_from=V1EnvVarSource( secret_key_ref=V1SecretKeySelector( name=self.name, key="authentik_host_insecure", ) ), ), ], ) ], ), ), ), )
def __init__(self) -> None: metadata = V1ObjectMeta(name="environmentd", labels={"app": "environmentd"}) label_selector = V1LabelSelector(match_labels={"app": "environmentd"}) value_from = V1EnvVarSource(field_ref=V1ObjectFieldSelector( field_path="metadata.name")) env = [ V1EnvVar(name="MZ_POD_NAME", value_from=value_from), V1EnvVar(name="AWS_REGION", value="minio"), V1EnvVar(name="AWS_ACCESS_KEY_ID", value="minio"), V1EnvVar(name="AWS_SECRET_ACCESS_KEY", value="minio123"), ] ports = [V1ContainerPort(container_port=5432, name="sql")] volume_mounts = [ V1VolumeMount(name="data", mount_path="/data"), ] s3_endpoint = urllib.parse.quote("http://minio-service.default:9000") container = V1Container( name="environmentd", image=self.image("environmentd"), args=[ "--storaged-image=" + self.image("storaged"), "--computed-image=" + self.image("computed"), "--availability-zone=kind-worker", "--availability-zone=kind-worker2", "--availability-zone=kind-worker3", f"--persist-blob-url=s3://minio:minio123@persist/persist?endpoint={s3_endpoint}®ion=minio", "--orchestrator=kubernetes", "--orchestrator-kubernetes-image-pull-policy=never", "--persist-consensus-url=postgres://[email protected]?options=--search_path=consensus", "--adapter-stash-url=postgres://[email protected]?options=--search_path=catalog", "--storage-stash-url=postgres://[email protected]?options=--search_path=storage", "--unsafe-mode", ], env=env, ports=ports, volume_mounts=volume_mounts, ) pod_spec = V1PodSpec(containers=[container]) template_spec = V1PodTemplateSpec(metadata=metadata, spec=pod_spec) claim_templates = [ V1PersistentVolumeClaim( metadata=V1ObjectMeta(name="data"), spec=V1PersistentVolumeClaimSpec( access_modes=["ReadWriteOnce"], resources=V1ResourceRequirements( requests={"storage": "1Gi"}), ), ) ] self.stateful_set = V1StatefulSet( api_version="apps/v1", kind="StatefulSet", metadata=metadata, spec=V1StatefulSetSpec( service_name="environmentd", replicas=1, pod_management_policy="Parallel", selector=label_selector, template=template_spec, volume_claim_templates=claim_templates, ), )