def run_submission_with_function_preprocessor(capsys, deployer="job", builder="append", namespace="default", dockerfile_path=None, cleanup=False): py_version = ".".join([str(x) for x in sys.version_info[0:3]]) base_image = 'registry.hub.docker.com/library/python:{}'.format(py_version) if builder == 'cluster': if dockerfile_path is None: fairing.config.set_builder( builder, base_image=base_image, registry=DOCKER_REGISTRY, pod_spec_mutators=[ fairing.cloud.gcp.add_gcp_credentials_if_exists ], context_source=gcs_context.GCSContextSource( namespace=namespace), namespace=namespace) else: fairing.config.set_builder( builder, registry=DOCKER_REGISTRY, dockerfile_path=dockerfile_path, pod_spec_mutators=[ fairing.cloud.gcp.add_gcp_credentials_if_exists ], context_source=gcs_context.GCSContextSource( namespace=namespace), namespace=namespace) else: fairing.config.set_builder(builder, base_image=base_image, registry=DOCKER_REGISTRY) expected_result = str(uuid.uuid4()) fairing.config.set_deployer(deployer, namespace=namespace, cleanup=cleanup, labels={'pytest-id': expected_result}) remote_train = fairing.config.fn(lambda: train_fn(expected_result)) remote_train() captured = capsys.readouterr() assert expected_result in captured.out if deployer == "tfjob": if cleanup: assert expected_result not in str( get_tfjobs_with_labels('pytest-id=' + expected_result)) else: assert expected_result in str( get_tfjobs_with_labels('pytest-id=' + expected_result))
def get_builder( self, preprocessor, base_image, registry, needs_deps_installation=True, # pylint:disable=arguments-differ pod_spec_mutators=None): """Creates a builder instance with right config for GCP :param preprocessor: Preprocessor to use to modify inputs :param base_image: Base image to use for this job :param registry: Registry to push image to. Example: gcr.io/kubeflow-images :param needs_deps_installation: need depends on installation(Default value = True) :param pod_spec_mutators: list of functions that is used to mutate the podsspec. e.g. fairing.cloud.gcp.add_gcp_credentials_if_exists This can used to set things like volumes and security context. (Default value =None) """ pod_spec_mutators = pod_spec_mutators or [] pod_spec_mutators.append(gcp.add_gcp_credentials_if_exists) pod_spec_mutators.append(docker.add_docker_credentials_if_exists) # TODO (karthikv2k): Add cloud build as the deafult # once https://github.com/kubeflow/fairing/issues/145 is fixed if not needs_deps_installation: return AppendBuilder(preprocessor=preprocessor, base_image=base_image, registry=registry) elif utils.is_running_in_k8s(): return ClusterBuilder( preprocessor=preprocessor, base_image=base_image, registry=registry, pod_spec_mutators=pod_spec_mutators, context_source=gcs_context.GCSContextSource( namespace=utils.get_default_target_namespace())) elif ml_tasks_utils.is_docker_daemon_exists(): return DockerBuilder(preprocessor=preprocessor, base_image=base_image, registry=registry) else: # TODO (karthikv2k): Add more info on how to reolve this issue raise RuntimeError( "Not able to guess the right builder for this job!")
def __init__(self, namespace=None, build_context_source=None): super(GKEBackend, self).__init__(namespace, build_context_source) self._build_context_source = gcs_context.GCSContextSource( namespace=self._namespace)