def init_spark_on_k8s(master, container_image, num_executors, executor_cores, executor_memory="2g", driver_memory="1g", driver_cores=4, extra_executor_memory_for_ray=None, extra_python_lib=None, spark_log_level="WARN", redirect_spark_log=True, jars=None, conf=None, python_location=None): from zoo.util.spark import SparkRunner runner = SparkRunner(spark_log_level=spark_log_level, redirect_spark_log=redirect_spark_log) sc = runner.init_spark_on_k8s( master=master, container_image=container_image, num_executors=num_executors, executor_cores=executor_cores, executor_memory=executor_memory, driver_memory=driver_memory, driver_cores=driver_cores, extra_executor_memory_for_ray=extra_executor_memory_for_ray, extra_python_lib=extra_python_lib, jars=jars, conf=conf, python_location=python_location) return sc
def init_spark_on_k8s(master, container_image, num_executors, executor_cores, executor_memory="2g", driver_memory="1g", driver_cores=4, extra_executor_memory_for_ray=None, extra_python_lib=None, spark_log_level="WARN", redirect_spark_log=True, jars=None, conf=None, python_location=None): """ Create a SparkContext with Analytics Zoo configurations on Kubernetes cluster for k8s client mode. You are recommended to use the Docker image intelanalytics/hyperzoo:latest. You can refer to https://github.com/intel-analytics/analytics-zoo/tree/master/docker/hyperzoo to build your own Docker image. :param master: The master address of your k8s cluster. :param container_image: The name of the docker container image for Spark executors. For example, intelanalytics/hyperzoo:latest :param num_executors: The number of Spark executors. :param executor_cores: The number of cores for each executor. :param executor_memory: The memory for each executor. Default to be '2g'. :param driver_cores: The number of cores for the Spark driver. Default to be 4. :param driver_memory: The memory for the Spark driver. Default to be '1g'. :param extra_executor_memory_for_ray: The extra memory for Ray services. Default to be None. :param extra_python_lib: Extra python files or packages needed for distribution. Default to be None. :param spark_log_level: The log level for Spark. Default to be 'WARN'. :param redirect_spark_log: Whether to redirect the Spark log to local file. Default to be True. :param jars: Comma-separated list of jars to be included on driver and executor's classpath. Default to be None. :param conf: You can append extra conf for Spark in key-value format. i.e conf={"spark.executor.extraJavaOptions": "-XX:+PrintGCDetails"}. Default to be None. :param python_location: The path to your running Python executable. If not specified, the default Python interpreter in effect would be used. :return: An instance of SparkContext. """ from zoo.util.spark import SparkRunner runner = SparkRunner(spark_log_level=spark_log_level, redirect_spark_log=redirect_spark_log) sc = runner.init_spark_on_k8s( master=master, container_image=container_image, num_executors=num_executors, executor_cores=executor_cores, executor_memory=executor_memory, driver_memory=driver_memory, driver_cores=driver_cores, extra_executor_memory_for_ray=extra_executor_memory_for_ray, extra_python_lib=extra_python_lib, jars=jars, conf=conf, python_location=python_location) return sc
def init_spark_on_k8s(master, container_image, num_executors, executor_cores, executor_memory="2g", driver_memory="1g", driver_cores=4, extra_executor_memory_for_ray=None, extra_python_lib=None, spark_log_level="WARN", redirect_spark_log=True, jars=None, conf=None, python_location=None): """Returns the local TrainingOperator object. Be careful not to perturb its state, or else you can cause the system to enter an inconsistent state. Args: num_steps (int): Number of batches to compute update steps on per worker. This corresponds also to the number of times ``TrainingOperator.validate_batch`` is called per worker. profile (bool): Returns time stats for the evaluation procedure. reduce_results (bool): Whether to average all metrics across all workers into one dict. If a metric is a non-numerical value (or nested dictionaries), one value will be randomly selected among the workers. If False, returns a list of dicts. info (dict): Optional dictionary passed to the training operator for `validate` and `validate_batch`. Returns: TrainingOperator: The local TrainingOperator object. """ from zoo.util.spark import SparkRunner runner = SparkRunner(spark_log_level=spark_log_level, redirect_spark_log=redirect_spark_log) sc = runner.init_spark_on_k8s( master=master, container_image=container_image, num_executors=num_executors, executor_cores=executor_cores, executor_memory=executor_memory, driver_memory=driver_memory, driver_cores=driver_cores, extra_executor_memory_for_ray=extra_executor_memory_for_ray, extra_python_lib=extra_python_lib, jars=jars, conf=conf, python_location=python_location) return sc