def init_spark_on_local(cores=2, conf=None, python_location=None, spark_log_level="WARN", redirect_spark_log=True): """ Create a SparkContext with Analytics Zoo configurations on the local machine. :param cores: The number of cores for Spark local. Default to be 2. You can also set it to "*" to use all the available cores. i.e `init_spark_on_local(cores="*")` :param conf: You can append extra conf for Spark in key-value format. i.e conf={"spark.executor.extraJavaOptions": "-XX:+PrintGCDetails"}. Default to be None. :param python_location: The path to your running Python executable. If not specified, the default Python interpreter in effect would be used. :param spark_log_level: The log level for Spark. Default to be 'WARN'. :param redirect_spark_log: Whether to redirect the Spark log to local file. Default to be True. :return: An instance of SparkContext. """ from zoo.util.spark import SparkRunner runner = SparkRunner(spark_log_level=spark_log_level, redirect_spark_log=redirect_spark_log) set_python_home() return runner.init_spark_on_local(cores=cores, conf=conf, python_location=python_location)
def init_spark_on_local(cores=2, conf=None, python_location=None, spark_log_level="WARN", redirect_spark_log=True): """Saves the Trainer state to the provided checkpoint path. Args: checkpoint (str): Path to target checkpoint file. """ from zoo.util.spark import SparkRunner runner = SparkRunner(spark_log_level=spark_log_level, redirect_spark_log=redirect_spark_log) set_python_home() return runner.init_spark_on_local(cores=cores, conf=conf, python_location=python_location)
def init_spark_on_local(cores=2, conf=None, python_location=None, spark_log_level="WARN", redirect_spark_log=True): """ Create a SparkContext with Zoo configuration in local machine. :param cores: The default value is 2 and you can also set it to * meaning all of the available cores. i.e `init_on_local(cores="*")` :param conf: A key value dictionary appended to SparkConf. :param python_location: The path to your running python executable. :param spark_log_level: Log level of Spark :param redirect_spark_log: Redirect the Spark log to local file or not. :return: """ from zoo.util.spark import SparkRunner sparkrunner = SparkRunner(spark_log_level=spark_log_level, redirect_spark_log=redirect_spark_log) return sparkrunner.init_spark_on_local(cores=cores, conf=conf, python_location=python_location)