def get_zoo_bigdl_classpath_on_driver():
from bigdl.util.engine import get_bigdl_classpath
from zoo.util.engine import get_analytics_zoo_classpath
bigdl_classpath = get_bigdl_classpath()
assert bigdl_classpath, "Cannot find BigDL classpath, please check your installation"
zoo_classpath = get_analytics_zoo_classpath()
assert zoo_classpath, "Cannot find Analytics-Zoo classpath, please check your installation"
return zoo_classpath, bigdl_classpath
def _yarn_opt(jars):
from zoo.util.engine import get_analytics_zoo_classpath
command = " --archives {}#python_env --num-executors {} " \
" --executor-cores {} --executor-memory {}".\
format(penv_archive, num_executor, executor_cores, executor_memory)
path_to_zoo_jar = get_analytics_zoo_classpath()
if extra_python_lib:
command = command + " --py-files {} ".format(extra_python_lib)
if jars:
command = command + " --jars {},{} ".format(
jars, path_to_zoo_jar)
elif path_to_zoo_jar:
command = command + " --jars {} ".format(path_to_zoo_jar)
if path_to_zoo_jar:
command = command + " --conf spark.driver.extraClassPath={} ".\
format(get_analytics_zoo_classpath())
return command
def copy_zoo_jar(self):
jar_path = get_analytics_zoo_classpath()
if jar_path:
self.try_copy_bigdl_jar()
else:
"""
not install by pip, so run prepare_env here
"""
jar_paths = glob.glob(
os.path.abspath(__file__ + "/../../../../dist/lib/*.jar"))
assert len(jar_paths) > 0, "No zoo jar is found"
assert len(
jar_paths) == 1, "Expecting one jar: %s" % len(jar_paths)
jar_path = jar_paths[0]
shutil.copyfile(jar_path, self.zoo_jar)
def _get_zoo_jar_name_on_driver(self):
from zoo.util.engine import get_analytics_zoo_classpath
zoo_classpath = get_analytics_zoo_classpath()
assert zoo_classpath, "Cannot find Analytics-Zoo classpath"
return zoo_classpath.split("/")[-1]
def init_spark_standalone(self,
num_executors,
executor_cores,
executor_memory="10g",
driver_memory="1g",
driver_cores=4,
master=None,
extra_executor_memory_for_ray=None,
extra_python_lib=None,
conf=None,
jars=None):
import subprocess
import pyspark
from zoo.util.utils import get_node_ip
from zoo.util.engine import get_analytics_zoo_classpath
from bigdl.util.engine import get_bigdl_classpath
if 'PYSPARK_PYTHON' not in os.environ:
os.environ["PYSPARK_PYTHON"] = self._detect_python_location()
if not master:
pyspark_home = os.path.abspath(pyspark.__file__ + "/../")
zoo_standalone_home = os.path.abspath(
__file__ + "/../../share/bin/standalone")
node_ip = get_node_ip()
SparkRunner.standalone_env = {
"SPARK_HOME": pyspark_home,
"ZOO_STANDALONE_HOME": zoo_standalone_home,
# If not set this, by default master is hostname but not ip,
"SPARK_MASTER_HOST": node_ip
}
# The scripts installed from pip don't have execution permission
# and need to first give them permission.
pro = subprocess.Popen(
["chmod", "-R", "+x", "{}/sbin".format(zoo_standalone_home)])
os.waitpid(pro.pid, 0)
# Start master
start_master_pro = subprocess.Popen(
"{}/sbin/start-master.sh".format(zoo_standalone_home),
shell=True,
env=SparkRunner.standalone_env)
os.waitpid(start_master_pro.pid, 0)
master = "spark://{}:7077".format(
node_ip) # 7077 is the default port
# Start worker
start_worker_pro = subprocess.Popen(
"{}/sbin/start-worker.sh {}".format(zoo_standalone_home,
master),
shell=True,
env=SparkRunner.standalone_env)
os.waitpid(start_worker_pro.pid, 0)
else: # A Spark standalone cluster has already been started by the user.
assert master.startswith("spark://"), \
"Please input a valid master address for your Spark standalone cluster: " \
"spark://master:port"
# Start pyspark-shell
submit_args = " --master " + master
submit_args = submit_args + " --driver-cores {} --driver-memory {} --num-executors {}" \
" --executor-cores {} --executor-memory {}"\
.format(driver_cores, driver_memory, num_executors, executor_cores, executor_memory)
if extra_python_lib:
submit_args = submit_args + " --py-files {}".format(
extra_python_lib)
if jars:
submit_args = submit_args + " --jars {}".format(jars)
submit_args = submit_args + " pyspark-shell"
os.environ['PYSPARK_SUBMIT_ARGS'] = submit_args
zoo_bigdl_jar_path = ":".join(
[get_analytics_zoo_classpath(),
get_bigdl_classpath()])
spark_conf = init_spark_conf(conf) \
.set("spark.driver.cores", driver_cores) \
.set("spark.driver.memory", driver_memory) \
.set("spark.executor.instances", num_executors) \
.set("spark.executor.cores", executor_cores) \
.set("spark.cores.max", num_executors * executor_cores) \
.set("spark.executorEnv.PYTHONHOME",
"/".join(self._detect_python_location().split("/")[:-2]))
if extra_executor_memory_for_ray:
spark_conf.set("spark.executor.memoryOverhead",
extra_executor_memory_for_ray)
if spark_conf.contains("spark.executor.extraClassPath"):
spark_conf.set(
"spark.executor.extraClassPath",
"{}:{}".format(zoo_bigdl_jar_path,
conf.get("spark.executor.extraClassPath")))
else:
spark_conf.set("spark.executor.extraClassPath", zoo_bigdl_jar_path)
sc = init_nncontext(spark_conf,
redirect_spark_log=self.redirect_spark_log)
sc.setLogLevel(self.spark_log_level)
return sc
