def get_conf_param(self, param_name, default=None, node=None):
# Copy conf files from first host in the cluster
if node is None:
node = self.hosts[0]
temp_conf_files = self._get_conf_files(node)
# Look in conf files
for f in temp_conf_files:
value = read_param_in_props_file(f, param_name)
if value:
return value
return default
def _create_master_and_slave_conf(self, conf_dir):
"""Configure master and create slaves configuration files."""
defs_file = conf_dir + "/spark-defaults.conf"
# Configure master
spark_master = read_param_in_props_file(defs_file, "spark.master")
if spark_master and spark_master.startswith("local"):
logger.warn("Your default configuration executes Spark locally. "
"Note that unless otherwise specified when launching "
"your scripts, the distributed configuration will be "
"ignored.")
else:
if self.mode == STANDALONE_MODE:
# Always override?
spark_master = "spark://%s:%d" % (self.master.address,
self.port)
elif self.mode == YARN_MODE:
if spark_master:
if spark_master not in ["yarn-client", "yarn-cluster"]:
logger.warn("Provided spark.master is not compatible "
"with YARN mode. Overriding with "
"'yarn-client'")
spark_master = "yarn-client"
else:
spark_master = "yarn-client"
write_in_props_file(defs_file,
"spark.master", spark_master,
create_if_absent=True,
override=True)
# Configure slaves
with open(conf_dir + "/slaves", "w") as slaves_file:
for s in self.hosts:
slaves_file.write(s.address + "\n")
def _get_cluster_params(self, conf_dir, default_tuning=False):
"""Define host-dependant parameters.
Args:
conf_dir (str):
The path of the directory with the configuration files.
default_tuning (bool, optional):
Whether to use automatic tuning based on some best practices or
leave the default parameters.
"""
defs_file = conf_dir + "/spark-defaults.conf"
params = {}
min_exec_mem = self.hw.get_max_memory_cluster().get_memory()
total_execs = 0
for cluster in self.hw.get_clusters():
num_cores = cluster.get_num_cores()
spark_cores = min(1, num_cores - 2)
available_mem = cluster.get_memory()
if self.mode == STANDALONE_MODE:
spark_mem = available_mem
elif self.mode == YARN_MODE:
total_conts_mem = int(self.hc.get_conf_param(
"yarn.nodemanager.resource.memory-mb",
8192, # Default value in YARN
node=cluster.get_hosts()[0]
))
spark_mem = total_conts_mem
# Ideally: 5 threads / executor
execs_per_node = max(1, spark_cores / 5)
# Split memory and consider overhead
exec_mem = (spark_mem / execs_per_node)
mem_overhead = int(read_param_in_props_file(
defs_file,
"spark.yarn.executor.memoryOverhead",
max(385, 0.1 * exec_mem)))
if exec_mem < mem_overhead:
error_msg = "Not enough memory to use the specified memory " \
"overhead: overhead = %d, executor memory = %d" \
% (mem_overhead, exec_mem)
logger.error(error_msg)
raise SparkConfException(error_msg)
exec_mem -= mem_overhead
min_exec_mem = min(min_exec_mem, exec_mem)
# Accumulate executors
total_execs += len(cluster.get_hosts()) * execs_per_node
params[cluster.get_name()] = {}
# Global parameters
params["global"] = {
"exec_mem": min_exec_mem,
"exec_cores": 5,
"total_execs": total_execs
}
return params
