def get_conf_param(self, param_name, default=None, node=None): # Copy conf files from first host in the cluster if node is None: node = self.hosts[0] temp_conf_files = self._get_conf_files(node) # Look in conf files for f in temp_conf_files: value = read_param_in_props_file(f, param_name) if value: return value return default
def _create_master_and_slave_conf(self, conf_dir): """Configure master and create slaves configuration files.""" defs_file = conf_dir + "/spark-defaults.conf" # Configure master spark_master = read_param_in_props_file(defs_file, "spark.master") if spark_master and spark_master.startswith("local"): logger.warn("Your default configuration executes Spark locally. " "Note that unless otherwise specified when launching " "your scripts, the distributed configuration will be " "ignored.") else: if self.mode == STANDALONE_MODE: # Always override? spark_master = "spark://%s:%d" % (self.master.address, self.port) elif self.mode == YARN_MODE: if spark_master: if spark_master not in ["yarn-client", "yarn-cluster"]: logger.warn("Provided spark.master is not compatible " "with YARN mode. Overriding with " "'yarn-client'") spark_master = "yarn-client" else: spark_master = "yarn-client" write_in_props_file(defs_file, "spark.master", spark_master, create_if_absent=True, override=True) # Configure slaves with open(conf_dir + "/slaves", "w") as slaves_file: for s in self.hosts: slaves_file.write(s.address + "\n")
def _get_cluster_params(self, conf_dir, default_tuning=False): """Define host-dependant parameters. Args: conf_dir (str): The path of the directory with the configuration files. default_tuning (bool, optional): Whether to use automatic tuning based on some best practices or leave the default parameters. """ defs_file = conf_dir + "/spark-defaults.conf" params = {} min_exec_mem = self.hw.get_max_memory_cluster().get_memory() total_execs = 0 for cluster in self.hw.get_clusters(): num_cores = cluster.get_num_cores() spark_cores = min(1, num_cores - 2) available_mem = cluster.get_memory() if self.mode == STANDALONE_MODE: spark_mem = available_mem elif self.mode == YARN_MODE: total_conts_mem = int(self.hc.get_conf_param( "yarn.nodemanager.resource.memory-mb", 8192, # Default value in YARN node=cluster.get_hosts()[0] )) spark_mem = total_conts_mem # Ideally: 5 threads / executor execs_per_node = max(1, spark_cores / 5) # Split memory and consider overhead exec_mem = (spark_mem / execs_per_node) mem_overhead = int(read_param_in_props_file( defs_file, "spark.yarn.executor.memoryOverhead", max(385, 0.1 * exec_mem))) if exec_mem < mem_overhead: error_msg = "Not enough memory to use the specified memory " \ "overhead: overhead = %d, executor memory = %d" \ % (mem_overhead, exec_mem) logger.error(error_msg) raise SparkConfException(error_msg) exec_mem -= mem_overhead min_exec_mem = min(min_exec_mem, exec_mem) # Accumulate executors total_execs += len(cluster.get_hosts()) * execs_per_node params[cluster.get_name()] = {} # Global parameters params["global"] = { "exec_mem": min_exec_mem, "exec_cores": 5, "total_execs": total_execs } return params