def _extract_configs_to_extra(self, cluster):
sp_master = utils.get_instance(cluster, "master")
sp_slaves = utils.get_instances(cluster, "slave")
extra = dict()
config_master = config_slaves = ''
if sp_master is not None:
config_master = c_helper.generate_spark_env_configs(cluster)
if sp_slaves is not None:
slavenames = []
for slave in sp_slaves:
slavenames.append(slave.hostname())
config_slaves = c_helper.generate_spark_slaves_configs(slavenames)
else:
config_slaves = "\n"
# Any node that might be used to run spark-submit will need
# these libs for swift integration
config_defaults = c_helper.generate_spark_executor_classpath(cluster)
extra['job_cleanup'] = c_helper.generate_job_cleanup_config(cluster)
extra['sp_master'] = config_master
extra['sp_slaves'] = config_slaves
extra['sp_defaults'] = config_defaults
if c_helper.is_data_locality_enabled(cluster):
topology_data = th.generate_topology_map(
cluster, CONF.enable_hypervisor_awareness)
extra['topology_data'] = "\n".join(
[k + " " + v for k, v in topology_data.items()]) + "\n"
return extra
def _configure_rack_awareness(self, cluster):
api = self.get_api_client(cluster)
topology = t_helper.generate_topology_map(
cluster, is_node_awareness=False)
for host in api.get_all_hosts():
host.rackId = topology[host.ipAddress]
host.put_host()
def _get_topology_data(cluster):
if not t_helper.is_data_locality_enabled():
return {}
LOG.warning("Node group awareness is not implemented in YARN yet "
"so enable_hypervisor_awareness set to False "
"explicitly")
return t_helper.generate_topology_map(cluster, is_node_awareness=False)
def configure_topology_data(pctx, cluster):
if config_helper.is_data_locality_enabled(pctx, cluster):
LOG.warning("Node group awareness is not implemented in YARN yet "
"so enable_hypervisor_awareness set to False explicitly")
tpl_map = th.generate_topology_map(cluster, is_node_awareness=False)
topology_data = "\n".join([k + " " + v
for k, v in tpl_map.items()]) + "\n"
for ng in cluster.node_groups:
for i in ng.instances:
i.remote().write_file_to(HADOOP_CONF_DIR + "/topology.data",
topology_data,
run_as_root=True)
def topology_map(self):
return th.generate_topology_map(self.cluster, self.is_node_aware)