def get_open_ports(self, node_group):
cluster = node_group.cluster
ports_map = {
'namenode': [8020, 50070, 50470],
'datanode': [50010, 1004, 50075, 1006, 50020],
'master': [
int(utils.get_config_value_or_default("Spark", "Master port",
cluster)),
int(utils.get_config_value_or_default("Spark",
"Master webui port",
cluster)),
],
'slave': [
int(utils.get_config_value_or_default("Spark",
"Worker webui port",
cluster))
],
'zeppelin': [int(utils.get_config_value_or_default("Zeppelin",
"Web UI port",
cluster))]
}
ports = []
for process in node_group.node_processes:
if process in ports_map:
ports.extend(ports_map[process])
return ports
def __init__(self, cluster):
super(EdpEngine, self).__init__(cluster)
self.master = plugin_utils.get_instance(cluster, "master")
self.plugin_params["spark-user"] = ""
self.plugin_params["spark-submit"] = os.path.join(
plugin_utils.get_config_value_or_default("Spark", "Spark home", self.cluster), "bin/spark-submit"
)
self.plugin_params["deploy-mode"] = "client"
port_str = six.text_type(plugin_utils.get_config_value_or_default("Spark", "Master port", self.cluster))
self.plugin_params["master"] = "spark://%(host)s:" + port_str
driver_cp = plugin_utils.get_config_value_or_default("Spark", "Executor extra classpath", self.cluster)
self.plugin_params["driver-class-path"] = driver_cp
def generate_zk_basic_config(cluster):
args = {
'ticktime':
utils.get_config_value_or_default("ZooKeeper", "tickTime", cluster),
'initlimit':
utils.get_config_value_or_default("ZooKeeper", "initLimit", cluster),
'synclimit':
utils.get_config_value_or_default("ZooKeeper", "syncLimit", cluster)
}
zoo_cfg = f.get_file_text(
'plugins/vanilla/hadoop2/resources/zoo_sample.cfg')
return zoo_cfg.format(**args)
def __init__(self, cluster):
super(EdpSparkEngine, self).__init__(cluster)
self.master = plugin_utils.get_instance(cluster,
"spark history server")
self.plugin_params["spark-user"] = "******"
self.plugin_params["spark-submit"] = os.path.join(
plugin_utils.get_config_value_or_default("Spark", "Spark home",
self.cluster),
"bin/spark-submit")
self.plugin_params["deploy-mode"] = "cluster"
self.plugin_params["master"] = "yarn"
driver_cp = plugin_utils.get_config_value_or_default(
"Spark", "Executor extra classpath", self.cluster)
self.plugin_params["driver-class-path"] = driver_cp
def generate_spark_executor_classpath(cluster):
cp = utils.get_config_value_or_default("Spark",
"Executor extra classpath",
cluster)
if cp:
return "spark.executor.extraClassPath " + cp
return "\n"
def _validate_existing_ng_scaling(self, cluster, existing):
scalable_processes = self._get_scalable_processes()
dn_to_delete = 0
for ng in cluster.node_groups:
if ng.id in existing:
if ng.count > existing[ng.id] and ("datanode"
in ng.node_processes):
dn_to_delete += ng.count - existing[ng.id]
if not set(ng.node_processes).issubset(scalable_processes):
raise ex.NodeGroupCannotBeScaled(
ng.name,
_("Spark plugin cannot scale nodegroup"
" with processes: %s") % ' '.join(ng.node_processes))
dn_amount = len(utils.get_instances(cluster, "datanode"))
rep_factor = utils.get_config_value_or_default('HDFS',
"dfs.replication",
cluster)
if dn_to_delete > 0 and dn_amount - dn_to_delete < rep_factor:
raise ex.ClusterCannotBeScaled(
cluster.name,
_("Spark plugin cannot shrink cluster because "
"there would be not enough nodes for HDFS "
"replicas (replication factor is %s)") % rep_factor)
def validate(self, cluster):
if cluster.hadoop_version == "1.0.0":
raise exceptions.DeprecatedException(
_("Support for Spark version 1.0.0 is now deprecated and will" " be removed in the 2016.1 release.")
)
nn_count = sum([ng.count for ng in utils.get_node_groups(cluster, "namenode")])
if nn_count != 1:
raise ex.InvalidComponentCountException("namenode", 1, nn_count)
dn_count = sum([ng.count for ng in utils.get_node_groups(cluster, "datanode")])
if dn_count < 1:
raise ex.InvalidComponentCountException("datanode", _("1 or more"), nn_count)
rep_factor = utils.get_config_value_or_default("HDFS", "dfs.replication", cluster)
if dn_count < rep_factor:
raise ex.InvalidComponentCountException(
"datanode",
_("%s or more") % rep_factor,
dn_count,
_("Number of %(dn)s instances should not be less " "than %(replication)s")
% {"dn": "datanode", "replication": "dfs.replication"},
)
# validate Spark Master Node and Spark Slaves
sm_count = sum([ng.count for ng in utils.get_node_groups(cluster, "master")])
if sm_count != 1:
raise ex.RequiredServiceMissingException("Spark master")
sl_count = sum([ng.count for ng in utils.get_node_groups(cluster, "slave")])
if sl_count < 1:
raise ex.InvalidComponentCountException("Spark slave", _("1 or more"), sl_count)
def __init__(self, cluster):
super(EdpEngine, self).__init__(cluster)
self.master = plugin_utils.get_instance(cluster, "master")
self.plugin_params["spark-user"] = ""
self.plugin_params["spark-submit"] = os.path.join(
plugin_utils.
get_config_value_or_default("Spark", "Spark home", self.cluster),
"bin/spark-submit")
self.plugin_params["deploy-mode"] = "client"
port_str = six.text_type(
plugin_utils.get_config_value_or_default(
"Spark", "Master port", self.cluster))
self.plugin_params["master"] = ('spark://%(host)s:' + port_str)
driver_cp = plugin_utils.get_config_value_or_default(
"Spark", "Executor extra classpath", self.cluster)
self.plugin_params["driver-class-path"] = driver_cp
def _validate_existing_ng_scaling(self, cluster, existing):
scalable_processes = self._get_scalable_processes()
dn_to_delete = 0
for ng in cluster.node_groups:
if ng.id in existing:
if ng.count > existing[ng.id] and ("datanode" in
ng.node_processes):
dn_to_delete += ng.count - existing[ng.id]
if not set(ng.node_processes).issubset(scalable_processes):
raise ex.NodeGroupCannotBeScaled(
ng.name, _("Spark plugin cannot scale nodegroup"
" with processes: %s") %
' '.join(ng.node_processes))
dn_amount = len(utils.get_instances(cluster, "datanode"))
rep_factor = utils.get_config_value_or_default('HDFS',
"dfs.replication",
cluster)
if dn_to_delete > 0 and dn_amount - dn_to_delete < rep_factor:
raise ex.ClusterCannotBeScaled(
cluster.name, _("Spark plugin cannot shrink cluster because "
"there would be not enough nodes for HDFS "
"replicas (replication factor is %s)") %
rep_factor)
def generate_spark_executor_classpath(cluster):
cp = utils.get_config_value_or_default("Spark",
"Executor extra classpath",
cluster)
if cp:
return "spark.executor.extraClassPath " + cp
return "\n"
def _set_cluster_info(self, cluster):
nn = utils.get_instance(cluster, "namenode")
sp_master = utils.get_instance(cluster, "master")
info = {}
if nn:
address = utils.get_config_value_or_default("HDFS", "dfs.http.address", cluster)
port = address[address.rfind(":") + 1 :]
info["HDFS"] = {"Web UI": "http://%s:%s" % (nn.management_ip, port)}
info["HDFS"]["NameNode"] = "hdfs://%s:8020" % nn.hostname()
if sp_master:
port = utils.get_config_value_or_default("Spark", "Master webui port", cluster)
if port is not None:
info["Spark"] = {"Web UI": "http://%s:%s" % (sp_master.management_ip, port)}
ctx = context.ctx()
conductor.cluster_update(ctx, cluster, {"info": info})
def __init__(self, cluster):
super(EdpSparkEngine, self).__init__(cluster)
self.master = u.get_instance(cluster, "SPARK_YARN_HISTORY_SERVER")
self.plugin_params["spark-user"] = "******"
self.plugin_params["spark-submit"] = "spark-submit"
self.plugin_params["deploy-mode"] = "cluster"
self.plugin_params["master"] = "yarn-cluster"
driver_cp = u.get_config_value_or_default("Spark", "Executor extra classpath", self.cluster)
self.plugin_params["driver-class-path"] = driver_cp
def generate_spark_env_configs(cluster):
configs = []
# master configuration
sp_master = utils.get_instance(cluster, "master")
configs.append('SPARK_MASTER_IP=' + sp_master.hostname())
# point to the hadoop conf dir so that Spark can read things
# like the swift configuration without having to copy core-site
# to /opt/spark/conf
configs.append('HADOOP_CONF_DIR=' + HADOOP_CONF_DIR)
masterport = utils.get_config_value_or_default("Spark",
"Master port",
cluster)
if masterport and masterport != _get_spark_opt_default("Master port"):
configs.append('SPARK_MASTER_PORT=' + str(masterport))
masterwebport = utils.get_config_value_or_default("Spark",
"Master webui port",
cluster)
if (masterwebport and
masterwebport != _get_spark_opt_default("Master webui port")):
configs.append('SPARK_MASTER_WEBUI_PORT=' + str(masterwebport))
# configuration for workers
workercores = utils.get_config_value_or_default("Spark",
"Worker cores",
cluster)
if workercores and workercores != _get_spark_opt_default("Worker cores"):
configs.append('SPARK_WORKER_CORES=' + str(workercores))
workermemory = utils.get_config_value_or_default("Spark",
"Worker memory",
cluster)
if (workermemory and
workermemory != _get_spark_opt_default("Worker memory")):
configs.append('SPARK_WORKER_MEMORY=' + str(workermemory))
workerport = utils.get_config_value_or_default("Spark",
"Worker port",
cluster)
if workerport and workerport != _get_spark_opt_default("Worker port"):
configs.append('SPARK_WORKER_PORT=' + str(workerport))
workerwebport = utils.get_config_value_or_default("Spark",
"Worker webui port",
cluster)
if (workerwebport and
workerwebport != _get_spark_opt_default("Worker webui port")):
configs.append('SPARK_WORKER_WEBUI_PORT=' + str(workerwebport))
workerinstances = utils.get_config_value_or_default("Spark",
"Worker instances",
cluster)
if (workerinstances and
workerinstances != _get_spark_opt_default("Worker instances")):
configs.append('SPARK_WORKER_INSTANCES=' + str(workerinstances))
return '\n'.join(configs)
def generate_spark_env_configs(cluster):
configs = []
# master configuration
sp_master = utils.get_instance(cluster, "master")
configs.append('SPARK_MASTER_IP=' + sp_master.hostname())
# point to the hadoop conf dir so that Spark can read things
# like the swift configuration without having to copy core-site
# to /opt/spark/conf
configs.append('HADOOP_CONF_DIR=' + HADOOP_CONF_DIR)
masterport = utils.get_config_value_or_default("Spark",
"Master port",
cluster)
if masterport and masterport != _get_spark_opt_default("Master port"):
configs.append('SPARK_MASTER_PORT=' + str(masterport))
masterwebport = utils.get_config_value_or_default("Spark",
"Master webui port",
cluster)
if (masterwebport and
masterwebport != _get_spark_opt_default("Master webui port")):
configs.append('SPARK_MASTER_WEBUI_PORT=' + str(masterwebport))
# configuration for workers
workercores = utils.get_config_value_or_default("Spark",
"Worker cores",
cluster)
if workercores and workercores != _get_spark_opt_default("Worker cores"):
configs.append('SPARK_WORKER_CORES=' + str(workercores))
workermemory = utils.get_config_value_or_default("Spark",
"Worker memory",
cluster)
if (workermemory and
workermemory != _get_spark_opt_default("Worker memory")):
configs.append('SPARK_WORKER_MEMORY=' + str(workermemory))
workerport = utils.get_config_value_or_default("Spark",
"Worker port",
cluster)
if workerport and workerport != _get_spark_opt_default("Worker port"):
configs.append('SPARK_WORKER_PORT=' + str(workerport))
workerwebport = utils.get_config_value_or_default("Spark",
"Worker webui port",
cluster)
if (workerwebport and
workerwebport != _get_spark_opt_default("Worker webui port")):
configs.append('SPARK_WORKER_WEBUI_PORT=' + str(workerwebport))
workerinstances = utils.get_config_value_or_default("Spark",
"Worker instances",
cluster)
if (workerinstances and
workerinstances != _get_spark_opt_default("Worker instances")):
configs.append('SPARK_WORKER_INSTANCES=' + str(workerinstances))
return '\n'.join(configs)
def generate_job_cleanup_config(cluster):
args = {
'minimum_cleanup_megabytes': utils.get_config_value_or_default(
"Spark", "Minimum cleanup megabytes", cluster),
'minimum_cleanup_seconds': utils.get_config_value_or_default(
"Spark", "Minimum cleanup seconds", cluster),
'maximum_cleanup_seconds': utils.get_config_value_or_default(
"Spark", "Maximum cleanup seconds", cluster)
}
job_conf = {'valid': (args['maximum_cleanup_seconds'] > 0 and
(args['minimum_cleanup_megabytes'] > 0
and args['minimum_cleanup_seconds'] > 0))}
if job_conf['valid']:
job_conf['cron'] = f.get_file_text(
'plugins/spark/resources/spark-cleanup.cron'),
job_cleanup_script = f.get_file_text(
'plugins/spark/resources/tmp-cleanup.sh.template')
job_conf['script'] = job_cleanup_script.format(**args)
return job_conf
def generate_job_cleanup_config(cluster):
args = {
'minimum_cleanup_megabytes': utils.get_config_value_or_default(
"Spark", "Minimum cleanup megabytes", cluster),
'minimum_cleanup_seconds': utils.get_config_value_or_default(
"Spark", "Minimum cleanup seconds", cluster),
'maximum_cleanup_seconds': utils.get_config_value_or_default(
"Spark", "Maximum cleanup seconds", cluster)
}
job_conf = {'valid': (args['maximum_cleanup_seconds'] > 0 and
(args['minimum_cleanup_megabytes'] > 0
and args['minimum_cleanup_seconds'] > 0))}
if job_conf['valid']:
job_conf['cron'] = f.get_file_text(
'plugins/spark/resources/spark-cleanup.cron'),
job_cleanup_script = f.get_file_text(
'plugins/spark/resources/tmp-cleanup.sh.template')
job_conf['script'] = job_cleanup_script.format(**args)
return job_conf
def __init__(self, cluster):
super(EdpSparkEngine, self).__init__(cluster)
self.master = u.get_instance(cluster, "SPARK_YARN_HISTORY_SERVER")
self.plugin_params["spark-user"] = "******"
self.plugin_params["spark-submit"] = "spark-submit"
self.plugin_params["deploy-mode"] = "cluster"
self.plugin_params["master"] = "yarn-cluster"
driver_cp = u.get_config_value_or_default(
"Spark", "Executor extra classpath", self.cluster)
self.plugin_params["driver-class-path"] = driver_cp
def __init__(self, cluster):
super(EdpSparkEngine, self).__init__(cluster)
self.master = u.get_instance(cluster, "CLOUDERA_MANAGER")
self.plugin_params["spark-user"] = "******"
self.plugin_params["spark-submit"] = "spark-submit"
self.plugin_params["deploy-mode"] = "cluster"
self.plugin_params["master"] = "yarn-cluster"
driver_cp = u.get_config_value_or_default(
"Spark", "Executor extra classpath", self.cluster)
if driver_cp:
driver_cp = " --driver-class-path " + driver_cp
self.plugin_params["driver-class-path"] = driver_cp
def generate_zeppelin_setup_script(sp_master):
script_lines = ["#!/bin/bash -x"]
script_lines.append(
"echo 'export MASTER=spark://{0}:{1}' "
">> /opt/incubator-zeppelin/conf/zeppelin-env.sh".format(
sp_master['instance_name'],
utils.get_config_value_or_default(
"Spark", "Master port", sp_master.node_group.cluster)))
script_lines.append("echo 'export SPARK_HOME=/opt/spark' >> "
"/opt/incubator-zeppelin/conf/zeppelin-env.sh")
script_lines.append("echo 'export PYTHONPATH=$SPARK_HOME/python:"
"$SPARK_HOME/python/lib/py4j-0.8.2.1-src.zip:"
"$PYTHONPATH' >> "
"/opt/incubator-zeppelin/conf/zeppelin-env.sh")
script_lines.append("sed -i 's|<value>8080</value>|<value>{0}</value>|g'"
" /opt/incubator-zeppelin/conf/zeppelin-site.xml"
.format(utils.get_config_value_or_default(
"Zeppelin",
"Web UI port",
sp_master.node_group.cluster)))
return "\n".join(script_lines)
def __init__(self, cluster):
super(EdpSparkEngine, self).__init__(cluster)
self.master = u.get_instance(cluster, "CLOUDERA_MANAGER")
self.plugin_params["spark-user"] = "******"
self.plugin_params["spark-submit"] = "spark-submit"
self.plugin_params["deploy-mode"] = "cluster"
self.plugin_params["master"] = "yarn-cluster"
driver_cp = u.get_config_value_or_default("Spark",
"Executor extra classpath",
self.cluster)
if driver_cp:
driver_cp = " --driver-class-path " + driver_cp
self.plugin_params["driver-class-path"] = driver_cp
def _set_cluster_info(self, cluster):
nn = utils.get_instance(cluster, "namenode")
sp_master = utils.get_instance(cluster, "master")
info = {}
if nn:
address = utils.get_config_value_or_default(
'HDFS', 'dfs.http.address', cluster)
port = address[address.rfind(':') + 1:]
info['HDFS'] = {
'Web UI': 'http://%s:%s' % (nn.management_ip, port)
}
info['HDFS']['NameNode'] = 'hdfs://%s:8020' % nn.hostname()
if sp_master:
port = utils.get_config_value_or_default(
'Spark', 'Master webui port', cluster)
if port is not None:
info['Spark'] = {
'Web UI': 'http://%s:%s' % (sp_master.management_ip, port)
}
ctx = context.ctx()
conductor.cluster_update(ctx, cluster, {'info': info})
def _set_cluster_info(self, cluster):
nn = utils.get_instance(cluster, "namenode")
sp_master = utils.get_instance(cluster, "master")
info = {}
if nn:
address = utils.get_config_value_or_default(
'HDFS', 'dfs.http.address', cluster)
port = address[address.rfind(':') + 1:]
info['HDFS'] = {
'Web UI': 'http://%s:%s' % (nn.management_ip, port)
}
info['HDFS']['NameNode'] = 'hdfs://%s:8020' % nn.hostname()
if sp_master:
port = utils.get_config_value_or_default('Spark',
'Master webui port',
cluster)
if port is not None:
info['Spark'] = {
'Web UI': 'http://%s:%s' % (sp_master.management_ip, port)
}
ctx = context.ctx()
conductor.cluster_update(ctx, cluster, {'info': info})
def validate(self, cluster):
if cluster.hadoop_version == "1.0.0":
raise exceptions.DeprecatedException(
_("Support for Spark version 1.0.0 is now deprecated and will"
" be removed in the 2016.1 release."))
nn_count = sum(
[ng.count for ng in utils.get_node_groups(cluster, "namenode")])
if nn_count != 1:
raise ex.InvalidComponentCountException("namenode", 1, nn_count)
dn_count = sum(
[ng.count for ng in utils.get_node_groups(cluster, "datanode")])
if dn_count < 1:
raise ex.InvalidComponentCountException("datanode", _("1 or more"),
nn_count)
rep_factor = utils.get_config_value_or_default('HDFS',
"dfs.replication",
cluster)
if dn_count < rep_factor:
raise ex.InvalidComponentCountException(
'datanode',
_('%s or more') % rep_factor, dn_count,
_('Number of %(dn)s instances should not be less '
'than %(replication)s') % {
'dn': 'datanode',
'replication': 'dfs.replication'
})
# validate Spark Master Node and Spark Slaves
sm_count = sum(
[ng.count for ng in utils.get_node_groups(cluster, "master")])
if sm_count != 1:
raise ex.RequiredServiceMissingException("Spark master")
sl_count = sum(
[ng.count for ng in utils.get_node_groups(cluster, "slave")])
if sl_count < 1:
raise ex.InvalidComponentCountException("Spark slave",
_("1 or more"), sl_count)
def validate(self, cluster):
nn_count = sum(
[ng.count for ng in utils.get_node_groups(cluster, "namenode")])
if nn_count != 1:
raise ex.InvalidComponentCountException("namenode", 1, nn_count)
dn_count = sum(
[ng.count for ng in utils.get_node_groups(cluster, "datanode")])
if dn_count < 1:
raise ex.InvalidComponentCountException("datanode", _("1 or more"),
nn_count)
rep_factor = utils.get_config_value_or_default('HDFS',
"dfs.replication",
cluster)
if dn_count < rep_factor:
raise ex.InvalidComponentCountException(
'datanode',
_('%s or more') % rep_factor, dn_count,
_('Number of %(dn)s instances should not be less '
'than %(replication)s') % {
'dn': 'datanode',
'replication': 'dfs.replication'
})
# validate Spark Master Node and Spark Slaves
sm_count = sum(
[ng.count for ng in utils.get_node_groups(cluster, "master")])
if sm_count < 1:
raise ex.RequiredServiceMissingException("Spark master")
if sm_count >= 2:
raise ex.InvalidComponentCountException("Spark master", "1",
sm_count)
sl_count = sum(
[ng.count for ng in utils.get_node_groups(cluster, "slave")])
if sl_count < 1:
raise ex.InvalidComponentCountException("Spark slave",
_("1 or more"), sl_count)
def decommission_sl(master, inst_to_be_deleted, survived_inst):
if survived_inst is not None:
slavenames = []
for slave in survived_inst:
slavenames.append(slave.hostname())
slaves_content = c_helper.generate_spark_slaves_configs(slavenames)
else:
slaves_content = "\n"
cluster = master.cluster
sp_home = utils.get_config_value_or_default("Spark", "Spark home", cluster)
r_master = remote.get_remote(master)
run.stop_spark(r_master, sp_home)
# write new slave file to master
files = {os.path.join(sp_home, 'conf/slaves'): slaves_content}
r_master.write_files_to(files)
# write new slaves file to each survived slave as well
for i in survived_inst:
with remote.get_remote(i) as r:
r.write_files_to(files)
run.start_spark_master(r_master, sp_home)
def decommission_sl(master, inst_to_be_deleted, survived_inst):
if survived_inst is not None:
slavenames = []
for slave in survived_inst:
slavenames.append(slave.hostname())
slaves_content = c_helper.generate_spark_slaves_configs(slavenames)
else:
slaves_content = "\n"
cluster = master.cluster
sp_home = utils.get_config_value_or_default("Spark", "Spark home", cluster)
r_master = utils.get_remote(master)
run.stop_spark(r_master, sp_home)
# write new slave file to master
files = {os.path.join(sp_home, 'conf/slaves'): slaves_content}
r_master.write_files_to(files)
# write new slaves file to each survived slave as well
for i in survived_inst:
with utils.get_remote(i) as r:
r.write_files_to(files)
run.start_spark_master(r_master, sp_home)
def validate(self, cluster):
nn_count = sum([ng.count for ng
in utils.get_node_groups(cluster, "namenode")])
if nn_count != 1:
raise ex.InvalidComponentCountException("namenode", 1, nn_count)
dn_count = sum([ng.count for ng
in utils.get_node_groups(cluster, "datanode")])
if dn_count < 1:
raise ex.InvalidComponentCountException("datanode", _("1 or more"),
nn_count)
rep_factor = utils.get_config_value_or_default('HDFS',
"dfs.replication",
cluster)
if dn_count < rep_factor:
raise ex.InvalidComponentCountException(
'datanode', _('%s or more') % rep_factor, dn_count,
_('Number of %(dn)s instances should not be less '
'than %(replication)s')
% {'dn': 'datanode', 'replication': 'dfs.replication'})
# validate Spark Master Node and Spark Slaves
sm_count = sum([ng.count for ng
in utils.get_node_groups(cluster, "master")])
if sm_count != 1:
raise ex.RequiredServiceMissingException("Spark master")
sl_count = sum([ng.count for ng
in utils.get_node_groups(cluster, "slave")])
if sl_count < 1:
raise ex.InvalidComponentCountException("Spark slave",
_("1 or more"),
sl_count)
def get_admin_password(cluster):
# TODO(vgridnev): support in follow-up improved secret storage for
# configs
return pl_utils.get_config_value_or_default(
cluster=cluster, config=admin_password)
def get_policy_url(cluster):
return pl_utils.get_config_value_or_default(
cluster=cluster, config=policy_url)
def get_realm_name(cluster):
return pl_utils.get_config_value_or_default(
cluster=cluster, config=realm_name)
def get_admin_principal(cluster):
return pl_utils.get_config_value_or_default(
cluster=cluster, config=admin_principal)
def is_kerberos_security_enabled(cluster):
return pl_utils.get_config_value_or_default(cluster=cluster,
config=enable_kerberos)
def _spark_home(self, cluster):
return utils.get_config_value_or_default("Spark",
"Spark home",
cluster)
def get_realm_name(cluster):
return pl_utils.get_config_value_or_default(cluster=cluster,
config=realm_name)
def _spark_home(self, cluster):
return utils.get_config_value_or_default("Spark", "Spark home",
cluster)
def get_admin_password(cluster):
# TODO(vgridnev): support in follow-up improved secret storage for
# configs
return pl_utils.get_config_value_or_default(cluster=cluster,
config=admin_password)
def get_admin_principal(cluster):
return pl_utils.get_config_value_or_default(cluster=cluster,
config=admin_principal)
def get_policy_url(cluster):
return pl_utils.get_config_value_or_default(cluster=cluster,
config=policy_url)
def generate_spark_env_configs(cluster):
configs = []
# newly added: master configuration
sp_master = utils.get_instance(cluster, "master")
configs.append('SPARK_MASTER_IP=' + sp_master.hostname())
# point to the hadoop conf dir so that Spark can read things
# like the swift configuration without having to copy core-site
# to /opt/spark/conf
HADOOP_CONF_DIR = '/opt/hadoop/etc/hadoop'
configs.append('HADOOP_CONF_DIR=' + HADOOP_CONF_DIR)
# 获取Master Port的用户配置信息
masterport = utils.get_config_value_or_default("Spark", "Master port",
cluster)
if masterport and masterport != _get_spark_opt_default("Master port"):
configs.append('SPARK_MASTER_PORT=' + str(masterport))
# 获取Master Web Port的用户配置信息
masterwebport = utils.get_config_value_or_default("Spark",
"Master webui port",
cluster)
if (masterwebport
and masterwebport != _get_spark_opt_default("Master webui port")):
configs.append('SPARK_MASTER_WEBUI_PORT=' + str(masterwebport))
# work节点的配置信息
workercores = utils.get_config_value_or_default("Spark", "Worker cores",
cluster)
if workercores and workercores != _get_spark_opt_default("Worker cores"):
configs.append('SPARK_WORKER_CORES=' + str(workercores))
workermemory = utils.get_config_value_or_default("Spark", "Worker memory",
cluster)
if workermemory and workermemory != _get_spark_opt_default(
"Worker memory"):
configs.append('SPARK_WORKER_MEMORY=' + str(workermemory))
workerport = utils.get_config_value_or_default("Spark", "Worker port",
cluster)
if workerport and workerport != _get_spark_opt_default("Worker port"):
configs.append('SPARK_WORKER_PORT=' + str(workerport))
workerwebport = utils.get_config_value_or_default("Spark",
"Worker webui port",
cluster)
if workerwebport and workerwebport != _get_spark_opt_default(
"Worker webui port"):
configs.append('SPARK_WORKER_WEBUI_PORT=' + str(workerwebport))
workerinstances = utils.get_config_value_or_default(
"Spark", "Worker instances", cluster)
if workerinstances and workerinstances != _get_spark_opt_default(
"Worker instances"):
configs.append('SPARK_WORKER_INSTANCES=' + str(workerinstances))
# Hadoop and YARN configs there are in one folder
configs.append('YARN_CONF_DIR=' + HADOOP_CONF_DIR)
return '\n'.join(configs)
def test_get_config_value_or_default(self, mock_plugins_base):
# no config
self.assertRaises(RuntimeError,
pu.get_config_value_or_default)
config = mock.Mock()
config.applicable_target = "service"
config.name = "name"
config.default_value = "default_value"
# cluster has the config
cluster = mock.Mock()
cluster.cluster_configs = {"service": {"name": "name"}}
cluster.plugin_name = "plugin_name"
cluster.hadoop_version = "hadoop_version"
res = pu.get_config_value_or_default(cluster=cluster,
config=config)
self.assertEqual("name", res)
# node group has the config
cluster.cluster_configs = {}
node_group1 = mock.Mock()
node_group2 = mock.Mock()
node_group1.configuration = mock.Mock(return_value={"service": {}})
node_group2.configuration = mock.Mock(
return_value={"service": {"name": "name"}})
cluster.node_groups = [node_group1, node_group2]
res = pu.get_config_value_or_default(cluster=cluster,
config=config)
self.assertEqual("name", res)
# cluster doesn't have the config, neither the node groups
# so it returns the default value
cluster.node_groups = []
res = pu.get_config_value_or_default(cluster=cluster,
config=config)
self.assertEqual("default_value", res)
# no config specified, but there's a config for the plugin
# with this service and name
mock_get_all_configs = mock.Mock(return_value=[config])
mock_plugin = mock.Mock()
mock_plugin.get_all_configs = mock_get_all_configs
mock_get_plugin = mock.Mock(return_value=mock_plugin)
mock_PLUGINS = mock.Mock()
mock_PLUGINS.get_plugin = mock_get_plugin
mock_plugins_base.PLUGINS = mock_PLUGINS
res = pu.get_config_value_or_default(cluster=cluster,
service="service", name="name")
self.assertEqual("default_value", res)
mock_get_plugin.assert_called_once_with("plugin_name")
mock_get_all_configs.assert_called_once_with("hadoop_version")
# no config especified and no existing config for this plugin
# with this service or name
cluster.plugin_name = "plugin_name2"
cluster.hadoop_version = "hadoop_version2"
self.assertRaises(RuntimeError,
pu.get_config_value_or_default, cluster=cluster,
service="newService", name="name")
mock_get_plugin.assert_called_with("plugin_name2")
self.assertEqual(2, mock_get_plugin.call_count)
mock_get_all_configs.assert_called_with("hadoop_version2")
self.assertEqual(2, mock_get_all_configs.call_count)
def is_kerberos_security_enabled(cluster):
return pl_utils.get_config_value_or_default(
cluster=cluster, config=enable_kerberos)
def get_port_from_config(service, name, cluster=None):
address = utils.get_config_value_or_default(service, name, cluster)
return utils.get_port_from_address(address)
def get_kdc_server_ip(cluster):
return pl_utils.get_config_value_or_default(cluster=cluster,
config=kdc_server_ip)
def using_existing_kdc(cluster):
return pl_utils.get_config_value_or_default(
cluster=cluster, config=use_existing_kdc)
def using_existing_kdc(cluster):
return pl_utils.get_config_value_or_default(cluster=cluster,
config=use_existing_kdc)
def get_kdc_server_ip(cluster):
return pl_utils.get_config_value_or_default(
cluster=cluster, config=kdc_server_ip)
def get_port_from_config(service, name, cluster=None):
address = utils.get_config_value_or_default(service, name, cluster)
return utils.get_port_from_address(address)
def test_get_config_value_or_default(self, mock_plugins_base):
# no config
self.assertRaises(RuntimeError, pu.get_config_value_or_default)
config = mock.Mock()
config.applicable_target = "service"
config.name = "name"
config.default_value = "default_value"
# cluster has the config
cluster = mock.Mock()
cluster.cluster_configs = {"service": {"name": "name"}}
cluster.plugin_name = "plugin_name"
cluster.hadoop_version = "hadoop_version"
res = pu.get_config_value_or_default(cluster=cluster, config=config)
self.assertEqual("name", res)
# node group has the config
cluster.cluster_configs = {}
node_group1 = mock.Mock()
node_group2 = mock.Mock()
node_group1.configuration = mock.Mock(return_value={"service": {}})
node_group2.configuration = mock.Mock(
return_value={"service": {
"name": "name"
}})
cluster.node_groups = [node_group1, node_group2]
res = pu.get_config_value_or_default(cluster=cluster, config=config)
self.assertEqual("name", res)
# cluster doesn't have the config, neither the node groups
# so it returns the default value
cluster.node_groups = []
res = pu.get_config_value_or_default(cluster=cluster, config=config)
self.assertEqual("default_value", res)
# no config specified, but there's a config for the plugin
# with this service and name
mock_get_all_configs = mock.Mock(return_value=[config])
mock_plugin = mock.Mock()
mock_plugin.get_all_configs = mock_get_all_configs
mock_get_plugin = mock.Mock(return_value=mock_plugin)
mock_PLUGINS = mock.Mock()
mock_PLUGINS.get_plugin = mock_get_plugin
mock_plugins_base.PLUGINS = mock_PLUGINS
res = pu.get_config_value_or_default(cluster=cluster,
service="service",
name="name")
self.assertEqual("default_value", res)
mock_get_plugin.assert_called_once_with("plugin_name")
mock_get_all_configs.assert_called_once_with("hadoop_version")
# no config especified and no existing config for this plugin
# with this service or name
cluster.plugin_name = "plugin_name2"
cluster.hadoop_version = "hadoop_version2"
self.assertRaises(RuntimeError,
pu.get_config_value_or_default,
cluster=cluster,
service="newService",
name="name")
mock_get_plugin.assert_called_with("plugin_name2")
self.assertEqual(2, mock_get_plugin.call_count)
mock_get_all_configs.assert_called_with("hadoop_version2")
self.assertEqual(2, mock_get_all_configs.call_count)
