def _push_spark_configs_to_node(cluster, extra):
spark_master = vu.get_spark_history_server(cluster)
if spark_master:
_push_spark_configs_to_existing_node(spark_master, cluster, extra)
_push_cleanup_job(spark_master, extra)
with spark_master.remote() as r:
r.execute_command('sudo su - -c "mkdir /tmp/spark-events" hadoop')
def validate_job_execution(self, cluster, job, data):
if (not self.edp_supported(cluster.hadoop_version)
or not v_utils.get_spark_history_server(cluster)):
raise ex.PluginInvalidDataException(
_('Spark {base} or higher required to run {type} jobs').format(
base=EdpSparkEngine.edp_base_version, type=job.type))
super(EdpSparkEngine, self).validate_job_execution(cluster, job, data)
def _set_cluster_info(self, cluster):
nn = vu.get_namenode(cluster)
rm = vu.get_resourcemanager(cluster)
hs = vu.get_historyserver(cluster)
oo = vu.get_oozie(cluster)
sp = vu.get_spark_history_server(cluster)
info = {}
if rm:
info['YARN'] = {
'Web UI':
'http://%s:%s' % (rm.get_ip_or_dns_name(), '8088'),
'ResourceManager':
'http://%s:%s' % (rm.get_ip_or_dns_name(), '8032')
}
if nn:
info['HDFS'] = {
'Web UI': 'http://%s:%s' % (nn.get_ip_or_dns_name(), '50070'),
'NameNode': 'hdfs://%s:%s' % (nn.hostname(), '9000')
}
if oo:
info['JobFlow'] = {
'Oozie': 'http://%s:%s' % (oo.get_ip_or_dns_name(), '11000')
}
if hs:
info['MapReduce JobHistory Server'] = {
'Web UI': 'http://%s:%s' % (hs.get_ip_or_dns_name(), '19888')
}
if sp:
info['Apache Spark'] = {
'Spark UI':
'http://%s:%s' % (sp.management_ip, '4040'),
'Spark History Server UI':
'http://%s:%s' % (sp.management_ip, '18080')
}
ctx = context.ctx()
conductor.cluster_update(ctx, cluster, {'info': info})
def configure_spark(cluster):
spark_servers = vu.get_spark_history_server(cluster)
if spark_servers:
extra = _extract_spark_configs_to_extra(cluster)
_push_spark_configs_to_node(cluster, extra)
def start_spark(cluster):
spark = vu.get_spark_history_server(cluster)
if spark:
run.start_spark_history_server(spark)