def install_spark_standalone(zks, peers):
"""
Called in local/standalone mode after Juju has elected a leader.
"""
hosts = {
'spark-master': leadership.leader_get('master-fqdn'),
}
# If zks have changed and we are not handling a departed spark peer,
# give the ensemble time to settle. Otherwise we might try to start
# spark master with data from the wrong zk leader. Doing so will cause
# spark-master to shutdown:
# https://issues.apache.org/jira/browse/SPARK-15544
if (zks and data_changed('zks', zks)
and not is_state('sparkpeers.departed')):
hookenv.status_set('maintenance',
'waiting for zookeeper ensemble to settle')
hookenv.log(
"Waiting 2m to ensure zk ensemble has settled: {}".format(zks))
time.sleep(120)
# Let spark know if we have cuda libs installed.
# NB: spark packages prereq hadoop (boo), so even in standalone mode, we'll
# have hadoop libs installed. May as well include them in our lib path.
extra_libs = ["/usr/lib/hadoop/lib/native"]
if is_state('cuda.installed'):
extra_libs.append("/usr/local/cuda/lib64")
spark = Spark()
spark.configure(hosts, zk_units=zks, peers=peers, extra_libs=extra_libs)
set_deployment_mode_state('spark.standalone.installed')
def client_present(client):
if is_state('leadership.is_leader'):
client.set_spark_started()
spark = Spark()
master_ip = utils.resolve_private_address(hookenv.unit_private_ip())
master_url = spark.get_master_url(master_ip)
client.send_master_info(master_url, master_ip)
def install_spark_yarn():
"""
Called in 'yarn-*' mode after Juju has elected a leader. The
'hadoop.yarn.ready' state must be set.
"""
hosts = {
'spark-master': leadership.leader_get('master-fqdn'),
}
hadoop = (RelationBase.from_state('hadoop.yarn.ready')
or RelationBase.from_state('hadoop.hdfs.ready'))
rms = hadoop.resourcemanagers()
hosts['resourcemanager'] = rms[0]
# Probably don't need to check this since yarn.ready implies hdfs.ready
# for us, but it doesn't hurt.
if is_state('hadoop.hdfs.ready'):
nns = hadoop.namenodes()
hosts['namenode'] = nns[0]
# Always include native hadoop libs in yarn mode; add cuda libs if present.
extra_libs = ["/usr/lib/hadoop/lib/native"]
if is_state('cuda.installed'):
extra_libs.append("/usr/local/cuda/lib64")
spark = Spark()
spark.configure(hosts, zk_units=None, peers=None, extra_libs=extra_libs)
set_deployment_mode_state('spark.yarn.installed')
def install_spark_standalone(zks, peers):
"""
Called in local/standalone mode after Juju has elected a leader.
"""
hosts = {
'spark-master': leadership.leader_get('master-fqdn'),
}
# If zks have changed and we are not handling a departed spark peer,
# give the ensemble time to settle. Otherwise we might try to start
# spark master with data from the wrong zk leader. Doing so will cause
# spark-master to shutdown:
# https://issues.apache.org/jira/browse/SPARK-15544
if (zks and data_changed('zks', zks) and not is_state('sparkpeers.departed')):
hookenv.status_set('maintenance',
'waiting for zookeeper ensemble to settle')
hookenv.log("Waiting 2m to ensure zk ensemble has settled: {}".format(zks))
time.sleep(120)
# Let spark know if we have cuda libs installed.
# NB: spark packages prereq hadoop (boo), so even in standalone mode, we'll
# have hadoop libs installed. May as well include them in our lib path.
extra_libs = ["/usr/lib/hadoop/lib/native"]
if is_state('cuda.installed'):
extra_libs.append("/usr/local/cuda/lib64")
spark = Spark()
spark.configure(hosts, zk_units=zks, peers=peers, extra_libs=extra_libs)
set_deployment_mode_state('spark.standalone.installed')
def install_spark_yarn():
"""
Called in 'yarn-*' mode after Juju has elected a leader. The
'hadoop.yarn.ready' state must be set.
"""
hosts = {
'spark-master': leadership.leader_get('master-fqdn'),
}
hadoop = (RelationBase.from_state('hadoop.yarn.ready') or
RelationBase.from_state('hadoop.hdfs.ready'))
rms = hadoop.resourcemanagers()
hosts['resourcemanager'] = rms[0]
# Probably don't need to check this since yarn.ready implies hdfs.ready
# for us, but it doesn't hurt.
if is_state('hadoop.hdfs.ready'):
nns = hadoop.namenodes()
hosts['namenode'] = nns[0]
# Always include native hadoop libs in yarn mode; add cuda libs if present.
extra_libs = ["/usr/lib/hadoop/lib/native"]
if is_state('cuda.installed'):
extra_libs.append("/usr/local/cuda/lib64")
spark = Spark()
spark.configure(hosts, zk_units=None, peers=None, extra_libs=extra_libs)
set_deployment_mode_state('spark.yarn.installed')
def install_spark(hadoop=None):
spark_master_host = leadership.leader_get('master-fqdn')
hosts = {
'spark-master': spark_master_host,
}
if is_state('hadoop.yarn.ready'):
rms = hadoop.resourcemanagers()
hosts['resourcemanager'] = rms[0]
if is_state('hadoop.hdfs.ready'):
nns = hadoop.namenodes()
hosts['namenode'] = nns[0]
dist = get_dist_config()
spark = Spark(dist)
spark.configure(hosts)
def install_spark(hadoop=None, zks=None):
spark_master_host = leadership.leader_get('master-fqdn')
if not spark_master_host:
hookenv.status_set('waiting', 'master not elected yet')
return False
hosts = {
'spark-master': spark_master_host,
}
if is_state('hadoop.yarn.ready'):
rms = hadoop.resourcemanagers()
hosts['resourcemanager'] = rms[0]
if is_state('hadoop.hdfs.ready'):
nns = hadoop.namenodes()
hosts['namenode'] = nns[0]
spark = Spark()
spark.configure(hosts, zks, get_spark_peers())
return True
def install_spark_yarn():
"""
Called in 'yarn-*' mode after Juju has elected a leader. The
'hadoop.yarn.ready' state must be set.
"""
hosts = {
'spark-master': leadership.leader_get('master-fqdn'),
}
hadoop = (RelationBase.from_state('hadoop.yarn.ready') or
RelationBase.from_state('hadoop.hdfs.ready'))
rms = hadoop.resourcemanagers()
hosts['resourcemanager'] = rms[0]
# Probably don't need to check this since yarn.ready implies hdfs.ready
# for us, but it doesn't hurt.
if is_state('hadoop.hdfs.ready'):
nns = hadoop.namenodes()
hosts['namenode'] = nns[0]
spark = Spark()
spark.configure(hosts, zk_units=None, peers=None)
set_deployment_mode_state('spark.yarn.installed')
def install_spark_standalone(zks, peers):
"""
Called in local/standalone mode after Juju has elected a leader.
"""
hosts = {
'spark-master': leadership.leader_get('master-fqdn'),
}
# If zks have changed and we are not handling a departed spark peer,
# give the ensemble time to settle. Otherwise we might try to start
# spark master with data from the wrong zk leader. Doing so will cause
# spark-master to shutdown:
# https://issues.apache.org/jira/browse/SPARK-15544
if (zks and data_changed('zks', zks) and not is_state('sparkpeers.departed')):
hookenv.status_set('maintenance',
'waiting for zookeeper ensemble to settle')
hookenv.log("Waiting 2m to ensure zk ensemble has settled: {}".format(zks))
time.sleep(120)
spark = Spark()
spark.configure(hosts, zks, peers)
set_deployment_mode_state('spark.standalone.installed')
def reinstall_spark():
"""
This is tricky. We want to fire on config or leadership changes, or when
hadoop, sparkpeers, or zookeepers come and go. In the future this should
fire when Cassandra or any other storage comes or goes. We always fire
this method (or rather, when bigtop is ready and juju has elected a
master). We then build a deployment-matrix and (re)install as things
change.
"""
spark_master_host = leadership.leader_get('master-fqdn')
if not spark_master_host:
hookenv.status_set('maintenance', 'juju leader not elected yet')
return
mode = hookenv.config()['spark_execution_mode']
peers = None
zks = None
# If mode is standalone and ZK is ready, we are in HA. Do not consider
# the master_host from juju leadership in our matrix. ZK handles this.
if (mode == 'standalone' and is_state('zookeeper.ready')):
spark_master_host = ''
zk = RelationBase.from_state('zookeeper.ready')
zks = zk.zookeepers()
# peers are only used to set our MASTER_URL in standalone HA mode
peers = get_spark_peers()
deployment_matrix = {
'spark_master': spark_master_host,
'yarn_ready': is_state('hadoop.yarn.ready'),
'hdfs_ready': is_state('hadoop.hdfs.ready'),
'zookeepers': zks,
'peers': peers,
}
# If neither config nor our matrix is changing, there is nothing to do.
if not (is_state('config.changed') or
data_changed('deployment_matrix', deployment_matrix)):
return
# (Re)install based on our execution mode
hookenv.status_set('maintenance', 'configuring spark in {} mode'.format(mode))
hookenv.log("Configuring spark with deployment matrix: {}".format(deployment_matrix))
if mode.startswith('yarn') and is_state('hadoop.yarn.ready'):
install_spark_yarn()
elif mode.startswith('local') or mode == 'standalone':
install_spark_standalone(zks, peers)
else:
# Something's wrong (probably requested yarn without yarn.ready).
remove_state('spark.started')
report_status()
return
# restart services to pick up possible config changes
spark = Spark()
spark.stop()
spark.start()
set_state('spark.started')
report_status()
def reinstall_spark(force=False):
"""
Gather the state of our deployment and (re)install when leaders, hadoop,
sparkpeers, or zookeepers change. In the future this should also
fire when Cassandra or any other storage comes or goes. Config changed
events will also call this method, but that is invoked with a separate
handler below.
Use a deployment-matrix dict to track changes and (re)install as needed.
"""
spark_master_host = leadership.leader_get('master-fqdn')
if not spark_master_host:
hookenv.status_set('maintenance', 'juju leader not elected yet')
return
mode = hookenv.config()['spark_execution_mode']
peers = None
zks = None
# If mode is standalone and ZK is ready, we are in HA. Do not consider
# the master_host from juju leadership in our matrix. ZK handles this.
if (mode == 'standalone' and is_state('zookeeper.ready')):
spark_master_host = ''
zk = RelationBase.from_state('zookeeper.ready')
zks = zk.zookeepers()
# peers are only used to set our MASTER_URL in standalone HA mode
peers = get_spark_peers()
# Construct a deployment matrix
sample_data = hookenv.resource_get('sample-data')
deployment_matrix = {
'hdfs_ready': is_state('hadoop.hdfs.ready'),
'peers': peers,
'sample_data': host.file_hash(sample_data) if sample_data else None,
'spark_master': spark_master_host,
'yarn_ready': is_state('hadoop.yarn.ready'),
'zookeepers': zks,
}
# No-op if we are not forcing a reinstall or our matrix is unchanged.
if not (force or data_changed('deployment_matrix', deployment_matrix)):
report_status()
return
# (Re)install based on our execution mode
hookenv.status_set('maintenance', 'configuring spark in {} mode'.format(mode))
hookenv.log("Configuring spark with deployment matrix: {}".format(deployment_matrix))
if mode.startswith('yarn') and is_state('hadoop.yarn.ready'):
install_spark_yarn()
elif mode.startswith('local') or mode == 'standalone':
install_spark_standalone(zks, peers)
else:
# Something's wrong (probably requested yarn without yarn.ready).
remove_state('spark.started')
report_status()
return
# restart services to pick up possible config changes
spark = Spark()
spark.stop()
spark.start()
set_state('spark.started')
report_status()
def reinstall_spark(force=False):
"""
Gather the state of our deployment and (re)install when leaders, hadoop,
sparkpeers, or zookeepers change. In the future this should also
fire when Cassandra or any other storage comes or goes. Config changed
events will also call this method, but that is invoked with a separate
handler below.
Use a deployment-matrix dict to track changes and (re)install as needed.
"""
spark_master_host = leadership.leader_get('master-fqdn')
if not spark_master_host:
hookenv.status_set('maintenance', 'juju leader not elected yet')
return
mode = hookenv.config()['spark_execution_mode']
peers = None
zks = None
# If mode is standalone and ZK is ready, we are in HA. Do not consider
# the master_host from juju leadership in our matrix. ZK handles this.
if (mode == 'standalone' and is_state('zookeeper.ready')):
spark_master_host = ''
zk = RelationBase.from_state('zookeeper.ready')
zks = zk.zookeepers()
# peers are only used to set our MASTER_URL in standalone HA mode
peers = get_spark_peers()
# Construct a deployment matrix
sample_data = hookenv.resource_get('sample-data')
deployment_matrix = {
'hdfs_ready': is_state('hadoop.hdfs.ready'),
'peers': peers,
'sample_data': host.file_hash(sample_data) if sample_data else None,
'spark_master': spark_master_host,
'yarn_ready': is_state('hadoop.yarn.ready'),
'zookeepers': zks,
}
# No-op if we are not forcing a reinstall or our matrix is unchanged.
if not (force or data_changed('deployment_matrix', deployment_matrix)):
report_status()
return
# (Re)install based on our execution mode
hookenv.status_set('maintenance',
'configuring spark in {} mode'.format(mode))
hookenv.log("Configuring spark with deployment matrix: {}".format(
deployment_matrix))
if mode.startswith('yarn') and is_state('hadoop.yarn.ready'):
install_spark_yarn()
elif mode.startswith('local') or mode == 'standalone':
install_spark_standalone(zks, peers)
else:
# Something's wrong (probably requested yarn without yarn.ready).
remove_state('spark.started')
report_status()
return
# restart services to pick up possible config changes
spark = Spark()
spark.stop()
spark.start()
set_state('spark.started')
report_status()
