def install_spark_yarn(): """ Called in 'yarn-*' mode after Juju has elected a leader. The 'hadoop.yarn.ready' state must be set. """ hosts = { 'spark-master': leadership.leader_get('master-fqdn'), } hadoop = (RelationBase.from_state('hadoop.yarn.ready') or RelationBase.from_state('hadoop.hdfs.ready')) rms = hadoop.resourcemanagers() hosts['resourcemanager'] = rms[0] # Probably don't need to check this since yarn.ready implies hdfs.ready # for us, but it doesn't hurt. if is_state('hadoop.hdfs.ready'): nns = hadoop.namenodes() hosts['namenode'] = nns[0] # Always include native hadoop libs in yarn mode; add cuda libs if present. extra_libs = ["/usr/lib/hadoop/lib/native"] if is_state('cuda.installed'): extra_libs.append("/usr/local/cuda/lib64") spark = Spark() spark.configure(hosts, zk_units=None, peers=None, extra_libs=extra_libs) set_deployment_mode_state('spark.yarn.installed')
def client_present(client): if is_state('leadership.is_leader'): client.set_spark_started() spark = Spark() master_ip = utils.resolve_private_address(hookenv.unit_private_ip()) master_url = spark.get_master_url(master_ip) client.send_master_info(master_url, master_ip)
def install_spark_standalone(zks, peers): """ Called in local/standalone mode after Juju has elected a leader. """ hosts = { 'spark-master': leadership.leader_get('master-fqdn'), } # If zks have changed and we are not handling a departed spark peer, # give the ensemble time to settle. Otherwise we might try to start # spark master with data from the wrong zk leader. Doing so will cause # spark-master to shutdown: # https://issues.apache.org/jira/browse/SPARK-15544 if (zks and data_changed('zks', zks) and not is_state('sparkpeers.departed')): hookenv.status_set('maintenance', 'waiting for zookeeper ensemble to settle') hookenv.log( "Waiting 2m to ensure zk ensemble has settled: {}".format(zks)) time.sleep(120) # Let spark know if we have cuda libs installed. # NB: spark packages prereq hadoop (boo), so even in standalone mode, we'll # have hadoop libs installed. May as well include them in our lib path. extra_libs = ["/usr/lib/hadoop/lib/native"] if is_state('cuda.installed'): extra_libs.append("/usr/local/cuda/lib64") spark = Spark() spark.configure(hosts, zk_units=zks, peers=peers, extra_libs=extra_libs) set_deployment_mode_state('spark.standalone.installed')
def install_spark(hadoop=None, zks=None): spark_master_host = leadership.leader_get('master-fqdn') if not spark_master_host: hookenv.status_set('waiting', 'master not elected yet') return False hosts = { 'spark-master': spark_master_host, } if is_state('hadoop.yarn.ready'): rms = hadoop.resourcemanagers() hosts['resourcemanager'] = rms[0] if is_state('hadoop.hdfs.ready'): nns = hadoop.namenodes() hosts['namenode'] = nns[0] spark = Spark() spark.configure(hosts, zks, get_spark_peers()) return True
def install_spark_yarn(): """ Called in 'yarn-*' mode after Juju has elected a leader. The 'hadoop.yarn.ready' state must be set. """ hosts = { 'spark-master': leadership.leader_get('master-fqdn'), } hadoop = (RelationBase.from_state('hadoop.yarn.ready') or RelationBase.from_state('hadoop.hdfs.ready')) rms = hadoop.resourcemanagers() hosts['resourcemanager'] = rms[0] # Probably don't need to check this since yarn.ready implies hdfs.ready # for us, but it doesn't hurt. if is_state('hadoop.hdfs.ready'): nns = hadoop.namenodes() hosts['namenode'] = nns[0] spark = Spark() spark.configure(hosts, zk_units=None, peers=None) set_deployment_mode_state('spark.yarn.installed')
def install_spark_standalone(zks, peers): """ Called in local/standalone mode after Juju has elected a leader. """ hosts = { 'spark-master': leadership.leader_get('master-fqdn'), } # If zks have changed and we are not handling a departed spark peer, # give the ensemble time to settle. Otherwise we might try to start # spark master with data from the wrong zk leader. Doing so will cause # spark-master to shutdown: # https://issues.apache.org/jira/browse/SPARK-15544 if (zks and data_changed('zks', zks) and not is_state('sparkpeers.departed')): hookenv.status_set('maintenance', 'waiting for zookeeper ensemble to settle') hookenv.log("Waiting 2m to ensure zk ensemble has settled: {}".format(zks)) time.sleep(120) spark = Spark() spark.configure(hosts, zks, peers) set_deployment_mode_state('spark.standalone.installed')
def reinstall_spark(): """ This is tricky. We want to fire on config or leadership changes, or when hadoop, sparkpeers, or zookeepers come and go. In the future this should fire when Cassandra or any other storage comes or goes. We always fire this method (or rather, when bigtop is ready and juju has elected a master). We then build a deployment-matrix and (re)install as things change. """ spark_master_host = leadership.leader_get('master-fqdn') if not spark_master_host: hookenv.status_set('maintenance', 'juju leader not elected yet') return mode = hookenv.config()['spark_execution_mode'] peers = None zks = None # If mode is standalone and ZK is ready, we are in HA. Do not consider # the master_host from juju leadership in our matrix. ZK handles this. if (mode == 'standalone' and is_state('zookeeper.ready')): spark_master_host = '' zk = RelationBase.from_state('zookeeper.ready') zks = zk.zookeepers() # peers are only used to set our MASTER_URL in standalone HA mode peers = get_spark_peers() deployment_matrix = { 'spark_master': spark_master_host, 'yarn_ready': is_state('hadoop.yarn.ready'), 'hdfs_ready': is_state('hadoop.hdfs.ready'), 'zookeepers': zks, 'peers': peers, } # If neither config nor our matrix is changing, there is nothing to do. if not (is_state('config.changed') or data_changed('deployment_matrix', deployment_matrix)): return # (Re)install based on our execution mode hookenv.status_set('maintenance', 'configuring spark in {} mode'.format(mode)) hookenv.log("Configuring spark with deployment matrix: {}".format(deployment_matrix)) if mode.startswith('yarn') and is_state('hadoop.yarn.ready'): install_spark_yarn() elif mode.startswith('local') or mode == 'standalone': install_spark_standalone(zks, peers) else: # Something's wrong (probably requested yarn without yarn.ready). remove_state('spark.started') report_status() return # restart services to pick up possible config changes spark = Spark() spark.stop() spark.start() set_state('spark.started') report_status()
def reinstall_spark(force=False): """ Gather the state of our deployment and (re)install when leaders, hadoop, sparkpeers, or zookeepers change. In the future this should also fire when Cassandra or any other storage comes or goes. Config changed events will also call this method, but that is invoked with a separate handler below. Use a deployment-matrix dict to track changes and (re)install as needed. """ spark_master_host = leadership.leader_get('master-fqdn') if not spark_master_host: hookenv.status_set('maintenance', 'juju leader not elected yet') return mode = hookenv.config()['spark_execution_mode'] peers = None zks = None # If mode is standalone and ZK is ready, we are in HA. Do not consider # the master_host from juju leadership in our matrix. ZK handles this. if (mode == 'standalone' and is_state('zookeeper.ready')): spark_master_host = '' zk = RelationBase.from_state('zookeeper.ready') zks = zk.zookeepers() # peers are only used to set our MASTER_URL in standalone HA mode peers = get_spark_peers() # Construct a deployment matrix sample_data = hookenv.resource_get('sample-data') deployment_matrix = { 'hdfs_ready': is_state('hadoop.hdfs.ready'), 'peers': peers, 'sample_data': host.file_hash(sample_data) if sample_data else None, 'spark_master': spark_master_host, 'yarn_ready': is_state('hadoop.yarn.ready'), 'zookeepers': zks, } # No-op if we are not forcing a reinstall or our matrix is unchanged. if not (force or data_changed('deployment_matrix', deployment_matrix)): report_status() return # (Re)install based on our execution mode hookenv.status_set('maintenance', 'configuring spark in {} mode'.format(mode)) hookenv.log("Configuring spark with deployment matrix: {}".format( deployment_matrix)) if mode.startswith('yarn') and is_state('hadoop.yarn.ready'): install_spark_yarn() elif mode.startswith('local') or mode == 'standalone': install_spark_standalone(zks, peers) else: # Something's wrong (probably requested yarn without yarn.ready). remove_state('spark.started') report_status() return # restart services to pick up possible config changes spark = Spark() spark.stop() spark.start() set_state('spark.started') report_status()