def reconfigure_spark():
config = hookenv.config()
maintenance = config['maintenance_mode']
if maintenance:
remove_state('not.upgrading')
spark = Spark(get_dist_config())
report_status(spark)
spark.stop()
current_version = spark.get_current_version()
if config['upgrade_immediately'] and config['spark_version'] != current_version:
upgrade_spark()
return
else:
set_state('not.upgrading')
mode = hookenv.config()['spark_execution_mode']
hookenv.status_set('maintenance', 'Configuring Apache Spark')
spark = Spark(get_dist_config())
spark.stop()
if is_state('hadoop.ready') and mode.startswith('yarn') and (not is_state('yarn.configured')):
# was in a mode other than yarn, going to yarn
hookenv.status_set('maintenance', 'Setting up Apache Spark for YARN')
spark.configure_yarn_mode()
set_state('yarn.configured')
if is_state('hadoop.ready') and (not mode.startswith('yarn')) and is_state('yarn.configured'):
# was in a yarn mode and going to another mode
hookenv.status_set('maintenance', 'Disconnecting Apache Spark from YARN')
spark.disable_yarn_mode()
remove_state('yarn.configured')
spark.configure()
spark.start()
report_status(spark)
def install_oozie(hadoop):
hookenv.status_set('maintenance', 'Installing Apache Oozie')
oozie = Oozie(get_dist_config())
if oozie.verify_resources():
oozie.install()
set_state('oozie.installed')
def reconfigure_hive(database):
hookenv.status_set('active', 'Configuring Hive')
hive = Hive(get_dist_config())
hive.stop()
hive.configure_hive(database)
hive.start()
hookenv.status_set('active', 'Ready')
def configure_zookeeper(zks):
hookenv.status_set('maintenance', 'Configuring Hue for Zookeeper')
hue = Hue(get_dist_config())
hue.configure_zookeeper(zks.zookeepers())
hue.update_apps()
hue.restart()
set_state('zookeeper.configured')
def stop_hive_wait_db(hdfs):
hookenv.status_set('maintenance', 'Stopping Apache Hive')
hive = Hive(get_dist_config())
hive.stop()
hive.close_ports()
remove_state('hive.started')
hookenv.status_set('blocked', 'Waiting for database connection')
def configure_zookeeper(zks):
hookenv.status_set('maintenance', 'Configuring Hue for Zookeeper')
hue = Hue(get_dist_config())
hue.configure_zookeeper(zks.zookeepers())
hue.update_apps()
hue.restart()
set_state('zookeeper.configured')
def install_sqoop(db):
hookenv.status_set('maintenance', 'Installing Sqoop Server')
dist = get_dist_config()
sqoop = Sqoop(dist)
sqoop.install_sqoop()
sqoop.open_ports()
sqoop.start()
set_state('sqoop.installed')
def start_spark():
hookenv.status_set('maintenance', 'Setting up Apache Spark')
spark = Spark(get_dist_config())
spark.configure()
spark.start()
spark.open_ports()
set_state('spark.started')
report_status(spark)
def configure_oozie(oozie):
oozie_host = oozie.get_private_ip()
oozie_port = oozie.get_port()
hue = Hue(get_dist_config())
hue.configure_oozie(oozie_host, oozie_port)
hue.update_apps()
hue.restart()
set_state('oozie.configured')
def start_spark():
hookenv.status_set('maintenance', 'Setting up Apache Spark')
spark = Spark(get_dist_config())
spark.configure()
spark.start()
spark.open_ports()
set_state('spark.started')
report_status(spark)
def configure_oozie(oozie):
oozie_host = oozie.get_private_ip()
oozie_port = oozie.get_port()
hue = Hue(get_dist_config())
hue.configure_oozie(oozie_host, oozie_port)
hue.update_apps()
hue.restart()
set_state('oozie.configured')
def disable_yarn():
hookenv.status_set('maintenance', 'Disconnecting Apache Spark from YARN')
spark = Spark(get_dist_config())
spark.stop()
spark.disable_yarn_mode()
spark.start()
remove_state('yarn.configured')
report_status(spark)
def install_oozie(db):
hookenv.status_set('maintenance', 'Installing Oozie')
dist = get_dist_config()
oozie = Oozie(dist)
oozie.install_oozie()
oozie.open_ports()
oozie.start()
set_state('oozie.installed')
def install_sqoop(db):
hookenv.status_set('maintenance', 'Installing Sqoop Server')
dist = get_dist_config()
sqoop = Sqoop(dist)
sqoop.install_sqoop()
sqoop.open_ports()
sqoop.start()
set_state('sqoop.installed')
def disable_yarn():
hookenv.status_set('maintenance', 'Disconnecting Apache Spark from YARN')
spark = Spark(get_dist_config())
spark.stop()
spark.disable_yarn_mode()
spark.start()
remove_state('yarn.configured')
report_status(spark)
def disable_zookeepers():
hookenv.status_set('maintenance', 'Disabling high availability')
spark = Spark(get_dist_config())
spark.stop()
spark.disable_ha()
spark.configure()
spark.start()
remove_state('zookeeper.configured')
report_status(spark)
def stop_datagrid_services():
dc = get_dist_config()
ie_home = dc.path('spark')
if utils.jps("insightedge.marker=master"):
d = dict(os.environ)
d["TIMEOUT"] = str(10)
subprocess.call([ie_home / "sbin" / "stop-datagrid-master.sh"], env=d)
if utils.jps("insightedge.marker=slave"):
subprocess.call([ie_home / "sbin" / "stop-datagrid-slave.sh"])
def install_spark():
dist = get_dist_config()
spark = Spark(dist)
if spark.verify_resources():
hookenv.status_set('maintenance', 'Installing Apache Spark')
spark.install()
spark.setup_spark_config()
spark.install_demo()
set_state('spark.installed')
def configure_spark(spark):
hookenv.status_set('maintenance', 'Configuring Hue for Spark')
spark_host = spark.get_private_ip()
spark_rest_port = spark.get_rest_port()
hue = Hue(get_dist_config())
hue.configure_spark(spark_host, spark_rest_port)
hue.update_apps()
hue.restart()
set_state('spark.configured')
def configure_spark(spark):
hookenv.status_set('maintenance', 'Configuring Hue for Spark')
spark_host = spark.get_private_ip()
spark_rest_port = spark.get_rest_port()
hue = Hue(get_dist_config())
hue.configure_spark(spark_host, spark_rest_port)
hue.update_apps()
hue.restart()
set_state('spark.configured')
def configure_hive(hive):
hookenv.status_set('maintenance', 'Configuring Hue for Hive')
hive_host = hive.get_private_ip()
hive_port = hive.get_port()
hue = Hue(get_dist_config())
hue.configure_hive(hive_host, hive_port)
hue.update_apps()
hue.restart()
set_state('hive.configured')
def start_hive(hdfs, database):
hookenv.status_set('maintenance', 'Setting up Apache Hive')
hive = Hive(get_dist_config())
hive.setup_hive_config()
hive.configure_hive(database)
hive.open_ports()
hive.start()
set_state('hive.started')
hookenv.status_set('active', 'Ready')
def configure_hive(hive):
hookenv.status_set('maintenance', 'Configuring Hue for Hive')
hive_host = hive.get_private_ip()
hive_port = hive.get_port()
hue = Hue(get_dist_config())
hue.configure_hive(hive_host, hive_port)
hue.update_apps()
hue.restart()
set_state('hive.configured')
def install_pig():
pig = Pig(get_dist_config())
if pig.verify_resources():
hookenv.status_set('maintenance', 'installing pig')
hookenv.log('Installing Apache Pig')
pig.install()
pig.initial_config()
set_state('pig.installed')
hookenv.status_set('waiting', 'waiting to configure pig')
hookenv.log('Apache Pig is installed and ready to be configured')
def configure_hue(hadoop):
namenodes = hadoop.namenodes()
resmngmrs = hadoop.resourcemanagers()
hdfs_port = hadoop.hdfs_port()
yarn_port = hadoop.yarn_port()
hookenv.status_set('maintenance', 'Setting up Hue')
hue = Hue(get_dist_config())
hue.setup_hue(namenodes, resmngmrs, hdfs_port,
yarn_port, yarn_http, yarn_ipcp)
set_state('hue.configured')
def configure_zookeepers(zk):
zks = zk.zookeepers()
if data_changed('available.zookeepers', zks):
spark = Spark(get_dist_config())
hookenv.status_set('maintenance', 'Updating Apache Spark HA')
spark.stop()
spark.configure_ha(zks)
spark.configure()
spark.start()
set_state('zookeeper.configured')
report_status(spark)
def configure_zookeepers(zk):
zks = zk.zookeepers()
if data_changed('available.zookeepers', zks):
spark = Spark(get_dist_config())
hookenv.status_set('maintenance', 'Updating Apache Spark HA')
spark.stop()
spark.configure_ha(zks)
spark.configure()
spark.start()
set_state('zookeeper.configured')
report_status(spark)
def update_peers_config(nodes):
nodes.sort()
if data_changed('available.peers', nodes):
spark = Spark(get_dist_config())
# We need to reconfigure spark only if the master changes or if we
# are in HA mode and a new potential master is added
if spark.update_peers(nodes):
hookenv.status_set('maintenance', 'Updating Apache Spark config')
spark.stop()
spark.configure()
spark.start()
report_status(spark)
def update_peers_config(nodes):
nodes.sort()
if data_changed('available.peers', nodes):
spark = Spark(get_dist_config())
# We need to reconfigure spark only if the master changes or if we
# are in HA mode and a new potential master is added
if spark.update_peers(nodes):
hookenv.status_set('maintenance', 'Updating Apache Spark config')
spark.stop()
spark.configure()
spark.start()
report_status(spark)
def install_hive(hadoop):
# Hive cannot handle - in the metastore db name and
# mysql uses the service name to name the db
if "-" in hookenv.service_name():
hookenv.status_set('blocked', 'Service name should not contain -. '
'Redeploy with a different name.')
return
hive = Hive(get_dist_config())
if hive.verify_resources():
hookenv.status_set('maintenance', 'Installing Apache Hive')
hive.install()
set_state('hive.installed')
def install_spark():
dist = get_dist_config()
spark = Spark(dist)
hookenv.status_set('maintenance', 'Installing Apache Spark')
try:
spark.install()
except ResourceError as e:
hookenv.status_set('blocked', str(e))
return
spark.setup_spark_config()
spark.install_demo()
set_state('spark.installed')
set_state('not.upgrading')
def install_spark():
dist = get_dist_config()
spark = Spark(dist)
hookenv.status_set('maintenance', 'Installing Apache Spark')
try:
spark.install()
except ResourceError as e:
hookenv.status_set('blocked', str(e))
return
spark.setup_spark_config()
spark.install_demo()
set_state('spark.installed')
set_state('not.upgrading')
def restart_services():
dc = get_dist_config()
spark = Spark(dc)
peers = RelationBase.from_state('sparkpeers.joined')
is_scaled = peers and len(peers.get_nodes()) > 0
is_master = spark.is_master()
is_slave = not is_master or not is_scaled
master_url = spark.get_master()
master_ip = spark.get_master_ip()
if data_changed('insightedge.master_url', master_url):
stop_datagrid_services()
start_datagrid_services(master_url, master_ip, is_master, is_slave)
set_state('insightedge.ready')
hookenv.status_set('active', 'ready')
def switch_to_yarn(hadoop):
'''
In case you first change the config and then connect the plugin.
'''
mode = hookenv.config()['spark_execution_mode']
if mode.startswith('yarn'):
spark = Spark(get_dist_config())
hookenv.status_set('maintenance', 'Setting up Apache Spark for YARN')
spark.stop()
spark.configure_yarn_mode()
set_state('yarn.configured')
spark.configure()
spark.start()
report_status(spark)
def upgrade_spark():
if is_state('not.upgrading'):
return (False, "Please enter maintenance mode before triggering the upgrade.")
version = hookenv.config()['spark_version']
hookenv.status_set('maintenance', 'Upgrading to {}'.format(version))
hookenv.log("Upgrading to {}".format(version))
try:
spark = Spark(get_dist_config())
spark.switch_version(version)
except ResourceError:
return (False, "Download failed")
hookenv.status_set('maintenance', 'Upgrade complete. You can exit maintenance mode')
return (True, "ok")
def switch_to_yarn(hadoop):
'''
In case you first change the config and then connect the plugin.
'''
mode = hookenv.config()['spark_execution_mode']
if mode.startswith('yarn'):
spark = Spark(get_dist_config())
hookenv.status_set('maintenance', 'Setting up Apache Spark for YARN')
spark.stop()
spark.configure_yarn_mode()
set_state('yarn.configured')
spark.configure()
spark.start()
report_status(spark)
def configure_insightedge_spark():
hookenv.status_set('maintenance', 'configuring insightedge')
dc = get_dist_config()
destination = dc.path('spark')
spark = Spark(dc)
with host.chdir(destination):
insightedge_jars = subprocess.check_output([
'bash', '-c', '. {}; get_libs ,'.format(
destination / 'sbin' / 'common-insightedge.sh')
],
env={
'INSIGHTEDGE_HOME':
destination
}).decode('utf8')
spark.register_classpaths(insightedge_jars.split(','))
set_state('insightedge-spark.configured')
def start_datagrid_services(master_url, master_ip, is_master, is_slave):
# TODO:
# * some of the below settings should be exposed as charm config
dc = get_dist_config()
ie_home = dc.path('spark')
if is_master:
subprocess.call([
ie_home / "sbin" / "start-datagrid-master.sh", "--master",
master_ip, "--size", "1G"
])
if is_slave:
subprocess.call([
ie_home / "sbin" / "start-datagrid-slave.sh", "--master",
master_url, "--locator", "{}:4174".format(master_ip), "--group",
"insightedge", "--name", "insightedge-space", "--topology", "2,0",
"--size", "1G", "--instances", "id=1;id=2"
])
def install_spark(hadoop=None):
spark_master_host = leadership.leader_get('master-fqdn')
hosts = {
'spark-master': spark_master_host,
}
if is_state('hadoop.yarn.ready'):
rms = hadoop.resourcemanagers()
hosts['resourcemanager'] = rms[0]
if is_state('hadoop.hdfs.ready'):
nns = hadoop.namenodes()
hosts['namenode'] = nns[0]
dist = get_dist_config()
spark = Spark(dist)
spark.configure(hosts)
def configure_pig():
pig = Pig(get_dist_config())
hadoop_ready = is_state('hadoop.ready')
if hadoop_ready:
hookenv.status_set('maintenance', 'configuring pig (mapreduce)')
hookenv.log('YARN is ready, configuring Apache Pig in MapReduce mode')
pig.configure_yarn()
remove_state('pig.configured.local')
set_state('pig.configured.yarn')
hookenv.status_set('active', 'ready (mapreduce)')
hookenv.log('Apache Pig is ready in MapReduce mode')
else:
hookenv.status_set('maintenance', 'configuring pig (local)')
hookenv.log('YARN is not ready, configuring Pig in local mode')
pig.configure_local()
remove_state('pig.configured.yarn')
set_state('pig.configured.local')
hookenv.status_set('active', 'ready (local)')
hookenv.log('Apache Pig is ready in local mode')
def reconfigure_spark():
mode = hookenv.config()['spark_execution_mode']
hookenv.status_set('maintenance', 'Configuring Apache Spark')
spark = Spark(get_dist_config())
spark.stop()
if is_state('hadoop.ready') and mode.startswith('yarn') and (not is_state('yarn.configured')):
# was in a mode other than yarn, going to yarn
hookenv.status_set('maintenance', 'Setting up Apache Spark for YARN')
spark.configure_yarn_mode()
set_state('yarn.configured')
if is_state('hadoop.ready') and (not mode.startswith('yarn')) and is_state('yarn.configured'):
# was in a yarn mode and going to another mode
hookenv.status_set('maintenance', 'Disconnecting Apache Spark from YARN')
spark.disable_yarn_mode()
remove_state('yarn.configured')
spark.configure()
spark.start()
report_status(spark)
def depart_spark():
hookenv.status_set('maintenance', 'Disconnecting Spark from Hue')
remove_state('spark.configured')
hue = Hue(get_dist_config())
hue.update_apps()
hue.restart()
def depart_zookeeper():
hookenv.status_set('maintenance', 'Disconnecting Zookeeper from Hue')
remove_state('zookeeper.configured')
hue = Hue(get_dist_config())
hue.update_apps()
hue.restart()
def restart_hue():
# Can't seem to mix @when_file_changed and @when('hue.started')
hue = Hue(get_dist_config())
hue.restart()
def check_relations(*args):
hue = Hue(get_dist_config())
hue.check_relations()
def stop_hue():
hue = Hue(get_dist_config())
hue.stop()
remove_state('hue.started')
hookenv.status_set('blocked', 'Waiting for Hadoop connection')
def start_hue(hadoop):
hookenv.status_set('maintenance', 'Setting up Hue')
hue = Hue(get_dist_config())
hue.open_ports()
hue.start()
set_state('hue.started')
def install_hue(hadoop):
hue = Hue(get_dist_config())
if hue.verify_resources():
hookenv.status_set('maintenance', 'Installing Hue')
hue.install()
set_state('hue.installed')
def stop_hue():
hue = Hue(get_dist_config())
hue.stop()
remove_state('hue.started')
hookenv.status_set('blocked', 'Waiting for Hadoop connection')
def start_hue(hadoop):
hookenv.status_set('maintenance', 'Setting up Hue')
hue = Hue(get_dist_config())
hue.open_ports()
hue.start()
set_state('hue.started')
def restart_hue():
# Can't seem to mix @when_file_changed and @when('hue.started')
hue = Hue(get_dist_config())
hue.restart()
def check_relations(*args):
hue = Hue(get_dist_config())
hue.check_relations()
