def stop(self):
spark_home = self.dist_config.path('spark')
# Only stop services if they're running
if utils.jps("HistoryServer"):
utils.run_as('ubuntu', '{}/sbin/stop-history-server.sh'.format(spark_home))
if utils.jps("Master"):
utils.run_as('ubuntu', '{}/sbin/stop-master.sh'.format(spark_home))
if utils.jps("Worker"):
utils.run_as('ubuntu', '{}/sbin/stop-slave.sh'.format(spark_home))
def stop_datagrid_services():
if utils.jps("insightedge.marker=master"):
d = dict(os.environ)
d["TIMEOUT"] = str(10)
cmd = "/usr/lib/insightedge/sbin/stop-datagrid-master.sh"
subprocess.call(cmd, shell=True, env=d)
if utils.jps("insightedge.marker=slave"):
cmd = "/usr/lib/insightedge/sbin/stop-datagrid-slave.sh"
subprocess.call(cmd, shell=False)
def stop_datagrid_services():
dc = get_dist_config()
ie_home = dc.path('spark')
if utils.jps("insightedge.marker=master"):
d = dict(os.environ)
d["TIMEOUT"] = str(10)
subprocess.call([ie_home / "sbin" / "stop-datagrid-master.sh"], env=d)
if utils.jps("insightedge.marker=slave"):
subprocess.call([ie_home / "sbin" / "stop-datagrid-slave.sh"])
def stop(self):
if not unitdata.kv().get('spark.installed', False):
return
# Only stop services if they're running
if utils.jps("HistoryServer"):
host.service_stop('spark-history-server')
if utils.jps("Master"):
host.service_stop('spark-master')
if utils.jps("Worker"):
host.service_stop('spark-worker')
def stop(self):
spark_home = self.dist_config.path('spark')
# Only stop services if they're running
if utils.jps("HistoryServer"):
utils.run_as('ubuntu',
'{}/sbin/stop-history-server.sh'.format(spark_home))
if utils.jps("Master"):
utils.run_as('ubuntu', '{}/sbin/stop-master.sh'.format(spark_home))
if utils.jps("Worker"):
utils.run_as('ubuntu', '{}/sbin/stop-slave.sh'.format(spark_home))
def stop(self):
if not unitdata.kv().get('spark.installed', False):
return
# Only stop services if they're running
if utils.jps("HistoryServer"):
host.service_stop('spark-history')
if utils.jps("Master"):
host.service_stop('spark-master')
if utils.jps("Worker"):
host.service_stop('spark-slave')
def stop(self):
if not unitdata.kv().get('spark.installed', False):
return
spark_home = self.dist_config.path('spark')
# Only stop services if they're running
if utils.jps("HistoryServer"):
utils.run_as('ubuntu', '{}/sbin/stop-history-server.sh'.format(spark_home))
if utils.jps("Master"):
utils.run_as('ubuntu', '{}/sbin/stop-master.sh'.format(spark_home))
if utils.jps("Worker"):
utils.run_as('ubuntu', '{}/sbin/stop-slave.sh'.format(spark_home))
def stop(self):
if not unitdata.kv().get('spark.installed', False):
return
spark_home = self.dist_config.path('spark')
# Only stop services if they're running
if utils.jps("HistoryServer"):
utils.run_as('ubuntu',
'{}/sbin/stop-history-server.sh'.format(spark_home))
if utils.jps("Master"):
utils.run_as('ubuntu', '{}/sbin/stop-master.sh'.format(spark_home))
if utils.jps("Worker"):
utils.run_as('ubuntu', '{}/sbin/stop-slave.sh'.format(spark_home))
def stop(self):
if utils.jps("zeppelin"):
host.service_stop('zeppelin')
# wait for the process to stop, since issuing a start while the
# process is still running (i.e., restart) could cause it to not
# start up again
self.wait_for_stop(30)
def start(self):
# Start if we're not already running. We currently dont have any
# runtime config options, so no need to restart when hooks fire.
if not utils.jps("zeppelin"):
host.service_start('zeppelin')
# wait up to 30s for server to start responding, lest API requests fail
self.wait_for_api(30)
def register_slaves(self, slaves=None):
if not slaves: # FIXME hack-around until transition to layers is complete
slaves = helpers.all_ready_units('datanode')
slaves = [data['hostname'] for slave, data in slaves]
self.hadoop_base.register_slaves(slaves)
if utils.jps('NameNode'):
self.hadoop_base.run('hdfs', 'bin/hdfs', 'dfsadmin', '-refreshNodes')
def register_slaves(self, slaves=None):
if not slaves: # FIXME hack-around until transition to layers is complete
slaves = helpers.all_ready_units('nodemanager')
slaves = [data['hostname'] for slave, data in slaves]
self.hadoop_base.register_slaves(slaves)
if utils.jps('ResourceManager'):
self.hadoop_base.run('mapred', 'bin/yarn', 'rmadmin', '-refreshNodes')
def start_secondarynamenode(self):
if not utils.jps('SecondaryNameNode'):
self._hadoop_daemon('start', 'secondarynamenode')
# Some hadoop processes take a bit of time to start
# we need to let them get to a point where they are
# ready to accept connections - increase the value for hadoop 2.4.1
time.sleep(30)
def start(self):
# Start if we're not already running. We currently dont have any
# runtime config options, so no need to restart when hooks fire.
if not utils.jps("livy"):
livy_log = self.dist_config.path("livy_logs") + "livy-server.log"
livy_home = self.dist_config.path("livy")
# chdir here because things like zepp tutorial think ZEPPELIN_HOME
# is wherever the daemon was started from.
os.chdir(livy_home)
utils.run_as("hue", "./bin/livy-server", "2>&1", livy_log, "&")
def start(self):
if not utils.jps("Main"):
livy_log = self.dist_config.path('livy_logs') / 'livy-server.log'
livy_home = self.dist_config.path('livy')
# chdir here because things like zepp tutorial think ZEPPELIN_HOME
# is wherever the daemon was started from.
os.chdir(livy_home)
#utils.run_as(self.user, './bin/livy-server', '2>&1', livy_log, '&')
#utils.run_as(self.user, './bin/livy-server', '&')
os.system("su {} -c \"./bin/livy-server 2>&1 > {} &\"".format(self.user, livy_log))
def start(self):
# Start if we're not already running. We currently dont have any
# runtime config options, so no need to restart when hooks fire.
if not utils.jps("zeppelin"):
zeppelin_conf = self.dist_config.path('zeppelin_conf')
zeppelin_home = self.dist_config.path('zeppelin')
# chdir here because things like zepp tutorial think ZEPPELIN_HOME
# is wherever the daemon was started from.
os.chdir(zeppelin_home)
utils.run_as('ubuntu', '{}/bin/zeppelin-daemon.sh'.format(zeppelin_home), '--config', zeppelin_conf, 'start')
def start(self):
if not utils.jps("Main"):
livy_log = self.dist_config.path('livy_logs') / 'livy-server.log'
livy_home = self.dist_config.path('livy')
# chdir here because things like zepp tutorial think ZEPPELIN_HOME
# is wherever the daemon was started from.
os.chdir(livy_home)
#utils.run_as(self.user, './bin/livy-server', '2>&1', livy_log, '&')
#utils.run_as(self.user, './bin/livy-server', '&')
os.system("su {} -c \"./bin/livy-server 2>&1 > {} &\"".format(
self.user, livy_log))
def start(self):
# Start if we're not already running. We currently dont have any
# runtime config options, so no need to restart when hooks fire.
if not utils.jps("zeppelin"):
zeppelin_conf = self.dist_config.path('zeppelin_conf')
zeppelin_home = self.dist_config.path('zeppelin')
# chdir here because things like zepp tutorial think ZEPPELIN_HOME
# is wherever the daemon was started from.
os.chdir(zeppelin_home)
utils.run_as('ubuntu',
'{}/bin/zeppelin-daemon.sh'.format(zeppelin_home),
'--config', zeppelin_conf, 'start')
def start(self):
"""
Override start to use InsightEdge's wrapper.
"""
# Start if we're not already running. We currently dont have any
# runtime config options, so no need to restart when hooks fire.
if not utils.jps("zeppelin"):
ie_home = self.dist_config.path('insightedge')
zeppelin_home = self.dist_config.path('zeppelin')
# chdir here because things like zepp tutorial think ZEPPELIN_HOME
# is wherever the daemon was started from.
with host.chdir(zeppelin_home):
utils.run_as('ubuntu',
'{}/sbin/start-zeppelin.sh'.format(ie_home))
# wait up to 30s for API to start, lest requests fail
self.wait_for_api(30)
def stop(self):
"""
Override start to use InsightEdge's wrapper.
"""
# Start if we're not already running. We currently dont have any
# runtime config options, so no need to restart when hooks fire.
if not utils.jps("zeppelin"):
ie_home = self.dist_config.path('insightedge')
zeppelin_home = self.dist_config.path('zeppelin')
# chdir here because things like zepp tutorial think ZEPPELIN_HOME
# is wherever the daemon was started from.
with host.chdir(zeppelin_home):
utils.run_as('ubuntu',
'{}/sbin/stop-zeppelin.sh'.format(ie_home))
# wait for the process to stop, since issuing a start while the
# process is still running (i.e., restart) could cause it to not
# start up again
self.wait_for_stop(30)
def start_nodemanager(self):
if not utils.jps('NodeManager'):
host.service_start('nodemanager')
def start_journalnode(self):
if not utils.jps('JournalNode'):
self._hadoop_daemon('start', 'journalnode')
def stop(self):
flume_path = self.dist_config.path('flume')
flume_pids = utils.jps(r'-cp .*{}'.format(flume_path))
for pid in flume_pids:
os.kill(int(pid), signal.SIGKILL)
def start_journalnode(self):
if not utils.jps('JournalNode'):
host.service_start('journalnode')
def reload_slaves(self):
if utils.jps('NameNode'):
self.hadoop_base.run('hdfs', 'bin/hdfs', 'dfsadmin', '-refreshNodes')
def stop(self):
if utils.jps("zeppelin"):
zeppelin_conf = self.dist_config.path('zeppelin_conf')
zeppelin_home = self.dist_config.path('zeppelin')
daemon = '{}/bin/zeppelin-daemon.sh'.format(zeppelin_home)
utils.run_as('ubuntu', daemon, '--config', zeppelin_conf, 'stop')
def start_nodemanager(self):
if not utils.jps('NodeManager'):
self._yarn_daemon('start', 'nodemanager')
def start_resourcemanager(self):
if not utils.jps('ResourceManager'):
self._yarn_daemon('start', 'resourcemanager')
def start_datanode(self):
if not utils.jps('DataNode'):
host.service_start('datanode')
def start_secondarynamenode(self):
if not utils.jps('SecondaryNameNode'):
host.service_start('secondarynamenode')
def start_namenode(self):
if not utils.jps('NameNode'):
host.service_start('namenode')
def start_namenode(self):
if not utils.jps('NameNode'):
host.service_start('namenode')
def register_slaves(self, slaves):
self.hadoop_base.register_slaves(slaves)
if utils.jps('ResourceManager'):
self.hadoop_base.run('mapred', 'bin/yarn', 'rmadmin', '-refreshNodes')
def stop(self):
flume_pids = utils.jps(r'-cp .*{}'.format(
self.dist_config.path('flume')))
for pid in flume_pids:
os.kill(int(pid), signal.SIGKILL)
def start_jobhistory(self):
if not utils.jps('JobHistoryServer'):
host.service_start('historyserver')
def start_resourcemanager(self):
if not utils.jps('ResourceManager'):
host.service_start('resourcemanager')
def start_nodemanager(self):
if not utils.jps('NodeManager'):
host.service_start('nodemanager')
def wait_for_stop(self, timeout):
start = time.time()
while utils.jps("zeppelin"):
time.sleep(1)
if time.time() - start > timeout:
raise utils.TimeoutError('Zeppelin did not stop')
def start_datanode(self):
if not utils.jps('DataNode'):
self._hadoop_daemon('start', 'datanode')
def start(self):
spark_home = self.dist_config.path('spark')
if utils.jps("HistoryServer"):
self.stop()
utils.run_as('ubuntu', '{}/sbin/start-history-server.sh'.format(spark_home), 'hdfs:///user/ubuntu/directory')
def start_datanode(self):
if not utils.jps('DataNode'):
host.service_start('datanode')
def restart(self, user='******'):
# check for a java process with our flume dir in the classpath
if utils.jps(r'-cp .*{}'.format(self.dist_config.path('flume'))):
self.stop()
self.start(user)
def stop(self):
kafka_pids = utils.jps('kafka.Kafka')
for pid in kafka_pids:
os.kill(int(pid), signal.SIGKILL)
def stop(self):
hive_pids = utils.jps('HiveServer2')
for pid in hive_pids:
os.kill(int(pid), signal.SIGTERM)
def stop(self):
hive_pids = utils.jps('HiveServer2')
for pid in hive_pids:
os.kill(int(pid), signal.SIGTERM)
def register_slaves(self, slaves):
self.hadoop_base.register_slaves(slaves)
if utils.jps('ResourceManager'):
self.hadoop_base.run('mapred', 'bin/yarn', 'rmadmin',
'-refreshNodes')
def start_jobhistory(self):
if not utils.jps('JobHistoryServer'):
host.service_start('historyserver')
def start_journalnode(self):
if not utils.jps('JournalNode'):
host.service_start('journalnode')
def start_secondarynamenode(self):
if not utils.jps('SecondaryNameNode'):
host.service_start('secondarynamenode')
def start_jobhistory(self):
if not utils.jps('JobHistoryServer'):
self._jobhistory_daemon('start', 'historyserver')
def reload_slaves(self):
if utils.jps('NameNode'):
self.hadoop_base.run('hdfs', 'bin/hdfs', 'dfsadmin',
'-refreshNodes')
def start_resourcemanager(self):
if not utils.jps('ResourceManager'):
host.service_start('resourcemanager')
def restart(self):
# check for a java process with our flume dir in the classpath
if utils.jps(r'-cp .*{}'.format(self.dist_config.path('flume'))):
self.stop()
self.start()