def stop(self): spark_home = self.dist_config.path('spark') # Only stop services if they're running if utils.jps("HistoryServer"): utils.run_as('ubuntu', '{}/sbin/stop-history-server.sh'.format(spark_home)) if utils.jps("Master"): utils.run_as('ubuntu', '{}/sbin/stop-master.sh'.format(spark_home)) if utils.jps("Worker"): utils.run_as('ubuntu', '{}/sbin/stop-slave.sh'.format(spark_home))
def stop_datagrid_services(): if utils.jps("insightedge.marker=master"): d = dict(os.environ) d["TIMEOUT"] = str(10) cmd = "/usr/lib/insightedge/sbin/stop-datagrid-master.sh" subprocess.call(cmd, shell=True, env=d) if utils.jps("insightedge.marker=slave"): cmd = "/usr/lib/insightedge/sbin/stop-datagrid-slave.sh" subprocess.call(cmd, shell=False)
def stop_datagrid_services(): dc = get_dist_config() ie_home = dc.path('spark') if utils.jps("insightedge.marker=master"): d = dict(os.environ) d["TIMEOUT"] = str(10) subprocess.call([ie_home / "sbin" / "stop-datagrid-master.sh"], env=d) if utils.jps("insightedge.marker=slave"): subprocess.call([ie_home / "sbin" / "stop-datagrid-slave.sh"])
def stop(self): if not unitdata.kv().get('spark.installed', False): return # Only stop services if they're running if utils.jps("HistoryServer"): host.service_stop('spark-history-server') if utils.jps("Master"): host.service_stop('spark-master') if utils.jps("Worker"): host.service_stop('spark-worker')
def stop(self): if not unitdata.kv().get('spark.installed', False): return # Only stop services if they're running if utils.jps("HistoryServer"): host.service_stop('spark-history') if utils.jps("Master"): host.service_stop('spark-master') if utils.jps("Worker"): host.service_stop('spark-slave')
def stop(self): if not unitdata.kv().get('spark.installed', False): return spark_home = self.dist_config.path('spark') # Only stop services if they're running if utils.jps("HistoryServer"): utils.run_as('ubuntu', '{}/sbin/stop-history-server.sh'.format(spark_home)) if utils.jps("Master"): utils.run_as('ubuntu', '{}/sbin/stop-master.sh'.format(spark_home)) if utils.jps("Worker"): utils.run_as('ubuntu', '{}/sbin/stop-slave.sh'.format(spark_home))
def stop(self): if utils.jps("zeppelin"): host.service_stop('zeppelin') # wait for the process to stop, since issuing a start while the # process is still running (i.e., restart) could cause it to not # start up again self.wait_for_stop(30)
def start(self): # Start if we're not already running. We currently dont have any # runtime config options, so no need to restart when hooks fire. if not utils.jps("zeppelin"): host.service_start('zeppelin') # wait up to 30s for server to start responding, lest API requests fail self.wait_for_api(30)
def register_slaves(self, slaves=None): if not slaves: # FIXME hack-around until transition to layers is complete slaves = helpers.all_ready_units('datanode') slaves = [data['hostname'] for slave, data in slaves] self.hadoop_base.register_slaves(slaves) if utils.jps('NameNode'): self.hadoop_base.run('hdfs', 'bin/hdfs', 'dfsadmin', '-refreshNodes')
def register_slaves(self, slaves=None): if not slaves: # FIXME hack-around until transition to layers is complete slaves = helpers.all_ready_units('nodemanager') slaves = [data['hostname'] for slave, data in slaves] self.hadoop_base.register_slaves(slaves) if utils.jps('ResourceManager'): self.hadoop_base.run('mapred', 'bin/yarn', 'rmadmin', '-refreshNodes')
def start_secondarynamenode(self): if not utils.jps('SecondaryNameNode'): self._hadoop_daemon('start', 'secondarynamenode') # Some hadoop processes take a bit of time to start # we need to let them get to a point where they are # ready to accept connections - increase the value for hadoop 2.4.1 time.sleep(30)
def start(self): # Start if we're not already running. We currently dont have any # runtime config options, so no need to restart when hooks fire. if not utils.jps("livy"): livy_log = self.dist_config.path("livy_logs") + "livy-server.log" livy_home = self.dist_config.path("livy") # chdir here because things like zepp tutorial think ZEPPELIN_HOME # is wherever the daemon was started from. os.chdir(livy_home) utils.run_as("hue", "./bin/livy-server", "2>&1", livy_log, "&")
def start(self): if not utils.jps("Main"): livy_log = self.dist_config.path('livy_logs') / 'livy-server.log' livy_home = self.dist_config.path('livy') # chdir here because things like zepp tutorial think ZEPPELIN_HOME # is wherever the daemon was started from. os.chdir(livy_home) #utils.run_as(self.user, './bin/livy-server', '2>&1', livy_log, '&') #utils.run_as(self.user, './bin/livy-server', '&') os.system("su {} -c \"./bin/livy-server 2>&1 > {} &\"".format(self.user, livy_log))
def start(self): # Start if we're not already running. We currently dont have any # runtime config options, so no need to restart when hooks fire. if not utils.jps("zeppelin"): zeppelin_conf = self.dist_config.path('zeppelin_conf') zeppelin_home = self.dist_config.path('zeppelin') # chdir here because things like zepp tutorial think ZEPPELIN_HOME # is wherever the daemon was started from. os.chdir(zeppelin_home) utils.run_as('ubuntu', '{}/bin/zeppelin-daemon.sh'.format(zeppelin_home), '--config', zeppelin_conf, 'start')
def start(self): if not utils.jps("Main"): livy_log = self.dist_config.path('livy_logs') / 'livy-server.log' livy_home = self.dist_config.path('livy') # chdir here because things like zepp tutorial think ZEPPELIN_HOME # is wherever the daemon was started from. os.chdir(livy_home) #utils.run_as(self.user, './bin/livy-server', '2>&1', livy_log, '&') #utils.run_as(self.user, './bin/livy-server', '&') os.system("su {} -c \"./bin/livy-server 2>&1 > {} &\"".format( self.user, livy_log))
def start(self): """ Override start to use InsightEdge's wrapper. """ # Start if we're not already running. We currently dont have any # runtime config options, so no need to restart when hooks fire. if not utils.jps("zeppelin"): ie_home = self.dist_config.path('insightedge') zeppelin_home = self.dist_config.path('zeppelin') # chdir here because things like zepp tutorial think ZEPPELIN_HOME # is wherever the daemon was started from. with host.chdir(zeppelin_home): utils.run_as('ubuntu', '{}/sbin/start-zeppelin.sh'.format(ie_home)) # wait up to 30s for API to start, lest requests fail self.wait_for_api(30)
def stop(self): """ Override start to use InsightEdge's wrapper. """ # Start if we're not already running. We currently dont have any # runtime config options, so no need to restart when hooks fire. if not utils.jps("zeppelin"): ie_home = self.dist_config.path('insightedge') zeppelin_home = self.dist_config.path('zeppelin') # chdir here because things like zepp tutorial think ZEPPELIN_HOME # is wherever the daemon was started from. with host.chdir(zeppelin_home): utils.run_as('ubuntu', '{}/sbin/stop-zeppelin.sh'.format(ie_home)) # wait for the process to stop, since issuing a start while the # process is still running (i.e., restart) could cause it to not # start up again self.wait_for_stop(30)
def start_nodemanager(self): if not utils.jps('NodeManager'): host.service_start('nodemanager')
def start_journalnode(self): if not utils.jps('JournalNode'): self._hadoop_daemon('start', 'journalnode')
def stop(self): flume_path = self.dist_config.path('flume') flume_pids = utils.jps(r'-cp .*{}'.format(flume_path)) for pid in flume_pids: os.kill(int(pid), signal.SIGKILL)
def start_journalnode(self): if not utils.jps('JournalNode'): host.service_start('journalnode')
def reload_slaves(self): if utils.jps('NameNode'): self.hadoop_base.run('hdfs', 'bin/hdfs', 'dfsadmin', '-refreshNodes')
def stop(self): if utils.jps("zeppelin"): zeppelin_conf = self.dist_config.path('zeppelin_conf') zeppelin_home = self.dist_config.path('zeppelin') daemon = '{}/bin/zeppelin-daemon.sh'.format(zeppelin_home) utils.run_as('ubuntu', daemon, '--config', zeppelin_conf, 'stop')
def start_nodemanager(self): if not utils.jps('NodeManager'): self._yarn_daemon('start', 'nodemanager')
def start_resourcemanager(self): if not utils.jps('ResourceManager'): self._yarn_daemon('start', 'resourcemanager')
def start_datanode(self): if not utils.jps('DataNode'): host.service_start('datanode')
def start_secondarynamenode(self): if not utils.jps('SecondaryNameNode'): host.service_start('secondarynamenode')
def start_namenode(self): if not utils.jps('NameNode'): host.service_start('namenode')
def register_slaves(self, slaves): self.hadoop_base.register_slaves(slaves) if utils.jps('ResourceManager'): self.hadoop_base.run('mapred', 'bin/yarn', 'rmadmin', '-refreshNodes')
def stop(self): flume_pids = utils.jps(r'-cp .*{}'.format( self.dist_config.path('flume'))) for pid in flume_pids: os.kill(int(pid), signal.SIGKILL)
def start_jobhistory(self): if not utils.jps('JobHistoryServer'): host.service_start('historyserver')
def start_resourcemanager(self): if not utils.jps('ResourceManager'): host.service_start('resourcemanager')
def wait_for_stop(self, timeout): start = time.time() while utils.jps("zeppelin"): time.sleep(1) if time.time() - start > timeout: raise utils.TimeoutError('Zeppelin did not stop')
def start_datanode(self): if not utils.jps('DataNode'): self._hadoop_daemon('start', 'datanode')
def start(self): spark_home = self.dist_config.path('spark') if utils.jps("HistoryServer"): self.stop() utils.run_as('ubuntu', '{}/sbin/start-history-server.sh'.format(spark_home), 'hdfs:///user/ubuntu/directory')
def restart(self, user='******'): # check for a java process with our flume dir in the classpath if utils.jps(r'-cp .*{}'.format(self.dist_config.path('flume'))): self.stop() self.start(user)
def stop(self): kafka_pids = utils.jps('kafka.Kafka') for pid in kafka_pids: os.kill(int(pid), signal.SIGKILL)
def stop(self): hive_pids = utils.jps('HiveServer2') for pid in hive_pids: os.kill(int(pid), signal.SIGTERM)
def start_jobhistory(self): if not utils.jps('JobHistoryServer'): self._jobhistory_daemon('start', 'historyserver')
def restart(self): # check for a java process with our flume dir in the classpath if utils.jps(r'-cp .*{}'.format(self.dist_config.path('flume'))): self.stop() self.start()