def start_in_client_mode(cluster_name, opts):
# check cluster status trickly
if utils.check_cluster_status(cluster_name, ['Running']):
print "Cluster %s is `Running`, please `Stop` it first." % cluster_name
sys.exit(1)
do_validity_check(opts)
(masters, slaves) = utils.get_masters_and_slaves(opts.mode)
if len(masters + slaves) <= 0:
print "There is no master or slave, check it first please."
sys.exit(1)
print "==> Restarting spark cluster..."
if opts.pwd == "":
opts.pwd = getpass.getpass(
"You need to provide the password for ECS instance:")
spark.start_spark_cluster(masters[0], slaves, opts)
if opts.enable_spark_notebook:
spark_notebook.start_spark_notebook(masters, opts)
if opts.enable_hue:
hue.start_hue(masters, opts)
if opts.enable_hdfs:
hdfs.setup_hdfs(masters, slaves, opts)
if opts.enable_slave_public_ip:
utils.save_public_ips(masters, slaves)
master_ip = ecs.get_instance_info(
masters[0])['PublicIpAddress']['IpAddress'][0]
utils.open_nginx(opts, masters)
utils.end_of_startup(opts, master_ip, masters)
# update cluster status
os.system("echo Running > %s%s" % (GlobalVar.CLUSTER_STATUS, cluster_name))
def launch_in_client_mode(cluster_name, opts):
# check cluster status trickly
if utils.check_cluster_status(cluster_name, ['Running', 'Stopped']):
print "Cluster %s has been launched, please `Destroy` it first." % cluster_name
sys.exit(1)
do_validity_check(opts)
(masters, slaves) = utils.get_masters_and_slaves(opts.mode)
if len(masters) <= 0:
print >> stderr, "ERROR: You have to start as least 1 master"
sys.exit(1)
if len(slaves) <= 0:
print >> stderr, "ERROR: You have to start as least 1 slave"
sys.exit(1)
# Now we only support single-node master.
spark.setup_cluster(masters, slaves, opts, True)
if opts.enable_spark_notebook:
spark_notebook.start_spark_notebook(masters, opts)
if opts.enable_hue:
hue.start_hue(masters, opts)
if opts.enable_hdfs:
hdfs.setup_hdfs(masters, slaves, opts)
if opts.enable_slave_public_ip:
utils.save_public_ips(masters, slaves)
master_ip = ecs.get_instance_info(
masters[0])['PublicIpAddress']['IpAddress'][0]
utils.open_nginx(opts, masters)
utils.end_of_startup(opts, master_ip, masters)
# update cluster status
os.system("echo Running > %s%s" % (GlobalVar.CLUSTER_STATUS, cluster_name))
def stop_in_client_mode(cluster_name, opts):
# check cluster status trickly
if utils.check_cluster_status(cluster_name, ['Stopped']):
print "Cluster %s has been `Stopped`, you can not stop it again." % cluster_name
sys.exit(1)
do_validity_check(opts)
(masters, slaves) = utils.get_masters_and_slaves(opts.mode)
if len(masters + slaves) <= 0:
print "There is no master or slave running, check it first please."
sys.exit(1)
print "==> Stopping spark cluster..."
utils.warning()
msg = "Stopping Spark cluster will stop HDFS, spark-notebook and Hue at the same time. " \
"Stop %s? (Y/n): " % cluster_name
to_stop = raw_input(msg)
if to_stop == "Y":
if opts.pwd == "":
opts.pwd = getpass.getpass(
"You need to provide the password for ECS instance:")
spark.stop_spark_cluster(masters, slaves, opts)
hdfs.stop_hdfs(masters, slaves, opts)
hue.stop_hue(masters, opts)
spark_notebook.stop_spark_notebook(masters, opts)
utils.stop_nginx(opts, masters)
# update cluster status
os.system("echo Stopped > %s%s" %
(GlobalVar.CLUSTER_STATUS, cluster_name))
else:
print "Not `Y`, give up stopping cluster %s" % cluster_name
def launch_in_cluster_mode(cluster_name, opts):
# check cluster status trickly
if utils.check_cluster_status(cluster_name, ['Running', 'Stopped']):
print "Cluster %s has been launched, please `Destroy` it first." % cluster_name
sys.exit(1)
do_validity_check(opts)
if opts.slaves <= 0:
print >> stderr, "ERROR: You have to start as least 1 slave"
sys.exit(1)
(masters, slaves, master_ip) = utils.launch_cluster(opts, cluster_name)
utils.wait_for_cluster_state(cluster_state=['Running'],
instances=masters + slaves)
utils.mount_disk(masters, slaves, opts)
spark.setup_cluster(masters, slaves, opts, True)
if opts.enable_spark_notebook:
spark_notebook.start_spark_notebook(masters, opts)
if opts.enable_hue:
hue.start_hue(masters, opts)
if opts.enable_hdfs:
hdfs.setup_hdfs(masters, slaves, opts)
if opts.enable_slave_public_ip:
utils.save_public_ips(masters, slaves)
utils.open_nginx(opts, masters)
utils.end_of_startup(opts, master_ip, masters)
# update cluster status
os.system("echo Running > %s%s" % (GlobalVar.CLUSTER_STATUS, cluster_name))
def launch_in_cluster_mode(cluster_name, opts):
# check cluster status trickly
if utils.check_cluster_status(cluster_name, ['Running', 'Stopped']):
print "Cluster %s has been launched, please `Destroy` it first." % cluster_name
sys.exit(1)
do_validity_check(opts)
if opts.slaves <= 0:
print >> stderr, "ERROR: You have to start as least 1 slave"
sys.exit(1)
(masters, slaves, master_ip) = utils.launch_cluster(opts, cluster_name)
utils.wait_for_cluster_state(
cluster_state=['Running'],
instances=masters + slaves)
utils.mount_disk(masters, slaves, opts)
spark.setup_cluster(masters, slaves, opts, True)
if opts.enable_spark_notebook:
spark_notebook.start_spark_notebook(masters, opts)
if opts.enable_hue:
hue.start_hue(masters, opts)
if opts.enable_hdfs:
hdfs.setup_hdfs(masters, slaves, opts)
if opts.enable_slave_public_ip:
utils.save_public_ips(masters, slaves)
utils.open_nginx(opts, masters)
utils.end_of_startup(opts, master_ip, masters)
# update cluster status
os.system("echo Running > %s%s" % (GlobalVar.CLUSTER_STATUS, cluster_name))
def start_in_client_mode(cluster_name, opts):
# check cluster status trickly
if utils.check_cluster_status(cluster_name, ['Running']):
print "Cluster %s is `Running`, please `Stop` it first." % cluster_name
sys.exit(1)
do_validity_check(opts)
(masters, slaves) = utils.get_masters_and_slaves(opts.mode)
if len(masters + slaves) <= 0:
print "There is no master or slave, check it first please."
sys.exit(1)
print "==> Restarting spark cluster..."
if opts.pwd == "":
opts.pwd = getpass.getpass("You need to provide the password for ECS instance:")
spark.start_spark_cluster(masters[0], slaves, opts)
if opts.enable_spark_notebook:
spark_notebook.start_spark_notebook(masters, opts)
if opts.enable_hue:
hue.start_hue(masters, opts)
if opts.enable_hdfs:
hdfs.setup_hdfs(masters, slaves, opts)
if opts.enable_slave_public_ip:
utils.save_public_ips(masters, slaves)
master_ip = ecs.get_instance_info(masters[0])['PublicIpAddress']['IpAddress'][0]
utils.open_nginx(opts, masters)
utils.end_of_startup(opts, master_ip, masters)
# update cluster status
os.system("echo Running > %s%s" % (GlobalVar.CLUSTER_STATUS, cluster_name))
def stop_in_client_mode(cluster_name, opts):
# check cluster status trickly
if utils.check_cluster_status(cluster_name, ['Stopped']):
print "Cluster %s has been `Stopped`, you can not stop it again." % cluster_name
sys.exit(1)
do_validity_check(opts)
(masters, slaves) = utils.get_masters_and_slaves(opts.mode)
if len(masters + slaves) <= 0:
print "There is no master or slave running, check it first please."
sys.exit(1)
print "==> Stopping spark cluster..."
utils.warning()
msg = "Stopping Spark cluster will stop HDFS, spark-notebook and Hue at the same time. " \
"Stop %s? (Y/n): " % cluster_name
to_stop = raw_input(msg)
if to_stop == "Y":
if opts.pwd == "":
opts.pwd = getpass.getpass("You need to provide the password for ECS instance:")
spark.stop_spark_cluster(masters, slaves, opts)
hdfs.stop_hdfs(masters, slaves, opts)
hue.stop_hue(masters, opts)
spark_notebook.stop_spark_notebook(masters, opts)
utils.stop_nginx(opts,masters)
# update cluster status
os.system("echo Stopped > %s%s" % (GlobalVar.CLUSTER_STATUS, cluster_name))
else:
print "Not `Y`, give up stopping cluster %s" % cluster_name
def launch_in_client_mode(cluster_name, opts):
# check cluster status trickly
if utils.check_cluster_status(cluster_name, ['Running', 'Stopped']):
print "Cluster %s has been launched, please `Destroy` it first." % cluster_name
sys.exit(1)
do_validity_check(opts)
(masters, slaves) = utils.get_masters_and_slaves(opts.mode)
if len(masters) <= 0:
print >> stderr, "ERROR: You have to start as least 1 master"
sys.exit(1)
if len(slaves) <= 0:
print >> stderr, "ERROR: You have to start as least 1 slave"
sys.exit(1)
# Now we only support single-node master.
spark.setup_cluster(masters, slaves, opts, True)
if opts.enable_spark_notebook:
spark_notebook.start_spark_notebook(masters, opts)
if opts.enable_hue:
hue.start_hue(masters, opts)
if opts.enable_hdfs:
hdfs.setup_hdfs(masters, slaves, opts)
if opts.enable_slave_public_ip:
utils.save_public_ips(masters, slaves)
master_ip = ecs.get_instance_info(masters[0])['PublicIpAddress']['IpAddress'][0]
utils.open_nginx(opts, masters)
utils.end_of_startup(opts, master_ip, masters)
# update cluster status
os.system("echo Running > %s%s" % (GlobalVar.CLUSTER_STATUS, cluster_name))