def stop_in_client_mode(cluster_name, opts): # check cluster status trickly if utils.check_cluster_status(cluster_name, ['Stopped']): print "Cluster %s has been `Stopped`, you can not stop it again." % cluster_name sys.exit(1) do_validity_check(opts) (masters, slaves) = utils.get_masters_and_slaves(opts.mode) if len(masters + slaves) <= 0: print "There is no master or slave running, check it first please." sys.exit(1) print "==> Stopping spark cluster..." utils.warning() msg = "Stopping Spark cluster will stop HDFS, spark-notebook and Hue at the same time. " \ "Stop %s? (Y/n): " % cluster_name to_stop = raw_input(msg) if to_stop == "Y": if opts.pwd == "": opts.pwd = getpass.getpass( "You need to provide the password for ECS instance:") spark.stop_spark_cluster(masters, slaves, opts) hdfs.stop_hdfs(masters, slaves, opts) hue.stop_hue(masters, opts) spark_notebook.stop_spark_notebook(masters, opts) utils.stop_nginx(opts, masters) # update cluster status os.system("echo Stopped > %s%s" % (GlobalVar.CLUSTER_STATUS, cluster_name)) else: print "Not `Y`, give up stopping cluster %s" % cluster_name
def start_in_client_mode(cluster_name, opts): # check cluster status trickly if utils.check_cluster_status(cluster_name, ['Running']): print "Cluster %s is `Running`, please `Stop` it first." % cluster_name sys.exit(1) do_validity_check(opts) (masters, slaves) = utils.get_masters_and_slaves(opts.mode) if len(masters + slaves) <= 0: print "There is no master or slave, check it first please." sys.exit(1) print "==> Restarting spark cluster..." if opts.pwd == "": opts.pwd = getpass.getpass( "You need to provide the password for ECS instance:") spark.start_spark_cluster(masters[0], slaves, opts) if opts.enable_spark_notebook: spark_notebook.start_spark_notebook(masters, opts) if opts.enable_hue: hue.start_hue(masters, opts) if opts.enable_hdfs: hdfs.setup_hdfs(masters, slaves, opts) if opts.enable_slave_public_ip: utils.save_public_ips(masters, slaves) master_ip = ecs.get_instance_info( masters[0])['PublicIpAddress']['IpAddress'][0] utils.open_nginx(opts, masters) utils.end_of_startup(opts, master_ip, masters) # update cluster status os.system("echo Running > %s%s" % (GlobalVar.CLUSTER_STATUS, cluster_name))
def launch_in_client_mode(cluster_name, opts): # check cluster status trickly if utils.check_cluster_status(cluster_name, ['Running', 'Stopped']): print "Cluster %s has been launched, please `Destroy` it first." % cluster_name sys.exit(1) do_validity_check(opts) (masters, slaves) = utils.get_masters_and_slaves(opts.mode) if len(masters) <= 0: print >> stderr, "ERROR: You have to start as least 1 master" sys.exit(1) if len(slaves) <= 0: print >> stderr, "ERROR: You have to start as least 1 slave" sys.exit(1) # Now we only support single-node master. spark.setup_cluster(masters, slaves, opts, True) if opts.enable_spark_notebook: spark_notebook.start_spark_notebook(masters, opts) if opts.enable_hue: hue.start_hue(masters, opts) if opts.enable_hdfs: hdfs.setup_hdfs(masters, slaves, opts) if opts.enable_slave_public_ip: utils.save_public_ips(masters, slaves) master_ip = ecs.get_instance_info( masters[0])['PublicIpAddress']['IpAddress'][0] utils.open_nginx(opts, masters) utils.end_of_startup(opts, master_ip, masters) # update cluster status os.system("echo Running > %s%s" % (GlobalVar.CLUSTER_STATUS, cluster_name))
def start_in_client_mode(cluster_name, opts): # check cluster status trickly if utils.check_cluster_status(cluster_name, ['Running']): print "Cluster %s is `Running`, please `Stop` it first." % cluster_name sys.exit(1) do_validity_check(opts) (masters, slaves) = utils.get_masters_and_slaves(opts.mode) if len(masters + slaves) <= 0: print "There is no master or slave, check it first please." sys.exit(1) print "==> Restarting spark cluster..." if opts.pwd == "": opts.pwd = getpass.getpass("You need to provide the password for ECS instance:") spark.start_spark_cluster(masters[0], slaves, opts) if opts.enable_spark_notebook: spark_notebook.start_spark_notebook(masters, opts) if opts.enable_hue: hue.start_hue(masters, opts) if opts.enable_hdfs: hdfs.setup_hdfs(masters, slaves, opts) if opts.enable_slave_public_ip: utils.save_public_ips(masters, slaves) master_ip = ecs.get_instance_info(masters[0])['PublicIpAddress']['IpAddress'][0] utils.open_nginx(opts, masters) utils.end_of_startup(opts, master_ip, masters) # update cluster status os.system("echo Running > %s%s" % (GlobalVar.CLUSTER_STATUS, cluster_name))
def stop_in_client_mode(cluster_name, opts): # check cluster status trickly if utils.check_cluster_status(cluster_name, ['Stopped']): print "Cluster %s has been `Stopped`, you can not stop it again." % cluster_name sys.exit(1) do_validity_check(opts) (masters, slaves) = utils.get_masters_and_slaves(opts.mode) if len(masters + slaves) <= 0: print "There is no master or slave running, check it first please." sys.exit(1) print "==> Stopping spark cluster..." utils.warning() msg = "Stopping Spark cluster will stop HDFS, spark-notebook and Hue at the same time. " \ "Stop %s? (Y/n): " % cluster_name to_stop = raw_input(msg) if to_stop == "Y": if opts.pwd == "": opts.pwd = getpass.getpass("You need to provide the password for ECS instance:") spark.stop_spark_cluster(masters, slaves, opts) hdfs.stop_hdfs(masters, slaves, opts) hue.stop_hue(masters, opts) spark_notebook.stop_spark_notebook(masters, opts) utils.stop_nginx(opts,masters) # update cluster status os.system("echo Stopped > %s%s" % (GlobalVar.CLUSTER_STATUS, cluster_name)) else: print "Not `Y`, give up stopping cluster %s" % cluster_name
def launch_in_client_mode(cluster_name, opts): # check cluster status trickly if utils.check_cluster_status(cluster_name, ['Running', 'Stopped']): print "Cluster %s has been launched, please `Destroy` it first." % cluster_name sys.exit(1) do_validity_check(opts) (masters, slaves) = utils.get_masters_and_slaves(opts.mode) if len(masters) <= 0: print >> stderr, "ERROR: You have to start as least 1 master" sys.exit(1) if len(slaves) <= 0: print >> stderr, "ERROR: You have to start as least 1 slave" sys.exit(1) # Now we only support single-node master. spark.setup_cluster(masters, slaves, opts, True) if opts.enable_spark_notebook: spark_notebook.start_spark_notebook(masters, opts) if opts.enable_hue: hue.start_hue(masters, opts) if opts.enable_hdfs: hdfs.setup_hdfs(masters, slaves, opts) if opts.enable_slave_public_ip: utils.save_public_ips(masters, slaves) master_ip = ecs.get_instance_info(masters[0])['PublicIpAddress']['IpAddress'][0] utils.open_nginx(opts, masters) utils.end_of_startup(opts, master_ip, masters) # update cluster status os.system("echo Running > %s%s" % (GlobalVar.CLUSTER_STATUS, cluster_name))
def disable_module(name, opts): if len(name.split(":")) != 2: print "\nYou need to provide a <cluster_name>:<module_name>\n" sys.exit(1) cluster_name = name.split(":")[0] module_name = name.split(":")[1] do_validity_check(opts) (masters, slaves) = utils.get_masters_and_slaves(opts.mode, cluster_name) if module_name == "hdfs": hdfs.stop_hdfs(masters, slaves, opts) elif module_name == "hue": hue.stop_hue(masters, opts) elif module_name == "spark-notebook": spark_notebook.stop_spark_notebook(masters, opts) else: print "Now we only support 3 module: hdfs, hue, spark-notebook" sys.exit(1)
def destroy_in_client_mode(cluster_name, opts): do_validity_check(opts) (masters, slaves) = utils.get_masters_and_slaves(opts.mode) if len(masters + slaves) <= 0: print "There is no master or slave, check it first please." sys.exit(1) print "Are you sure you want to destroy the cluster %s?" % cluster_name print "The following instances will be terminated:" instances = masters + slaves gateway = ecs.get_gateway_instance_info(opts)['InstanceId'] if gateway in instances: instances.remove(gateway) to_release = [] for ins in instances: try: instance_info = ecs.get_instance_info(ins) to_release.append(ins) print "> %s" % (instance_info['HostName']) except Exception, e: if 'InvalidInstanceId.NotFound' in e.args: print "> %s, invalid `InstanceId` not found, skip it." % ins else: raise e