コード例 #1
0
def launch_in_client_mode(cluster_name, opts):
    # check cluster status trickly
    if utils.check_cluster_status(cluster_name, ['Running', 'Stopped']):
        print "Cluster %s has been launched, please `Destroy` it first." % cluster_name
        sys.exit(1)
    do_validity_check(opts)

    (masters, slaves) = utils.get_masters_and_slaves(opts.mode)
    if len(masters) <= 0:
        print >> stderr, "ERROR: You have to start as least 1 master"
        sys.exit(1)
    if len(slaves) <= 0:
        print >> stderr, "ERROR: You have to start as least 1 slave"
        sys.exit(1)

    # Now we only support single-node master.
    spark.setup_cluster(masters, slaves, opts, True)
    if opts.enable_spark_notebook:
        spark_notebook.start_spark_notebook(masters, opts)
    if opts.enable_hue:
        hue.start_hue(masters, opts)
    if opts.enable_hdfs:
        hdfs.setup_hdfs(masters, slaves, opts)
    if opts.enable_slave_public_ip:
        utils.save_public_ips(masters, slaves)
    master_ip = ecs.get_instance_info(
        masters[0])['PublicIpAddress']['IpAddress'][0]

    utils.open_nginx(opts, masters)
    utils.end_of_startup(opts, master_ip, masters)
    # update cluster status
    os.system("echo Running > %s%s" % (GlobalVar.CLUSTER_STATUS, cluster_name))
コード例 #2
0
def start_in_client_mode(cluster_name, opts):
    # check cluster status trickly
    if utils.check_cluster_status(cluster_name, ['Running']):
        print "Cluster %s is `Running`, please `Stop` it first." % cluster_name
        sys.exit(1)
    do_validity_check(opts)

    (masters, slaves) = utils.get_masters_and_slaves(opts.mode)
    if len(masters + slaves) <= 0:
        print "There is no master or slave, check it first please."
        sys.exit(1)

    print "==> Restarting spark cluster..."
    if opts.pwd == "":
        opts.pwd = getpass.getpass(
            "You need to provide the password for ECS instance:")
    spark.start_spark_cluster(masters[0], slaves, opts)
    if opts.enable_spark_notebook:
        spark_notebook.start_spark_notebook(masters, opts)
    if opts.enable_hue:
        hue.start_hue(masters, opts)
    if opts.enable_hdfs:
        hdfs.setup_hdfs(masters, slaves, opts)
    if opts.enable_slave_public_ip:
        utils.save_public_ips(masters, slaves)
    master_ip = ecs.get_instance_info(
        masters[0])['PublicIpAddress']['IpAddress'][0]
    utils.open_nginx(opts, masters)
    utils.end_of_startup(opts, master_ip, masters)
    # update cluster status
    os.system("echo Running > %s%s" % (GlobalVar.CLUSTER_STATUS, cluster_name))
コード例 #3
0
def launch_in_cluster_mode(cluster_name, opts):
    # check cluster status trickly
    if utils.check_cluster_status(cluster_name, ['Running', 'Stopped']):
        print "Cluster %s has been launched, please `Destroy` it first." % cluster_name
        sys.exit(1)
    do_validity_check(opts)

    if opts.slaves <= 0:
        print >> stderr, "ERROR: You have to start as least 1 slave"
        sys.exit(1)
    (masters, slaves, master_ip) = utils.launch_cluster(opts, cluster_name)
    utils.wait_for_cluster_state(cluster_state=['Running'],
                                 instances=masters + slaves)
    utils.mount_disk(masters, slaves, opts)
    spark.setup_cluster(masters, slaves, opts, True)
    if opts.enable_spark_notebook:
        spark_notebook.start_spark_notebook(masters, opts)
    if opts.enable_hue:
        hue.start_hue(masters, opts)
    if opts.enable_hdfs:
        hdfs.setup_hdfs(masters, slaves, opts)
    if opts.enable_slave_public_ip:
        utils.save_public_ips(masters, slaves)
    utils.open_nginx(opts, masters)

    utils.end_of_startup(opts, master_ip, masters)
    # update cluster status
    os.system("echo Running > %s%s" % (GlobalVar.CLUSTER_STATUS, cluster_name))
コード例 #4
0
def launch_in_cluster_mode(cluster_name, opts):
    # check cluster status trickly
    if utils.check_cluster_status(cluster_name, ['Running', 'Stopped']):
        print "Cluster %s has been launched, please `Destroy` it first." % cluster_name
        sys.exit(1)
    do_validity_check(opts)

    if opts.slaves <= 0:
        print >> stderr, "ERROR: You have to start as least 1 slave"
        sys.exit(1)
    (masters, slaves, master_ip) = utils.launch_cluster(opts, cluster_name)
    utils.wait_for_cluster_state(
        cluster_state=['Running'],
        instances=masters + slaves)
    utils.mount_disk(masters, slaves, opts)
    spark.setup_cluster(masters, slaves, opts, True)
    if opts.enable_spark_notebook:
        spark_notebook.start_spark_notebook(masters, opts)
    if opts.enable_hue:
        hue.start_hue(masters, opts)
    if opts.enable_hdfs:
        hdfs.setup_hdfs(masters, slaves, opts)
    if opts.enable_slave_public_ip:
        utils.save_public_ips(masters, slaves)
    utils.open_nginx(opts, masters)

    utils.end_of_startup(opts, master_ip, masters)
    # update cluster status
    os.system("echo Running > %s%s" % (GlobalVar.CLUSTER_STATUS, cluster_name))
コード例 #5
0
def start_in_client_mode(cluster_name, opts):
    # check cluster status trickly
    if utils.check_cluster_status(cluster_name, ['Running']):
        print "Cluster %s is `Running`, please `Stop` it first." % cluster_name
        sys.exit(1)
    do_validity_check(opts)

    (masters, slaves) = utils.get_masters_and_slaves(opts.mode)
    if len(masters + slaves) <= 0:
        print "There is no master or slave, check it first please."
        sys.exit(1)

    print "==> Restarting spark cluster..."
    if opts.pwd == "":
        opts.pwd = getpass.getpass("You need to provide the password for ECS instance:")
    spark.start_spark_cluster(masters[0], slaves, opts)
    if opts.enable_spark_notebook:
        spark_notebook.start_spark_notebook(masters, opts)
    if opts.enable_hue:
        hue.start_hue(masters, opts)
    if opts.enable_hdfs:
        hdfs.setup_hdfs(masters, slaves, opts)
    if opts.enable_slave_public_ip:
        utils.save_public_ips(masters, slaves)
    master_ip = ecs.get_instance_info(masters[0])['PublicIpAddress']['IpAddress'][0]
    utils.open_nginx(opts, masters)
    utils.end_of_startup(opts, master_ip, masters)
    # update cluster status
    os.system("echo Running > %s%s" % (GlobalVar.CLUSTER_STATUS, cluster_name))
コード例 #6
0
def launch_in_client_mode(cluster_name, opts):
    # check cluster status trickly
    if utils.check_cluster_status(cluster_name, ['Running', 'Stopped']):
        print "Cluster %s has been launched, please `Destroy` it first." % cluster_name
        sys.exit(1)
    do_validity_check(opts)

    (masters, slaves) = utils.get_masters_and_slaves(opts.mode)
    if len(masters) <= 0:
        print >> stderr, "ERROR: You have to start as least 1 master"
        sys.exit(1)
    if len(slaves) <= 0:
        print >> stderr, "ERROR: You have to start as least 1 slave"
        sys.exit(1)

    # Now we only support single-node master.
    spark.setup_cluster(masters, slaves, opts, True)
    if opts.enable_spark_notebook:
        spark_notebook.start_spark_notebook(masters, opts)
    if opts.enable_hue:
        hue.start_hue(masters, opts)
    if opts.enable_hdfs:
        hdfs.setup_hdfs(masters, slaves, opts)
    if opts.enable_slave_public_ip:
        utils.save_public_ips(masters, slaves)
    master_ip = ecs.get_instance_info(masters[0])['PublicIpAddress']['IpAddress'][0]

    utils.open_nginx(opts, masters)
    utils.end_of_startup(opts, master_ip, masters)
    # update cluster status
    os.system("echo Running > %s%s" % (GlobalVar.CLUSTER_STATUS, cluster_name))
コード例 #7
0
def enable_module(name, opts):
    if len(name.split(":")) != 2:
        print "\nYou need to provide a <cluster_name>:<module_name>\n"
        sys.exit(1)
    cluster_name = name.split(":")[0]
    module_name = name.split(":")[1]
    do_validity_check(opts)
    (masters, slaves) = utils.get_masters_and_slaves(opts.mode, cluster_name)
    if module_name == "hdfs":
        hdfs.setup_hdfs(masters, slaves, opts)
    elif module_name == "hue":
        hue.start_hue(masters, opts)
    elif module_name == "spark-notebook":
        spark_notebook.start_spark_notebook(masters, opts)
    else:
        print "Now we only support 3 module: hdfs, hue, spark-notebook"
        sys.exit(1)
コード例 #8
0
def enable_module(name, opts):
    if len(name.split(":")) != 2:
        print "\nYou need to provide a <cluster_name>:<module_name>\n"
        sys.exit(1)
    cluster_name = name.split(":")[0]
    module_name = name.split(":")[1]
    do_validity_check(opts)
    (masters, slaves) = utils.get_masters_and_slaves(opts.mode, cluster_name)
    if module_name == "hdfs":
        hdfs.setup_hdfs(masters, slaves, opts)
    elif module_name == "hue":
        hue.start_hue(masters, opts)
    elif module_name == "spark-notebook":
        spark_notebook.start_spark_notebook(masters, opts)
    else:
        print "Now we only support 3 module: hdfs, hue, spark-notebook"
        sys.exit(1)