Python SparkBuildManager Examples

Example #1

File: main.py Project: xiashuijun/spark-perf-1

# Restart Master and Workers
should_restart_cluster = config.RESTART_SPARK_CLUSTER
# Copy all the files in SPARK_HOME
should_rsync_spark_home = config.RSYNC_SPARK_HOME

# Check that commit ID's are specified in config_file.
if should_prep_spark:
    assert config.SPARK_COMMIT_ID is not "", \
        ("Please specify SPARK_COMMIT_ID in %s" % args.config_file)

# If a cluster is already running from the Spark EC2 scripts, try shutting it down.
if os.path.exists(config.SPARK_HOME_DIR
                  ) and should_restart_cluster and not config.IS_MESOS_MODE:
    Cluster(spark_home=config.SPARK_HOME_DIR).stop()

spark_build_manager = SparkBuildManager("%s/spark-build-cache" % PROJ_DIR,
                                        config.SPARK_GIT_REPO)

if config.IS_MESOS_MODE:
    cluster = MesosCluster(spark_home=config.SPARK_HOME_DIR,
                           spark_conf_dir=config.SPARK_CONF_DIR,
                           mesos_master=config.SPARK_CLUSTER_URL)
elif config.USE_CLUSTER_SPARK:
    cluster = Cluster(spark_home=config.SPARK_HOME_DIR,
                      spark_conf_dir=config.SPARK_CONF_DIR)
else:
    cluster = spark_build_manager.get_cluster(
        config.SPARK_COMMIT_ID, config.SPARK_CONF_DIR,
        config.SPARK_MERGE_COMMIT_INTO_MASTER)

# rsync Spark to all nodes in case there is a change in Worker config
if should_restart_cluster and should_rsync_spark_home:

Example #2

File: main.py Project: yangspeaking/spark-perf

# Restart Master and Workers
should_restart_cluster = config.RESTART_SPARK_CLUSTER
# Copy all the files in SPARK_HOME
should_rsync_spark_home = config.RSYNC_SPARK_HOME

# Check that commit ID's are specified in config_file.
if should_prep_spark:
    assert config.SPARK_COMMIT_ID is not "", \
        ("Please specify SPARK_COMMIT_ID in %s" % args.config_file)

# If a cluster is already running from the Spark EC2 scripts, try shutting it down.
if os.path.exists(config.SPARK_HOME_DIR
                  ) and should_restart_cluster and not config.IS_MESOS_MODE:
    Cluster(spark_home=config.SPARK_HOME_DIR).stop()

spark_build_manager = SparkBuildManager("%s/spark-build-cache" % PROJ_DIR,
                                        config.SPARK_GIT_REPO)

if config.IS_MESOS_MODE:
    cluster = MesosCluster(spark_home=config.SPARK_HOME_DIR,
                           spark_conf_dir=config.SPARK_CONF_DIR,
                           mesos_master=config.SPARK_CLUSTER_URL)
elif config.USE_CLUSTER_SPARK:
    cluster = Cluster(spark_home=config.SPARK_HOME_DIR,
                      spark_conf_dir=config.SPARK_CONF_DIR)
else:
    cluster = spark_build_manager.get_cluster(
        commit_id=config.SPARK_COMMIT_ID,
        conf_dir=config.SPARK_CONF_DIR,
        merge_commit_into_master=config.SPARK_MERGE_COMMIT_INTO_MASTER,
        is_yarn_mode=config.IS_YARN_MODE,
        additional_make_distribution_args=args.

Example #3

File: main.py Project: HuiHuiJane/spark-perf

# Restart Master and Workers
should_restart_cluster = config.RESTART_SPARK_CLUSTER
# Copy all the files in SPARK_HOME 
should_rsync_spark_home = config.RSYNC_SPARK_HOME

# Check that commit ID's are specified in config_file.
if should_prep_spark:
    assert config.SPARK_COMMIT_ID is not "", \
        ("Please specify SPARK_COMMIT_ID in %s" % args.config_file)

# If a cluster is already running from the Spark EC2 scripts, try shutting it down.
if os.path.exists(config.SPARK_HOME_DIR) and should_restart_cluster and not config.IS_MESOS_MODE:
    Cluster(spark_home=config.SPARK_HOME_DIR).stop()

spark_build_manager = SparkBuildManager("%s/spark-build-cache" % PROJ_DIR, config.SPARK_GIT_REPO)

if config.IS_MESOS_MODE:
    cluster = MesosCluster(spark_home=config.SPARK_HOME_DIR, spark_conf_dir=config.SPARK_CONF_DIR,
                           mesos_master=config.SPARK_CLUSTER_URL)
elif config.USE_CLUSTER_SPARK:
    cluster = Cluster(spark_home=config.SPARK_HOME_DIR, spark_conf_dir=config.SPARK_CONF_DIR)
else:
    cluster = spark_build_manager.get_cluster(config.SPARK_COMMIT_ID, config.SPARK_CONF_DIR,
                                              config.SPARK_MERGE_COMMIT_INTO_MASTER, config.IS_YARN_MODE)

# rsync Spark to all nodes in case there is a change in Worker config
if should_restart_cluster and should_rsync_spark_home:
    cluster.sync_spark()

# If a cluster is already running from an earlier test, try shutting it down.

Example #4

File: main.py Project: AlexGKing/spark-perf

# Restart Master and Workers
should_restart_cluster = config.RESTART_SPARK_CLUSTER
# Copy all the files in SPARK_HOME 
should_rsync_spark_home = config.RSYNC_SPARK_HOME

# Check that commit ID's are specified in config_file.
if should_prep_spark:
    assert config.SPARK_COMMIT_ID is not "", \
        ("Please specify SPARK_COMMIT_ID in %s" % args.config_file)

# If a cluster is already running from the Spark EC2 scripts, try shutting it down.
if os.path.exists(config.SPARK_HOME_DIR) and should_restart_cluster and not config.IS_MESOS_MODE:
    Cluster(spark_home=config.SPARK_HOME_DIR).stop()

spark_build_manager = SparkBuildManager("%s/spark-build-cache" % PROJ_DIR, config.SPARK_GIT_REPO)

if config.IS_MESOS_MODE:
    cluster = MesosCluster(spark_home=config.SPARK_HOME_DIR, spark_conf_dir=config.SPARK_CONF_DIR,
                           mesos_master=config.SPARK_CLUSTER_URL)
elif config.USE_CLUSTER_SPARK:
    cluster = Cluster(spark_home=config.SPARK_HOME_DIR, spark_conf_dir=config.SPARK_CONF_DIR)
else:
    cluster = spark_build_manager.get_cluster(
        commit_id=config.SPARK_COMMIT_ID,
        conf_dir=config.SPARK_CONF_DIR,
        merge_commit_into_master=config.SPARK_MERGE_COMMIT_INTO_MASTER,
        is_yarn_mode=config.IS_YARN_MODE,
        additional_make_distribution_args=args.additional_make_distribution_args)

# rsync Spark to all nodes in case there is a change in Worker config