sys.exit(1)
(action, cluster_name) = args
# Launch a cluster, setting several options to defaults
# (use spark-ec2.py included with Spark for more control)
if action == "launch":
try:
conn = ec2.connect_to_region(opts.region)
except Exception as e:
print >> stderr, (e)
sys.exit(1)
if opts.zone == "":
opts.zone = random.choice(conn.get_all_zones()).name
opts.ami = get_spark_ami(opts) # "ami-3ecd0c56"
opts.ebs_vol_size = 0
opts.master_instance_type = ""
opts.hadoop_major_version = "1"
opts.ganglia = True
opts.spark_version = "1.1.0"
opts.swap = 1024
opts.worker_instances = 1
opts.master_opts = ""
opts.user_data = ""
if opts.resume:
(master_nodes, slave_nodes) = get_existing_cluster(conn, opts, cluster_name)
else:
(master_nodes, slave_nodes) = launch_cluster(conn, opts, cluster_name)
sys.exit(1)
(action, cluster_name) = args
# Launch a cluster, setting several options to defaults
# (use spark-ec2.py included with Spark for more control)
if action == "launch":
try:
conn = ec2.connect_to_region(opts.region)
except Exception as e:
print >> stderr, (e)
sys.exit(1)
if opts.zone == "":
opts.zone = random.choice(conn.get_all_zones()).name
opts.ami = get_spark_ami(opts)
opts.ebs_vol_size = 0
opts.spot_price = None
opts.master_instance_type = ""
opts.wait = 160
opts.hadoop_major_version = "1"
opts.ganglia = True
opts.spark_version = "1.0.0"
opts.swap = 1024
opts.worker_instances = 1
opts.master_opts = ""
if opts.resume:
(master_nodes, slave_nodes) = get_existing_cluster(conn, opts, cluster_name)
else:
(master_nodes, slave_nodes) = launch_cluster(conn, opts, cluster_name)
sys.exit(1)
(action, cluster_name) = args
# Launch a cluster, setting several options to defaults
# (use spark-ec2.py included with Spark for more control)
if action == "launch":
try:
conn = ec2.connect_to_region(opts.region)
except Exception as e:
print >> stderr, (e)
sys.exit(1)
if opts.zone == "":
opts.zone = random.choice(conn.get_all_zones()).name
opts.ami = get_spark_ami(opts)
opts.ebs_vol_size = 0
opts.spot_price = None
opts.master_instance_type = ""
opts.wait = 160
opts.hadoop_major_version = "1"
opts.ganglia = True
opts.spark_version = "0.9.0"
opts.swap = 1024
if opts.resume:
(master_nodes,
slave_nodes) = get_existing_cluster(conn, opts, cluster_name)
else:
(master_nodes,
slave_nodes) = launch_cluster(conn, opts, cluster_name)
sys.exit(1)
(action, cluster_name) = args
spark_version_string = opts.spark_version
# check that requested spark version is <= the $SPARK_HOME version, or is a github hash
if '.' in spark_version_string:
# version string is dotted, not a hash
spark_cluster_loose_version = LooseVersion(spark_version_string)
if spark_cluster_loose_version > spark_home_loose_version:
raise ValueError("Requested cluster Spark version '%s' is greater " % spark_version_string
+ "than the local version of Spark in $SPARK_HOME, '%s'" % spark_home_version_string)
if spark_cluster_loose_version < LooseVersion(MINIMUM_SPARK_VERSION):
raise ValueError("Requested cluster Spark version '%s' is less " % spark_version_string
+ "than the minimum version required for Thunder, '%s'" % MINIMUM_SPARK_VERSION)
opts.ami = get_spark_ami(opts) # "ami-3ecd0c56"\
# get version string as github commit hash if needed (mainly to support Spark release candidates)
opts.spark_version = remap_spark_version_to_hash(spark_version_string)
# Launch a cluster, setting several options to defaults
# (use spark-ec2.py included with Spark for more control)
if action == "launch":
try:
conn = ec2.connect_to_region(opts.region)
except Exception as e:
print >> stderr, (e)
sys.exit(1)
if opts.zone == "":
opts.zone = random.choice(conn.get_all_zones()).name