def main():
logging.basicConfig()
opts = parse_args()
try:
conn = ec2.connect_to_region(opts.region)
except Exception as e:
print >> sys.stderr, (e)
sys.exit(1)
if opts.zone == '':
opts.zone = random.choice(conn.get_all_zones()).name
action = opts.action
cluster_name = opts.cluster_name
if action == 'launch':
master_nodes = launch_master(conn, opts)
elif action == 'addslave':
master_nodes = launch_slaves(conn, opts)
elif action == 'addspot':
master_nodes = launch_spot_slaves(conn, opts)
elif action == "get-master":
(master_nodes, slave_nodes) = ec2_util.get_existing_cluster(conn, cluster_name)
print master_nodes[0].private_dns_name
elif action == "login":
(master_nodes, slave_nodes) = ec2_util.get_existing_cluster(conn, cluster_name)
print(master_nodes)
# master = master_nodes[0].public_dns_name
master = master_nodes[0].private_dns_name
subprocess.check_call(
ssh_command(opts) + ['-t', "%s@%s" % (opts.user, master)])
elif action == "forward-port":
(master_nodes, slave_nodes) = ec2_util.get_existing_cluster(conn, cluster_name)
# master = master_nodes[0].public_dns_name
master = master_nodes[0].private_dns_name
subprocess.check_call(
ssh_command(opts) + ['-D', '9595'] + ['-t', "%s@%s" % (opts.user, master)])
# elif action == "stop":
# (master_nodes, slave_nodes) = ec2_util.get_existing_cluster(conn, cluster_name)
# ec2_util.stop_instances(conn, master_nodes)
# ec2_util.stop_instances(conn, slave_nodes)
elif action == "terminate":
(master_nodes, slave_nodes) = ec2_util.get_existing_cluster(conn, cluster_name)
ec2_util.terminate_instances(conn, master_nodes)
ec2_util.terminate_instances(conn, slave_nodes)
delete_groups(conn, cluster_name)
else:
print >> sys.stderr, "Invalid action: %s" % action
sys.exit(1)
def launch_slaves(conn, opts):
"""
Launch slaves of a cluster of the given name, by setting up its security groups,
and starting new instances in them. Fails if instances running in cluster's groups.
:param conn:
:param opts:
:return: tuple of EC2 reservation objects for the master and slaves
"""
cluster_name = opts.cluster_name
if opts.identity_file is None:
print("ERROR: Must provide an identity file (-i) for ssh connections.", file=sys.stderr)
sys.exit(1)
if opts.key_pair is None:
print("ERROR: Must provide a key pair name (-k) to use on instances.", file=sys.stderr)
sys.exit(1)
master_group = ec2_util.get_or_make_group(conn, cluster_name + "-master", False)
slave_group = ec2_util.get_or_make_group(conn, cluster_name + "-slave", False)
# Check if instances are already running in our groups
existing_masters, existing_slaves = ec2_util.get_existing_cluster(conn, cluster_name,
die_on_error=False)
if len(existing_masters) == 0:
print(("ERROR: Cannot find master machine on group" +
"group %s" % (master_group.name)), file=stderr)
sys.exit(1)
if opts.ami is None:
opts.ami = get_ami(opts.instance_type)
print("Launching instances...")
try:
image = conn.get_all_images(image_ids=[opts.ami])[0]
except:
print("Could not find AMI " + opts.ami, file=stderr)
sys.exit(1)
master = existing_masters[0]
block_map = ec2_util.get_block_device(opts.instance_type, 0)
zone = master.placement
slave_res = image.run(key_name=opts.key_pair,
security_groups=[slave_group],
instance_type=opts.instance_type,
placement=zone,
min_count=opts.slaves,
max_count=opts.slaves,
block_device_map=block_map,
user_data=get_user_data('bootstrap.py',
master.private_dns_name,
opts.instance_type,
opts.include_aws_key))
slave_nodes = slave_res.instances
print("Launched %d slaves in %s, regid = %s" % (len(slave_nodes),
zone, slave_res.id))
print('Waiting for slave to getup...')
ec2_util.wait_for_instances(conn, slave_nodes)
for slave in slave_nodes:
slave.add_tag(
key='Name',
value='{cn}-slave-{iid}'.format(cn=cluster_name, iid=slave.id))
print('Done...')
def launch_slaves(conn, opts):
cluster_name = opts.cluster_name
if opts.identity_file is None:
print >> sys.stderr, "ERROR: Must provide an identity file (-i) for ssh connections."
sys.exit(1)
if opts.key_pair is None:
print >> sys.stderr, "ERROR: Must provide a key pair name (-k) to use on instances."
sys.exit(1)
master_group = ec2_util.get_or_make_group(conn, cluster_name + "-master", False)
slave_group = ec2_util.get_or_make_group(conn, cluster_name + "-slave", False)
# Check if instances are already running in our groups
existing_masters, existing_slaves = ec2_util.get_existing_cluster(conn, cluster_name,
die_on_error=False)
if len(existing_masters) == 0:
print >> stderr, ("ERROR: Cannot find master machine on group" +
"group %s" % (master_group.name))
sys.exit(1)
if opts.ami is None:
opts.ami = get_ami(opts.instance_type)
print "Launching instances..."
try:
image = conn.get_all_images(image_ids=[opts.ami])[0]
except:
print >> stderr, "Could not find AMI " + opts.ami
sys.exit(1)
cpe_all_security_group = ec2_util.get_or_make_group(conn, "_CPE_allow_all_from_corp", create=False)
security_group_ids = [master_group.id]
if cpe_all_security_group:
security_group_ids.append(cpe_all_security_group.id)
master = existing_masters[0]
block_map = ec2_util.get_block_device(opts.instance_type, 0)
zone = master.placement
slave_res = image.run(key_name=opts.key_pair,
#security_groups=[slave_group],
instance_type=opts.instance_type,
placement=zone,
min_count=opts.slaves,
max_count=opts.slaves,
block_device_map=block_map,
security_group_ids=security_group_ids,
subnet_id=opts.subnet,
user_data=get_user_data('bootstrap.py',
master.private_dns_name,
opts.instance_type,
opts.include_aws_key))
slave_nodes = slave_res.instances
print "Launched %d slaves in %s, regid = %s" % (len(slave_nodes),
zone, slave_res.id)
print 'Waiting for slave to getup...'
ec2_util.wait_for_instances(conn, slave_nodes)
for slave in slave_nodes:
slave.add_tag(
key='Name',
value='{cn}-slave-{iid}'.format(cn=cluster_name, iid=slave.id))
print 'Done...'
def launch_spot_slaves(conn, opts):
"""
Launch spot instance slaves of named cluster, set up security groups, start new instances in them.
Fails if there already instances running in the cluster's groups.
:param conn:
:param opts:
:return: tuple of EC2 reservation objects for the master and slaves
"""
vcpu, vram, price = get_resource_map()
cluster_name = opts.cluster_name
if opts.identity_file is None:
print("ERROR: Must provide an identity file (-i) for ssh connections.", file=sys.stderr)
sys.exit(1)
if opts.spot_price is None:
opts.spot_price = price[opts.instance_type]
print("Spot price is not specified, bid the full price=%g for %s" % (opts.spot_price, opts.instance_type))
if opts.key_pair is None:
print("ERROR: Must provide a key pair name (-k) to use on instances.", file=sys.stderr)
sys.exit(1)
master_group = ec2_util.get_or_make_group(conn, cluster_name + "-master", False)
slave_group = ec2_util.get_or_make_group(conn, cluster_name + "-slave", False)
# Check if instances are already running in our groups
existing_masters, existing_slaves = ec2_util.get_existing_cluster(conn, cluster_name,
die_on_error=False)
if len(existing_masters) == 0:
print(("ERROR: Cannot find master machine on group" +
"group %s" % (master_group.name)), file=stderr)
sys.exit(1)
if opts.ami is None:
opts.ami = get_ami(opts.instance_type)
print("Launching Spot instances type=%s, price=%g..." % (opts.instance_type, opts.spot_price))
master = existing_masters[0]
block_map = ec2_util.get_block_device(opts.instance_type, 0)
zone = master.placement
slave_reqs = conn.request_spot_instances(
price=opts.spot_price,
image_id=opts.ami,
launch_group="launch-group-%s" % cluster_name,
placement=zone,
count=opts.slaves,
key_name=opts.key_pair,
security_groups=[slave_group],
instance_type=opts.instance_type,
block_device_map=block_map,
user_data=get_user_data('bootstrap.py', master.private_dns_name, opts.instance_type, opts.include_aws_key).encode('utf-8'))
print('Done... request is submitted')
def main():
logging.basicConfig()
opts = parse_args()
try:
conn = ec2.connect_to_region(opts.region)
except Exception as e:
print(e, file=sys.stderr)
sys.exit(1)
if opts.zone == '':
opts.zone = random.choice(conn.get_all_zones()).name
action = opts.action
cluster_name = opts.cluster_name
if action == 'launch':
master_nodes = launch_master(conn, opts)
elif action == 'addslave':
master_nodes = launch_slaves(conn, opts)
elif action == 'addspot':
master_nodes = launch_spot_slaves(conn, opts)
elif action == "get-master":
(master_nodes, slave_nodes) = ec2_util.get_existing_cluster(conn, cluster_name)
print(master_nodes[0].public_dns_name)
elif action == "login":
(master_nodes, slave_nodes) = ec2_util.get_existing_cluster(conn, cluster_name)
master = master_nodes[0].public_dns_name
subprocess.check_call(
ssh_command(opts) + ['-t', "%s@%s" % (opts.user, master)])
elif action == "forward-port":
(master_nodes, slave_nodes) = ec2_util.get_existing_cluster(conn, cluster_name)
master = master_nodes[0].public_dns_name
subprocess.check_call(
ssh_command(opts) + ['-D', '9595'] + ['-t', "%s@%s" % (opts.user, master)])
else:
print("Invalid action: %s" % action, file=sys.stderr)
sys.exit(1)
def launch_spot_slaves(conn, opts):
vcpu, vram, price = get_resource_map()
cluster_name = opts.cluster_name
if opts.identity_file is None:
print >> sys.stderr, "ERROR: Must provide an identity file (-i) for ssh connections."
sys.exit(1)
if opts.spot_price is None:
opts.spot_price = price[opts.instance_type]
print "Spot price is not specified, bid the full price=%g for %s" % (
opts.spot_price, opts.instance_type)
if opts.key_pair is None:
print >> sys.stderr, "ERROR: Must provide a key pair name (-k) to use on instances."
sys.exit(1)
master_group = ec2_util.get_or_make_group(conn, cluster_name + "-master",
False)
slave_group = ec2_util.get_or_make_group(conn, cluster_name + "-slave",
False)
# Check if instances are already running in our groups
existing_masters, existing_slaves = ec2_util.get_existing_cluster(
conn, cluster_name, die_on_error=False)
if len(existing_masters) == 0:
print >> stderr, ("ERROR: Cannot find master machine on group" +
"group %s" % (master_group.name))
sys.exit(1)
if opts.ami is None:
opts.ami = get_ami(opts.instance_type)
print "Launching Spot instances type=%s, price=%g..." % (
opts.instance_type, opts.spot_price)
master = existing_masters[0]
block_map = ec2_util.get_block_device(opts.instance_type, 0)
zone = master.placement
slave_reqs = conn.request_spot_instances(
price=opts.spot_price,
image_id=opts.ami,
launch_group="launch-group-%s" % cluster_name,
placement=zone,
count=opts.slaves,
key_name=opts.key_pair,
security_groups=[slave_group],
instance_type=opts.instance_type,
block_device_map=block_map,
user_data=get_user_data('bootstrap.py', master.private_dns_name,
opts.instance_type, opts.include_aws_key))
print 'Done... request is submitted'
def launch_spot_slaves(conn, opts):
vcpu, vram, price = get_resource_map()
cluster_name = opts.cluster_name
if opts.identity_file is None:
print >> sys.stderr, "ERROR: Must provide an identity file (-i) for ssh connections."
sys.exit(1)
if opts.spot_price is None:
opts.spot_price = price[opts.instance_type]
print "Spot price is not specified, bid the full price=%g for %s" % (opts.spot_price, opts.instance_type)
if opts.key_pair is None:
print >> sys.stderr, "ERROR: Must provide a key pair name (-k) to use on instances."
sys.exit(1)
master_group = ec2_util.get_or_make_group(conn, cluster_name + "-master", False)
slave_group = ec2_util.get_or_make_group(conn, cluster_name + "-slave", False)
# Check if instances are already running in our groups
existing_masters, existing_slaves = ec2_util.get_existing_cluster(conn, cluster_name,
die_on_error=False)
if len(existing_masters) == 0:
print >> stderr, ("ERROR: Cannot find master machine on group" +
"group %s" % (master_group.name))
sys.exit(1)
if opts.ami is None:
opts.ami = get_ami(opts.instance_type)
print "Launching Spot instances type=%s, price=%g..." % (opts.instance_type, opts.spot_price)
master = existing_masters[0]
block_map = ec2_util.get_block_device(opts.instance_type, 0)
zone = master.placement
slave_reqs = conn.request_spot_instances(
price=opts.spot_price,
image_id=opts.ami,
launch_group="launch-group-%s" % cluster_name,
placement=zone,
count=opts.slaves,
key_name=opts.key_pair,
security_groups=[slave_group],
instance_type=opts.instance_type,
block_device_map=block_map,
user_data=get_user_data('bootstrap.py',
master.private_dns_name,
opts.instance_type,
opts.include_aws_key))
print 'Done... request is submitted'
def launch_master(conn, opts):
cluster_name = opts.cluster_name
if opts.identity_file is None:
print >> stderr, "ERROR: Must provide an identity file (-i) for ssh connections."
sys.exit(1)
if opts.key_pair is None:
print >> stderr, "ERROR: Must provide a key pair name (-k) to use on instances."
sys.exit(1)
print "Setting up security groups..."
master_group = ec2_util.get_or_make_group(conn, cluster_name + "-master")
slave_group = ec2_util.get_or_make_group(conn, cluster_name + "-slave")
if master_group.rules == []: # Group was just now created
master_group.authorize(src_group=master_group)
master_group.authorize(src_group=slave_group)
master_group.authorize('tcp', 22, 22, '0.0.0.0/0')
master_group.authorize('tcp', 8000, 8100, '0.0.0.0/0')
master_group.authorize('tcp', 9000, 9999, '0.0.0.0/0')
master_group.authorize('tcp', 18080, 18080, '0.0.0.0/0')
master_group.authorize('tcp', 19999, 19999, '0.0.0.0/0')
master_group.authorize('tcp', 50000, 50100, '0.0.0.0/0')
master_group.authorize('tcp', 60070, 60070, '0.0.0.0/0')
master_group.authorize('tcp', 4040, 4045, '0.0.0.0/0')
master_group.authorize('tcp', 5080, 5080, '0.0.0.0/0')
master_group.authorize('udp', 0, 65535, '0.0.0.0/0')
if slave_group.rules == []: # Group was just now created
slave_group.authorize(src_group=master_group)
slave_group.authorize(src_group=slave_group)
slave_group.authorize('tcp', 22, 22, '0.0.0.0/0')
slave_group.authorize('tcp', 8000, 8100, '0.0.0.0/0')
slave_group.authorize('tcp', 9000, 9999, '0.0.0.0/0')
slave_group.authorize('tcp', 50000, 50100, '0.0.0.0/0')
slave_group.authorize('tcp', 60060, 60060, '0.0.0.0/0')
slave_group.authorize('tcp', 60075, 60075, '0.0.0.0/0')
slave_group.authorize('udp', 0, 65535, '0.0.0.0/0')
# Check if instances are already running in our groups
existing_masters, existing_slaves = ec2_util.get_existing_cluster(
conn, cluster_name, die_on_error=False)
if existing_slaves:
print >> stderr, ("ERROR: There are already instances running in " +
"group %s or %s" % (group.name, slave_group.name))
sys.exit(1)
if opts.ami is None:
opts.ami = get_ami(opts.instance_type)
print "Launching instances..."
try:
image = conn.get_all_images(image_ids=[opts.ami])[0]
except:
print >> stderr, "Could not find AMI " + opts.ami
sys.exit(1)
# Launch or resume masters
if existing_masters:
print "Starting master..."
for inst in existing_masters:
if inst.state not in ["shutting-down", "terminated"]:
inst.start()
master_nodes = existing_masters
else:
# Create block device mapping so that we can add an EBS volume if asked to
block_map = ec2_util.get_block_device(opts.instance_type, 0)
master_type = opts.instance_type
if opts.zone == 'all':
opts.zone = random.choice(conn.get_all_zones()).name
master_res = image.run(key_name=opts.key_pair,
security_groups=[master_group],
instance_type=master_type,
placement=opts.zone,
min_count=1,
max_count=1,
block_device_map=block_map,
user_data=get_user_data('bootstrap.py', '',
master_type,
opts.include_aws_key))
master_nodes = master_res.instances
print "Launched master in %s, regid = %s" % (opts.zone, master_res.id)
print 'Waiting for master to getup...'
ec2_util.wait_for_instances(conn, master_nodes)
# Give the instances descriptive names
for master in master_nodes:
master.add_tag(key='Name',
value='{cn}-master-{iid}'.format(cn=cluster_name,
iid=master.id))
master = master_nodes[0].public_dns_name
print 'finishing getting master %s' % master
# Return all the instances
return master_nodes
def launch_master(conn, opts):
"""
Launch master, set up security groups,and start new instances.
Fails if there instances running in the cluster's security groups.
:param conn:
:param opts:
:return: tuple of EC2 reservation objects for the master and slaves
"""
cluster_name = opts.cluster_name
if opts.identity_file is None:
print('ERROR: Must provide an identity file (-i) for ssh connections.', file=stderr)
sys.exit(1)
if opts.key_pair is None:
print('ERROR: Must provide a key pair name (-k) to use on instances.', file=stderr)
sys.exit(1)
print("Setting up security groups...")
master_group = ec2_util.get_or_make_group(conn, cluster_name + "-master")
slave_group = ec2_util.get_or_make_group(conn, cluster_name + "-slave")
if master_group.rules == []: # Group was just now created
master_group.authorize(src_group=master_group)
master_group.authorize(src_group=slave_group)
master_group.authorize('tcp', 22, 22, '0.0.0.0/0')
master_group.authorize('tcp', 8000, 8100, '0.0.0.0/0')
master_group.authorize('tcp', 9000, 9999, '0.0.0.0/0')
master_group.authorize('tcp', 18080, 18080, '0.0.0.0/0')
master_group.authorize('tcp', 19999, 19999, '0.0.0.0/0')
master_group.authorize('tcp', 50000, 50100, '0.0.0.0/0')
master_group.authorize('tcp', 60070, 60070, '0.0.0.0/0')
master_group.authorize('tcp', 4040, 4045, '0.0.0.0/0')
master_group.authorize('tcp', 5080, 5080, '0.0.0.0/0')
master_group.authorize('udp', 0, 65535, '0.0.0.0/0')
if slave_group.rules == []: # Group was just now created
slave_group.authorize(src_group=master_group)
slave_group.authorize(src_group=slave_group)
slave_group.authorize('tcp', 22, 22, '0.0.0.0/0')
slave_group.authorize('tcp', 8000, 8100, '0.0.0.0/0')
slave_group.authorize('tcp', 9000, 9999, '0.0.0.0/0')
slave_group.authorize('tcp', 50000, 50100, '0.0.0.0/0')
slave_group.authorize('tcp', 60060, 60060, '0.0.0.0/0')
slave_group.authorize('tcp', 60075, 60075, '0.0.0.0/0')
slave_group.authorize('udp', 0, 65535, '0.0.0.0/0')
# Check if instances are already running in our groups
existing_masters, existing_slaves = ec2_util.get_existing_cluster(conn, cluster_name, die_on_error=False)
if existing_slaves:
print(('ERROR: There are already instances running in group %s or %s' % (master_group.name, slave_group.name)), file=stderr)
sys.exit(1)
if opts.ami is None:
opts.ami = get_ami(opts.instance_type)
print("Launching instances...")
try:
image = conn.get_all_images(image_ids=[opts.ami])[0]
except:
print("Could not find AMI " + opts.ami, file=stderr)
sys.exit(1)
# Launch or resume masters
if existing_masters:
print("Starting master...")
for inst in existing_masters:
if inst.state not in ["shutting-down", "terminated"]:
inst.start()
master_nodes = existing_masters
else:
# Create block device mapping so that we can add an EBS volume if asked to
block_map = ec2_util.get_block_device(opts.instance_type, 0)
master_type = opts.instance_type
if opts.zone == 'all':
opts.zone = random.choice(conn.get_all_zones()).name
master_res = image.run(key_name=opts.key_pair,
security_groups=[master_group],
instance_type=master_type,
placement=opts.zone,
min_count=1,
max_count=1,
block_device_map=block_map,
user_data=get_user_data('bootstrap.py', '',
master_type, opts.include_aws_key))
master_nodes = master_res.instances
print("Launched master in %s, regid = %s" % (opts.zone, master_res.id))
print('Waiting for master to getup...')
ec2_util.wait_for_instances(conn, master_nodes)
# Give the instances descriptive names
for master in master_nodes:
master.add_tag(
key='Name',
value='{cn}-master-{iid}'.format(cn=cluster_name, iid=master.id))
master = master_nodes[0].public_dns_name
print('finishing getting master %s' % master)
# Return all the instances
return master_nodes