def print_info(args):
all_cc = common.ecluster_config(args.econfig)
print("Available clusters: %s" % ",".join(all_cc.cluster_conf.keys()))
print()
print("Configuration for cluster '%s':" % (args.cluster))
cluster_config = common.ecluster_config(args.econfig, args.cluster)
_cluster_info(cluster_config)
print()
print("AWS setup:")
_iam_info()
_sg_info(cluster_config)
_vpc_info(cluster_config)
print()
_instance_info(cluster_config)
def _setup_vpc(args):
cidr_regex = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/\d{1,2}$'
if not re.search(cidr_regex, args.network):
sys.stderr.write(
'Network {} is not in CIDR (a.b.c.d/e) format.\n'.format(
args.network))
sys.exit(1)
net, mask = args.network.split('/')
if int(mask) > 23:
sys.stderr.write('Network must be at least a /23 in size.\n')
sys.exit(1)
compute_subnet = '{}/24'.format(net)
cluster_config = common.ecluster_config(args.econfig, args.cluster)
conn = boto.connect_vpc(
aws_access_key_id=cluster_config['cloud']['ec2_access_key'],
aws_secret_access_key=cluster_config['cloud']['ec2_secret_key'])
existing_vpcs = conn.get_all_vpcs(filters={'tag:Name': args.cluster})
if existing_vpcs:
if args.recreate:
raise NotImplementedError(
"bcbio does not currently remove VPCs. "
"The easiest way is to do this manually in the console: "
"https://console.aws.amazon.com/vpc/home")
# FIXME: this doesn't automatically remove resources in the VPC
# like the AWS management console does.
conn.delete_vpc(existing_vpcs[0].id)
else:
print(
'VPC {} already exists. Skipping. Use --recreate to re-create if needed.'
.format(args.cluster))
return
vpc = conn.create_vpc(args.network)
vpc.add_tag('Name', args.cluster)
sg = conn.create_security_group('{}_cluster_sg'.format(args.cluster),
'bcbio cluster nodes', vpc.id)
sg.authorize(ip_protocol='tcp',
from_port=22,
to_port=22,
cidr_ip='0.0.0.0/0')
sg.authorize(ip_protocol='-1', src_group=sg)
igw = conn.create_internet_gateway()
igw.add_tag('Name', '{}_gw'.format(args.cluster))
conn.attach_internet_gateway(igw.id, vpc.id)
rtb = conn.create_route_table(vpc.id)
rtb.add_tag('Name', '{}_rtable'.format(args.cluster))
conn.create_route(rtb.id, '0.0.0.0/0', igw.id)
subnet = conn.create_subnet(vpc.id, compute_subnet)
subnet.add_tag('Name', '{}_cluster'.format(args.cluster))
conn.associate_route_table(rtb.id, subnet.id)
print("Created VPC: %s" % args.cluster)
return args.cluster
def run_remote(args):
"""Run a script on the frontend node inside a screen session."""
config = common.ecluster_config(args.econfig)
cluster = config.load_cluster(args.cluster)
frontend = cluster.get_frontend_node()
client = frontend.connect(known_hosts_file=cluster.known_hosts_file)
cmd = "echo $HOME"
stdin, stdout, stderr = client.exec_command(cmd)
remote_home = stdout.read().strip()
remote_file = os.path.join(remote_home, os.path.basename(args.script))
log_file = "%s.log" % os.path.splitext(remote_file)[0]
screen_name = os.path.splitext(os.path.basename(remote_file))[0]
sftp = client.open_sftp()
sftp.put(args.script, remote_file)
sftp.close()
cmd = "chmod a+x %s" % remote_file
client.exec_command(cmd)
cmd = "screen -d -m -S {} bash -c '{} &> {}'".format(
screen_name, remote_file, log_file)
stdin, stdout, stderr = client.exec_command(cmd)
stdout.read()
client.close()
print("Running {} on AWS in screen session {}".format(
remote_file, screen_name))
def run_remote(args):
"""Run a script on the frontend node inside a screen session."""
config = common.ecluster_config(args.econfig)
cluster = config.load_cluster(args.cluster)
frontend = cluster.get_frontend_node()
client = frontend.connect(known_hosts_file=cluster.known_hosts_file)
cmd = "echo $HOME"
stdin, stdout, stderr = client.exec_command(cmd)
remote_home = stdout.read().strip()
remote_file = os.path.join(remote_home, os.path.basename(args.script))
log_file = "%s.log" % os.path.splitext(remote_file)[0]
screen_name = os.path.splitext(os.path.basename(remote_file))[0]
sftp = client.open_sftp()
sftp.put(args.script, remote_file)
sftp.close()
cmd = "chmod a+x %s" % remote_file
client.exec_command(cmd)
cmd = "screen -d -m -S {} bash -c '{} &> {}'".format(
screen_name, remote_file, log_file)
stdin, stdout, stderr = client.exec_command(cmd)
stdout.read()
client.close()
print("Running {} on AWS in screen session {}".format(
remote_file, screen_name))
def _bootstrap_bcbio(args, ansible_base):
"""Install bcbio_vm and docker container with tools. Set core and memory usage.
"""
cluster = common.ecluster_config(args.econfig).load_cluster(args.cluster)
inventory_path = os.path.join(cluster.repository.storage_path,
'ansible-inventory.{}'.format(args.cluster))
playbook_path = os.path.join(ansible_base, "roles", "bcbio_bootstrap",
"tasks", "main.yml")
def _extra_vars(args, cluster_config):
# Calculate cores and memory
compute_nodes = int(
tz.get_in(["nodes", "frontend", "compute_nodes"], cluster_config,
0))
if compute_nodes > 0:
machine = tz.get_in(["nodes", "compute", "flavor"], cluster_config)
else:
machine = tz.get_in(["nodes", "frontend", "flavor"],
cluster_config)
cores, mem = AWS_INFO[machine]
cores = ipython.per_machine_target_cores(cores, compute_nodes)
return {
"target_cores": cores,
"target_memory": mem,
"upgrade_host_os_and_reboot": not args.no_reboot
}
common.run_ansible_pb(inventory_path, playbook_path, args, _extra_vars)
def _bootstrap_nfs(args, ansible_base):
"""Mount encrypted NFS volume on master node and expose across worker nodes.
"""
cluster = common.ecluster_config(args.econfig).load_cluster(args.cluster)
inventory_path = os.path.join(cluster.repository.storage_path,
'ansible-inventory.{}'.format(args.cluster))
nfs_clients = []
with open(inventory_path) as in_handle:
for line in in_handle:
if line.startswith("frontend"):
nfs_server = line.split()[0]
elif line.startswith("compute"):
nfs_clients.append(line.split()[0])
playbook_path = os.path.join(ansible_base, "roles", "encrypted_nfs",
"tasks", "main.yml")
def _extra_vars(args, cluster_config):
return {
"encrypted_mount":
"/encrypted",
"nfs_server":
nfs_server,
"nfs_clients":
",".join(nfs_clients),
"login_user":
tz.get_in(["nodes", "frontend", "login"], cluster_config),
"encrypted_device":
tz.get_in(["nodes", "frontend", "encrypted_volume_device"],
cluster_config, "/dev/xvdf")
}
common.run_ansible_pb(inventory_path, playbook_path, args, _extra_vars)
def create(args):
if args.network:
cidr_regex = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/\d{1,2}$'
if not re.search(cidr_regex, args.network):
sys.stderr.write(
'Network {} is not in CIDR (a.b.c.d/e) format.\n'.format(
args.network))
sys.exit(1)
config = common.ecluster_config(args.econfig)
cluster_config = config.cluster_conf[args.cluster]
try:
cluster = config.load_cluster(args.cluster)
cluster_storage_path = cluster.repository.storage_path
except elasticluster.exceptions.ClusterNotFound:
# Assume the default storage path if the cluster doesn't exist,
# so we can start an ICEL stack in parallel with cluster startup.
cluster_storage_path = common.get_storage_dir(args.econfig)
if not os.path.exists(cluster_storage_path):
os.makedirs(cluster_storage_path)
if not args.setup:
icel_param = {
'oss_count': args.oss_count,
'ost_vol_size': args.size / args.oss_count / args.lun_count,
'ost_vol_count': args.lun_count,
}
template_url = _upload_icel_cf_template(icel_param, args.bucket,
cluster_config['cloud'])
_create_stack(args.stack_name, template_url, args.network,
args.cluster, cluster_config, args.recreate)
try:
sys.stdout.write('Waiting for stack to launch (this will take '
'a few minutes)')
sys.stdout.flush()
_wait_for_stack(args.stack_name, 'CREATE_COMPLETE', 15 * 60,
cluster_config['cloud'])
except Exception as e:
sys.stderr.write('{}\n'.format(str(e)))
sys.exit(1)
ssh_config_path = os.path.join(
cluster_storage_path, 'icel-{}.ssh_config'.format(args.stack_name))
_write_ssh_config(ssh_config_path, args.stack_name, cluster_config)
ansible_config_path = os.path.join(
cluster_storage_path, 'icel-{}.ansible_config'.format(args.stack_name))
_write_ansible_config(ansible_config_path, args.stack_name,
cluster_storage_path)
inventory_path = os.path.join(cluster_storage_path,
'icel-{}.inventory'.format(args.stack_name))
_write_inventory(inventory_path, args.stack_name, cluster_config['cloud'])
playbook_path = os.path.join(common.ANSIBLE_BASE, "roles", "icel", "tasks",
"main.yml")
common.run_ansible_pb(inventory_path,
playbook_path,
args,
ansible_cfg=ansible_config_path)
def fetch_collectl(econfig_file, cluster_name, datadir, verbose=False):
config = ecluster_config(econfig_file)
cluster = config.load_cluster(cluster_name)
keys = set()
for type in cluster.nodes:
for node in cluster.nodes[type]:
keys.add(node.user_key_private)
with ssh_agent(keys):
ssh = paramiko.client.SSHClient()
ssh.set_missing_host_key_policy(paramiko.client.RejectPolicy())
ssh.load_host_keys(cluster.known_hosts_file)
for node in cluster.get_all_nodes():
if not node.preferred_ip:
# Instance is unavailable.
continue
_pull_collectl_data(
node.preferred_ip, node.image_user, datadir, ssh,
verbose=verbose)
# FIXME: load SSH host keys from ICEL instances.
ssh.set_missing_host_key_policy(paramiko.client.AutoAddPolicy())
aws_config = config.cluster_conf[cluster_name]['cloud']
_fetch_collectl_lustre(cluster, ssh, datadir, aws_config, verbose)
def fetch_collectl(econfig_file, cluster_name, datadir, verbose=False):
config = ecluster_config(econfig_file)
cluster = config.load_cluster(cluster_name)
keys = set()
for type in cluster.nodes:
for node in cluster.nodes[type]:
keys.add(node.user_key_private)
with ssh_agent(keys):
ssh = paramiko.client.SSHClient()
ssh.set_missing_host_key_policy(paramiko.client.RejectPolicy())
ssh.load_host_keys(cluster.known_hosts_file)
for node in cluster.get_all_nodes():
if not node.preferred_ip:
# Instance is unavailable.
continue
_pull_collectl_data(node.preferred_ip,
node.image_user,
datadir,
ssh,
verbose=verbose)
# FIXME: load SSH host keys from ICEL instances.
ssh.set_missing_host_key_policy(paramiko.client.AutoAddPolicy())
aws_config = config.cluster_conf[cluster_name]['cloud']
_fetch_collectl_lustre(cluster, ssh, datadir, aws_config, verbose)
def _bootstrap_bcbio(args, ansible_base):
"""Install bcbio_vm and docker container with tools. Set core and memory usage.
"""
cluster = common.ecluster_config(args.econfig).load_cluster(args.cluster)
inventory_path = os.path.join(
cluster.repository.storage_path,
'ansible-inventory.{}'.format(args.cluster))
playbook_path = os.path.join(
ansible_base, "roles", "bcbio_bootstrap", "tasks", "main.yml")
def _extra_vars(args, cluster_config):
# Calculate cores and memory
compute_nodes = int(
tz.get_in(["nodes", "frontend", "compute_nodes"], cluster_config, 0))
if compute_nodes > 0:
machine = tz.get_in(["nodes", "compute", "flavor"], cluster_config)
else:
machine = tz.get_in(["nodes", "frontend", "flavor"], cluster_config)
cores, mem = AWS_INFO[machine]
# For small number of compute nodes, leave space for runner and controller
if compute_nodes < 5 and compute_nodes > 0:
cores = cores - 2
return {"target_cores": cores, "target_memory": mem,
"upgrade_host_os_and_reboot": not args.no_reboot}
common.run_ansible_pb(
inventory_path, playbook_path, args, _extra_vars)
def create(args):
if args.network:
cidr_regex = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/\d{1,2}$'
if not re.search(cidr_regex, args.network):
sys.stderr.write(
'Network {} is not in CIDR (a.b.c.d/e) format.\n'.format(
args.network))
sys.exit(1)
config = common.ecluster_config(args.econfig)
cluster_config = config.cluster_conf[args.cluster]
try:
cluster = config.load_cluster(args.cluster)
cluster_storage_path = cluster.repository.storage_path
except elasticluster.exceptions.ClusterNotFound:
# Assume the default storage path if the cluster doesn't exist,
# so we can start an ICEL stack in parallel with cluster startup.
cluster_storage_path = elasticluster.conf.Configurator.default_storage_dir
icel_param = {
'oss_count': args.oss_count,
'ost_vol_size': args.size / args.oss_count / args.lun_count,
'ost_vol_count': args.lun_count,
}
template_url = _upload_icel_cf_template(
icel_param, args.bucket, cluster_config['cloud'])
_create_stack(
args.stack_name, template_url, args.network,
args.cluster, cluster_config, args.recreate)
try:
sys.stdout.write('Waiting for stack to launch (this will take '
'a few minutes)')
sys.stdout.flush()
_wait_for_stack(args.stack_name, 'CREATE_COMPLETE',
15 * 60, cluster_config['cloud'])
except Exception as e:
sys.stderr.write('{}\n'.format(str(e)))
sys.exit(1)
ssh_config_path = os.path.join(
cluster_storage_path, 'icel-{}.ssh_config'.format(args.stack_name))
_write_ssh_config(ssh_config_path, args.stack_name, cluster_config)
ansible_config_path = os.path.join(
cluster_storage_path,
'icel-{}.ansible_config'.format(args.stack_name))
_write_ansible_config(
ansible_config_path, args.stack_name, cluster_storage_path)
inventory_path = os.path.join(
cluster_storage_path,
'icel-{}.inventory'.format(args.stack_name))
_write_inventory(inventory_path, args.stack_name, cluster_config['cloud'])
playbook_path = os.path.join(
common.ANSIBLE_BASE, "roles", "icel", "tasks", "main.yml")
common.run_ansible_pb(
inventory_path, playbook_path, args, ansible_cfg=ansible_config_path)
def _setup_vpc(args):
cidr_regex = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/\d{1,2}$'
if not re.search(cidr_regex, args.network):
sys.stderr.write(
'Network {} is not in CIDR (a.b.c.d/e) format.\n'.format(
args.network))
sys.exit(1)
net, mask = args.network.split('/')
if int(mask) > 23:
sys.stderr.write('Network must be at least a /23 in size.\n')
sys.exit(1)
compute_subnet = '{}/24'.format(net)
cluster_config = common.ecluster_config(args.econfig, args.cluster)
conn = boto.connect_vpc(
aws_access_key_id=cluster_config['cloud']['ec2_access_key'],
aws_secret_access_key=cluster_config['cloud']['ec2_secret_key'])
existing_vpcs = conn.get_all_vpcs(filters={'tag:Name': args.cluster})
if existing_vpcs:
if args.recreate:
raise NotImplementedError("bcbio does not currently remove VPCs. "
"The easiest way is to do this manually in the console: "
"https://console.aws.amazon.com/vpc/home")
# FIXME: this doesn't automatically remove resources in the VPC
# like the AWS management console does.
conn.delete_vpc(existing_vpcs[0].id)
else:
print('VPC {} already exists. Skipping. Use --recreate to re-create if needed.'.format(args.cluster))
return
vpc = conn.create_vpc(args.network)
vpc.add_tag('Name', args.cluster)
sg = conn.create_security_group(
'{}_cluster_sg'.format(args.cluster),
'bcbio cluster nodes', vpc.id)
sg.authorize(ip_protocol='tcp', from_port=22, to_port=22,
cidr_ip='0.0.0.0/0')
sg.authorize(ip_protocol='-1', src_group=sg)
igw = conn.create_internet_gateway()
igw.add_tag('Name', '{}_gw'.format(args.cluster))
conn.attach_internet_gateway(igw.id, vpc.id)
rtb = conn.create_route_table(vpc.id)
rtb.add_tag('Name', '{}_rtable'.format(args.cluster))
conn.create_route(rtb.id, '0.0.0.0/0', igw.id)
subnet = conn.create_subnet(vpc.id, compute_subnet)
subnet.add_tag('Name', '{}_cluster'.format(args.cluster))
conn.associate_route_table(rtb.id, subnet.id)
print("Created VPC: %s" % args.cluster)
return args.cluster
def _bootstrap_baseline(args, ansible_base):
"""Install required tools -- docker and gof3r on system.
"""
cluster = common.ecluster_config(args.econfig).load_cluster(args.cluster)
inventory_path = os.path.join(cluster.repository.storage_path,
'ansible-inventory.{}'.format(args.cluster))
docker_pb = os.path.join(ansible_base, "roles", "docker", "tasks",
"main.yml")
common.run_ansible_pb(inventory_path, docker_pb, args)
gof3r_pb = os.path.join(ansible_base, "roles", "gof3r", "tasks",
"main.yml")
common.run_ansible_pb(inventory_path, gof3r_pb, args)
def _bootstrap_baseline(args, ansible_base):
"""Install required tools -- docker and gof3r on system.
"""
cluster = common.ecluster_config(args.econfig).load_cluster(args.cluster)
inventory_path = os.path.join(
cluster.repository.storage_path,
'ansible-inventory.{}'.format(args.cluster))
docker_pb = os.path.join(
ansible_base, "roles", "docker", "tasks", "main.yml")
common.run_ansible_pb(inventory_path, docker_pb, args)
gof3r_pb = os.path.join(
ansible_base, "roles", "gof3r", "tasks", "main.yml")
common.run_ansible_pb(inventory_path, gof3r_pb, args)
def _setup_placment_group(args, new_vpc):
cluster_config = common.ecluster_config(args.econfig, args.cluster)
conn = boto.connect_vpc(
aws_access_key_id=cluster_config['cloud']['ec2_access_key'],
aws_secret_access_key=cluster_config['cloud']['ec2_secret_key'])
pgname = "{}_cluster_pg".format(args.cluster)
pgs = conn.get_all_placement_groups()
if new_vpc or pgname not in [x.name for x in pgs]:
if pgname in [x.name for x in pgs]:
print("Refreshing placement group %s." % pgname)
conn.delete_placement_group(pgname)
conn.create_placement_group(pgname)
print("Placement group %s created." % pgname)
else:
print("Placement group %s already exists. Skipping" % pgname)
def _setup_placment_group(args, new_vpc):
cluster_config = common.ecluster_config(args.econfig, args.cluster)
conn = boto.connect_vpc(
aws_access_key_id=cluster_config['cloud']['ec2_access_key'],
aws_secret_access_key=cluster_config['cloud']['ec2_secret_key'])
pgname = "{}_cluster_pg".format(args.cluster)
pgs = conn.get_all_placement_groups()
if new_vpc or pgname not in [x.name for x in pgs]:
if pgname in [x.name for x in pgs]:
print("Refreshing placement group %s." % pgname)
conn.delete_placement_group(pgname)
conn.create_placement_group(pgname)
print("Placement group %s created." % pgname)
else:
print("Placement group %s already exists. Skipping" % pgname)
def mount_or_unmount(args, mount=True):
cluster = common.ecluster_config(args.econfig).load_cluster(args.cluster)
inventory_path = os.path.join(
cluster.repository.storage_path,
'ansible-inventory.{}'.format(args.cluster))
if mount:
playbook_file = "mount.yml"
else:
playbook_file = "unmount.yml"
playbook_path = os.path.join(
common.ANSIBLE_BASE, "roles", "lustre_client", "tasks", playbook_file)
def get_lustre_vars(args, cluster_config):
return {'lustre_fs_spec': _get_fs_spec(
args.stack_name, cluster_config['cloud'])}
common.run_ansible_pb(
inventory_path, playbook_path, args, get_lustre_vars)
def mount_or_unmount(args, mount=True):
cluster = common.ecluster_config(args.econfig).load_cluster(args.cluster)
inventory_path = os.path.join(cluster.repository.storage_path,
'ansible-inventory.{}'.format(args.cluster))
if mount:
playbook_file = "mount.yml"
else:
playbook_file = "unmount.yml"
playbook_path = os.path.join(common.ANSIBLE_BASE, "roles", "lustre_client",
"tasks", playbook_file)
def get_lustre_vars(args, cluster_config):
return {
'lustre_fs_spec':
_get_fs_spec(args.stack_name, cluster_config['cloud'])
}
common.run_ansible_pb(inventory_path, playbook_path, args, get_lustre_vars)
def _setup_placment_group(args, vpc_info):
import boto
import boto.ec2
cluster_config = common.ecluster_config(args.econfig, args.cluster)
ec2_conn = boto.ec2.connect_to_region(args.region)
conn = boto.connect_vpc(
aws_access_key_id=cluster_config['cloud']['ec2_access_key'],
aws_secret_access_key=cluster_config['cloud']['ec2_secret_key'],
region=ec2_conn.region)
pgname = "{}_cluster_pg".format(args.cluster)
pgs = conn.get_all_placement_groups()
if vpc_info.get("created") or pgname not in [x.name for x in pgs]:
if pgname in [x.name for x in pgs]:
print("Refreshing placement group %s." % pgname)
conn.delete_placement_group(pgname)
conn.create_placement_group(pgname)
print("Placement group %s created." % pgname)
else:
print("Placement group %s already exists. Skipping" % pgname)
def _setup_placment_group(args, vpc_info):
import boto
import boto.ec2
cluster_config = common.ecluster_config(args.econfig, args.cluster)
ec2_conn = boto.ec2.connect_to_region(args.region)
conn = boto.connect_vpc(
aws_access_key_id=cluster_config['cloud']['ec2_access_key'],
aws_secret_access_key=cluster_config['cloud']['ec2_secret_key'],
region=ec2_conn.region)
pgname = "{}_cluster_pg".format(args.cluster)
pgs = conn.get_all_placement_groups()
if vpc_info.get("created") or pgname not in [x.name for x in pgs]:
if pgname in [x.name for x in pgs]:
print("Refreshing placement group %s." % pgname)
conn.delete_placement_group(pgname)
conn.create_placement_group(pgname)
print("Placement group %s created." % pgname)
else:
print("Placement group %s already exists. Skipping" % pgname)
def _bootstrap_nfs(args, ansible_base):
"""Mount encrypted NFS volume on master node and expose across worker nodes.
"""
cluster = common.ecluster_config(args.econfig).load_cluster(args.cluster)
inventory_path = os.path.join(cluster.repository.storage_path,
'ansible-inventory.{}'.format(args.cluster))
nfs_clients = []
with open(inventory_path) as in_handle:
for line in in_handle:
if line.startswith("frontend"):
nfs_server = line.split()[0]
elif line.startswith("compute"):
nfs_clients.append(line.split()[0])
playbook_path = os.path.join(ansible_base, "roles", "encrypted_nfs", "tasks", "main.yml")
def _extra_vars(args, cluster_config):
return {"encrypted_mount": "/encrypted",
"nfs_server": nfs_server,
"nfs_clients": ",".join(nfs_clients),
"login_user": tz.get_in(["nodes", "frontend", "login"], cluster_config),
"encrypted_device": tz.get_in(["nodes", "frontend", "encrypted_volume_device"],
cluster_config, "/dev/xvdf")}
common.run_ansible_pb(inventory_path, playbook_path, args, _extra_vars)
def stop(args):
cluster_config = common.ecluster_config(args.econfig, args.cluster)
_delete_stack(args.stack_name, cluster_config)
def fs_spec(args):
cluster_config = common.ecluster_config(args.econfig, args.cluster)
print(_get_fs_spec(args.stack_name, cluster_config['cloud']))
def stop(args):
cluster_config = common.ecluster_config(args.econfig, args.cluster)
_delete_stack(args.stack_name, cluster_config)
def setup_vpc(args, region):
import boto
import boto.ec2
cidr_regex = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/\d{1,2}$'
if not re.search(cidr_regex, args.network):
raise ValueError(
'Network {} is not in CIDR (a.b.c.d/e) format.\n'.format(
args.network))
net, mask = args.network.split('/')
if int(mask) > 23:
sys.stderr.write('Network must be at least a /23 in size.\n')
sys.exit(1)
compute_subnet = '{}/24'.format(net)
ec2_conn = boto.ec2.connect_to_region(region)
if hasattr(args, "econfig"):
cluster_config = common.ecluster_config(args.econfig, args.cluster)
conn = boto.connect_vpc(
aws_access_key_id=cluster_config['cloud']['ec2_access_key'],
aws_secret_access_key=cluster_config['cloud']['ec2_secret_key'],
region=ec2_conn.region)
else:
conn = boto.connect_vpc(region=ec2_conn.region)
out = {"vpc": args.cluster,
"security_group": "%s_cluster_sg" % args.cluster}
azone = args.zone if hasattr(args, "zone") else None
existing_vpcs = conn.get_all_vpcs(filters={'tag:Name': args.cluster})
if existing_vpcs:
if hasattr(args, "recreate") and args.recreate:
raise NotImplementedError("bcbio does not currently remove VPCs. "
"The easiest way is to do this manually in the console: "
"https://console.aws.amazon.com/vpc/home")
# FIXME: this doesn't automatically remove resources in the VPC
# like the AWS management console does.
conn.delete_vpc(existing_vpcs[0].id)
else:
print('VPC {} already exists. Skipping creation.'.format(args.cluster))
out["created"] = False
out["subnet_id"] = _get_subnet_id(existing_vpcs[0], conn, azone)
return out
vpc = conn.create_vpc(args.network)
while vpc.state != "available":
vpc.update()
vpc.add_tag('Name', args.cluster)
conn.modify_vpc_attribute(vpc.id, enable_dns_support=True)
conn.modify_vpc_attribute(vpc.id, enable_dns_hostnames=True)
out["created"] = True
sg = conn.create_security_group(
'{}_cluster_sg'.format(args.cluster),
'bcbio cluster nodes', vpc.id)
sg.authorize(ip_protocol='tcp', from_port=22, to_port=22,
cidr_ip='0.0.0.0/0')
sg.authorize(ip_protocol='-1', src_group=sg)
igw = conn.create_internet_gateway()
igw.add_tag('Name', '{}_gw'.format(args.cluster))
conn.attach_internet_gateway(igw.id, vpc.id)
rtb = conn.create_route_table(vpc.id)
rtb.add_tag('Name', '{}_rtable'.format(args.cluster))
conn.create_route(rtb.id, '0.0.0.0/0', igw.id)
subnet = conn.create_subnet(vpc.id, compute_subnet, availability_zone=azone)
subnet.add_tag('Name', '{}_cluster'.format(args.cluster))
conn.associate_route_table(rtb.id, subnet.id)
print("Created VPC: %s" % args.cluster)
out["subnet_id"] = _get_subnet_id(vpc, conn, azone)
return out
def setup_vpc(args, region):
import boto
import boto.ec2
cidr_regex = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/\d{1,2}$'
if not re.search(cidr_regex, args.network):
raise ValueError(
'Network {} is not in CIDR (a.b.c.d/e) format.\n'.format(
args.network))
net, mask = args.network.split('/')
if int(mask) > 23:
sys.stderr.write('Network must be at least a /23 in size.\n')
sys.exit(1)
compute_subnet = '{}/24'.format(net)
ec2_conn = boto.ec2.connect_to_region(region)
if hasattr(args, "econfig"):
cluster_config = common.ecluster_config(args.econfig, args.cluster)
conn = boto.connect_vpc(
aws_access_key_id=cluster_config['cloud']['ec2_access_key'],
aws_secret_access_key=cluster_config['cloud']['ec2_secret_key'],
region=ec2_conn.region)
else:
conn = boto.connect_vpc(region=ec2_conn.region)
out = {
"vpc": args.cluster,
"security_group": "%s_cluster_sg" % args.cluster
}
azone = args.zone if hasattr(args, "zone") else None
existing_vpcs = conn.get_all_vpcs(filters={'tag:Name': args.cluster})
if existing_vpcs:
if hasattr(args, "recreate") and args.recreate:
raise NotImplementedError(
"bcbio does not currently remove VPCs. "
"The easiest way is to do this manually in the console: "
"https://console.aws.amazon.com/vpc/home")
# FIXME: this doesn't automatically remove resources in the VPC
# like the AWS management console does.
conn.delete_vpc(existing_vpcs[0].id)
else:
print('VPC {} already exists. Skipping creation.'.format(
args.cluster))
out["created"] = False
out["subnet_id"] = _get_subnet_id(existing_vpcs[0], conn, azone)
return out
vpc = conn.create_vpc(args.network)
while vpc.state != "available":
vpc.update()
vpc.add_tag('Name', args.cluster)
conn.modify_vpc_attribute(vpc.id, enable_dns_support=True)
conn.modify_vpc_attribute(vpc.id, enable_dns_hostnames=True)
out["created"] = True
sg = conn.create_security_group('{}_cluster_sg'.format(args.cluster),
'bcbio cluster nodes', vpc.id)
sg.authorize(ip_protocol='tcp',
from_port=22,
to_port=22,
cidr_ip='0.0.0.0/0')
sg.authorize(ip_protocol='-1', src_group=sg)
igw = conn.create_internet_gateway()
igw.add_tag('Name', '{}_gw'.format(args.cluster))
conn.attach_internet_gateway(igw.id, vpc.id)
rtb = conn.create_route_table(vpc.id)
rtb.add_tag('Name', '{}_rtable'.format(args.cluster))
conn.create_route(rtb.id, '0.0.0.0/0', igw.id)
subnet = conn.create_subnet(vpc.id,
compute_subnet,
availability_zone=azone)
subnet.add_tag('Name', '{}_cluster'.format(args.cluster))
conn.associate_route_table(rtb.id, subnet.id)
print("Created VPC: %s" % args.cluster)
out["subnet_id"] = _get_subnet_id(vpc, conn, azone)
return out
def fs_spec(args):
cluster_config = common.ecluster_config(args.econfig, args.cluster)
print(_get_fs_spec(args.stack_name, cluster_config['cloud']))
