def print_info(args): all_cc = common.ecluster_config(args.econfig) print("Available clusters: %s" % ",".join(all_cc.cluster_conf.keys())) print() print("Configuration for cluster '%s':" % (args.cluster)) cluster_config = common.ecluster_config(args.econfig, args.cluster) _cluster_info(cluster_config) print() print("AWS setup:") _iam_info() _sg_info(cluster_config) _vpc_info(cluster_config) print() _instance_info(cluster_config)
def _setup_vpc(args): cidr_regex = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/\d{1,2}$' if not re.search(cidr_regex, args.network): sys.stderr.write( 'Network {} is not in CIDR (a.b.c.d/e) format.\n'.format( args.network)) sys.exit(1) net, mask = args.network.split('/') if int(mask) > 23: sys.stderr.write('Network must be at least a /23 in size.\n') sys.exit(1) compute_subnet = '{}/24'.format(net) cluster_config = common.ecluster_config(args.econfig, args.cluster) conn = boto.connect_vpc( aws_access_key_id=cluster_config['cloud']['ec2_access_key'], aws_secret_access_key=cluster_config['cloud']['ec2_secret_key']) existing_vpcs = conn.get_all_vpcs(filters={'tag:Name': args.cluster}) if existing_vpcs: if args.recreate: raise NotImplementedError( "bcbio does not currently remove VPCs. " "The easiest way is to do this manually in the console: " "https://console.aws.amazon.com/vpc/home") # FIXME: this doesn't automatically remove resources in the VPC # like the AWS management console does. conn.delete_vpc(existing_vpcs[0].id) else: print( 'VPC {} already exists. Skipping. Use --recreate to re-create if needed.' .format(args.cluster)) return vpc = conn.create_vpc(args.network) vpc.add_tag('Name', args.cluster) sg = conn.create_security_group('{}_cluster_sg'.format(args.cluster), 'bcbio cluster nodes', vpc.id) sg.authorize(ip_protocol='tcp', from_port=22, to_port=22, cidr_ip='0.0.0.0/0') sg.authorize(ip_protocol='-1', src_group=sg) igw = conn.create_internet_gateway() igw.add_tag('Name', '{}_gw'.format(args.cluster)) conn.attach_internet_gateway(igw.id, vpc.id) rtb = conn.create_route_table(vpc.id) rtb.add_tag('Name', '{}_rtable'.format(args.cluster)) conn.create_route(rtb.id, '0.0.0.0/0', igw.id) subnet = conn.create_subnet(vpc.id, compute_subnet) subnet.add_tag('Name', '{}_cluster'.format(args.cluster)) conn.associate_route_table(rtb.id, subnet.id) print("Created VPC: %s" % args.cluster) return args.cluster
def run_remote(args): """Run a script on the frontend node inside a screen session.""" config = common.ecluster_config(args.econfig) cluster = config.load_cluster(args.cluster) frontend = cluster.get_frontend_node() client = frontend.connect(known_hosts_file=cluster.known_hosts_file) cmd = "echo $HOME" stdin, stdout, stderr = client.exec_command(cmd) remote_home = stdout.read().strip() remote_file = os.path.join(remote_home, os.path.basename(args.script)) log_file = "%s.log" % os.path.splitext(remote_file)[0] screen_name = os.path.splitext(os.path.basename(remote_file))[0] sftp = client.open_sftp() sftp.put(args.script, remote_file) sftp.close() cmd = "chmod a+x %s" % remote_file client.exec_command(cmd) cmd = "screen -d -m -S {} bash -c '{} &> {}'".format( screen_name, remote_file, log_file) stdin, stdout, stderr = client.exec_command(cmd) stdout.read() client.close() print("Running {} on AWS in screen session {}".format( remote_file, screen_name))
def _bootstrap_bcbio(args, ansible_base): """Install bcbio_vm and docker container with tools. Set core and memory usage. """ cluster = common.ecluster_config(args.econfig).load_cluster(args.cluster) inventory_path = os.path.join(cluster.repository.storage_path, 'ansible-inventory.{}'.format(args.cluster)) playbook_path = os.path.join(ansible_base, "roles", "bcbio_bootstrap", "tasks", "main.yml") def _extra_vars(args, cluster_config): # Calculate cores and memory compute_nodes = int( tz.get_in(["nodes", "frontend", "compute_nodes"], cluster_config, 0)) if compute_nodes > 0: machine = tz.get_in(["nodes", "compute", "flavor"], cluster_config) else: machine = tz.get_in(["nodes", "frontend", "flavor"], cluster_config) cores, mem = AWS_INFO[machine] cores = ipython.per_machine_target_cores(cores, compute_nodes) return { "target_cores": cores, "target_memory": mem, "upgrade_host_os_and_reboot": not args.no_reboot } common.run_ansible_pb(inventory_path, playbook_path, args, _extra_vars)
def _bootstrap_nfs(args, ansible_base): """Mount encrypted NFS volume on master node and expose across worker nodes. """ cluster = common.ecluster_config(args.econfig).load_cluster(args.cluster) inventory_path = os.path.join(cluster.repository.storage_path, 'ansible-inventory.{}'.format(args.cluster)) nfs_clients = [] with open(inventory_path) as in_handle: for line in in_handle: if line.startswith("frontend"): nfs_server = line.split()[0] elif line.startswith("compute"): nfs_clients.append(line.split()[0]) playbook_path = os.path.join(ansible_base, "roles", "encrypted_nfs", "tasks", "main.yml") def _extra_vars(args, cluster_config): return { "encrypted_mount": "/encrypted", "nfs_server": nfs_server, "nfs_clients": ",".join(nfs_clients), "login_user": tz.get_in(["nodes", "frontend", "login"], cluster_config), "encrypted_device": tz.get_in(["nodes", "frontend", "encrypted_volume_device"], cluster_config, "/dev/xvdf") } common.run_ansible_pb(inventory_path, playbook_path, args, _extra_vars)
def create(args): if args.network: cidr_regex = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/\d{1,2}$' if not re.search(cidr_regex, args.network): sys.stderr.write( 'Network {} is not in CIDR (a.b.c.d/e) format.\n'.format( args.network)) sys.exit(1) config = common.ecluster_config(args.econfig) cluster_config = config.cluster_conf[args.cluster] try: cluster = config.load_cluster(args.cluster) cluster_storage_path = cluster.repository.storage_path except elasticluster.exceptions.ClusterNotFound: # Assume the default storage path if the cluster doesn't exist, # so we can start an ICEL stack in parallel with cluster startup. cluster_storage_path = common.get_storage_dir(args.econfig) if not os.path.exists(cluster_storage_path): os.makedirs(cluster_storage_path) if not args.setup: icel_param = { 'oss_count': args.oss_count, 'ost_vol_size': args.size / args.oss_count / args.lun_count, 'ost_vol_count': args.lun_count, } template_url = _upload_icel_cf_template(icel_param, args.bucket, cluster_config['cloud']) _create_stack(args.stack_name, template_url, args.network, args.cluster, cluster_config, args.recreate) try: sys.stdout.write('Waiting for stack to launch (this will take ' 'a few minutes)') sys.stdout.flush() _wait_for_stack(args.stack_name, 'CREATE_COMPLETE', 15 * 60, cluster_config['cloud']) except Exception as e: sys.stderr.write('{}\n'.format(str(e))) sys.exit(1) ssh_config_path = os.path.join( cluster_storage_path, 'icel-{}.ssh_config'.format(args.stack_name)) _write_ssh_config(ssh_config_path, args.stack_name, cluster_config) ansible_config_path = os.path.join( cluster_storage_path, 'icel-{}.ansible_config'.format(args.stack_name)) _write_ansible_config(ansible_config_path, args.stack_name, cluster_storage_path) inventory_path = os.path.join(cluster_storage_path, 'icel-{}.inventory'.format(args.stack_name)) _write_inventory(inventory_path, args.stack_name, cluster_config['cloud']) playbook_path = os.path.join(common.ANSIBLE_BASE, "roles", "icel", "tasks", "main.yml") common.run_ansible_pb(inventory_path, playbook_path, args, ansible_cfg=ansible_config_path)
def fetch_collectl(econfig_file, cluster_name, datadir, verbose=False): config = ecluster_config(econfig_file) cluster = config.load_cluster(cluster_name) keys = set() for type in cluster.nodes: for node in cluster.nodes[type]: keys.add(node.user_key_private) with ssh_agent(keys): ssh = paramiko.client.SSHClient() ssh.set_missing_host_key_policy(paramiko.client.RejectPolicy()) ssh.load_host_keys(cluster.known_hosts_file) for node in cluster.get_all_nodes(): if not node.preferred_ip: # Instance is unavailable. continue _pull_collectl_data( node.preferred_ip, node.image_user, datadir, ssh, verbose=verbose) # FIXME: load SSH host keys from ICEL instances. ssh.set_missing_host_key_policy(paramiko.client.AutoAddPolicy()) aws_config = config.cluster_conf[cluster_name]['cloud'] _fetch_collectl_lustre(cluster, ssh, datadir, aws_config, verbose)
def fetch_collectl(econfig_file, cluster_name, datadir, verbose=False): config = ecluster_config(econfig_file) cluster = config.load_cluster(cluster_name) keys = set() for type in cluster.nodes: for node in cluster.nodes[type]: keys.add(node.user_key_private) with ssh_agent(keys): ssh = paramiko.client.SSHClient() ssh.set_missing_host_key_policy(paramiko.client.RejectPolicy()) ssh.load_host_keys(cluster.known_hosts_file) for node in cluster.get_all_nodes(): if not node.preferred_ip: # Instance is unavailable. continue _pull_collectl_data(node.preferred_ip, node.image_user, datadir, ssh, verbose=verbose) # FIXME: load SSH host keys from ICEL instances. ssh.set_missing_host_key_policy(paramiko.client.AutoAddPolicy()) aws_config = config.cluster_conf[cluster_name]['cloud'] _fetch_collectl_lustre(cluster, ssh, datadir, aws_config, verbose)
def _bootstrap_bcbio(args, ansible_base): """Install bcbio_vm and docker container with tools. Set core and memory usage. """ cluster = common.ecluster_config(args.econfig).load_cluster(args.cluster) inventory_path = os.path.join( cluster.repository.storage_path, 'ansible-inventory.{}'.format(args.cluster)) playbook_path = os.path.join( ansible_base, "roles", "bcbio_bootstrap", "tasks", "main.yml") def _extra_vars(args, cluster_config): # Calculate cores and memory compute_nodes = int( tz.get_in(["nodes", "frontend", "compute_nodes"], cluster_config, 0)) if compute_nodes > 0: machine = tz.get_in(["nodes", "compute", "flavor"], cluster_config) else: machine = tz.get_in(["nodes", "frontend", "flavor"], cluster_config) cores, mem = AWS_INFO[machine] # For small number of compute nodes, leave space for runner and controller if compute_nodes < 5 and compute_nodes > 0: cores = cores - 2 return {"target_cores": cores, "target_memory": mem, "upgrade_host_os_and_reboot": not args.no_reboot} common.run_ansible_pb( inventory_path, playbook_path, args, _extra_vars)
def create(args): if args.network: cidr_regex = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/\d{1,2}$' if not re.search(cidr_regex, args.network): sys.stderr.write( 'Network {} is not in CIDR (a.b.c.d/e) format.\n'.format( args.network)) sys.exit(1) config = common.ecluster_config(args.econfig) cluster_config = config.cluster_conf[args.cluster] try: cluster = config.load_cluster(args.cluster) cluster_storage_path = cluster.repository.storage_path except elasticluster.exceptions.ClusterNotFound: # Assume the default storage path if the cluster doesn't exist, # so we can start an ICEL stack in parallel with cluster startup. cluster_storage_path = elasticluster.conf.Configurator.default_storage_dir icel_param = { 'oss_count': args.oss_count, 'ost_vol_size': args.size / args.oss_count / args.lun_count, 'ost_vol_count': args.lun_count, } template_url = _upload_icel_cf_template( icel_param, args.bucket, cluster_config['cloud']) _create_stack( args.stack_name, template_url, args.network, args.cluster, cluster_config, args.recreate) try: sys.stdout.write('Waiting for stack to launch (this will take ' 'a few minutes)') sys.stdout.flush() _wait_for_stack(args.stack_name, 'CREATE_COMPLETE', 15 * 60, cluster_config['cloud']) except Exception as e: sys.stderr.write('{}\n'.format(str(e))) sys.exit(1) ssh_config_path = os.path.join( cluster_storage_path, 'icel-{}.ssh_config'.format(args.stack_name)) _write_ssh_config(ssh_config_path, args.stack_name, cluster_config) ansible_config_path = os.path.join( cluster_storage_path, 'icel-{}.ansible_config'.format(args.stack_name)) _write_ansible_config( ansible_config_path, args.stack_name, cluster_storage_path) inventory_path = os.path.join( cluster_storage_path, 'icel-{}.inventory'.format(args.stack_name)) _write_inventory(inventory_path, args.stack_name, cluster_config['cloud']) playbook_path = os.path.join( common.ANSIBLE_BASE, "roles", "icel", "tasks", "main.yml") common.run_ansible_pb( inventory_path, playbook_path, args, ansible_cfg=ansible_config_path)
def _setup_vpc(args): cidr_regex = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/\d{1,2}$' if not re.search(cidr_regex, args.network): sys.stderr.write( 'Network {} is not in CIDR (a.b.c.d/e) format.\n'.format( args.network)) sys.exit(1) net, mask = args.network.split('/') if int(mask) > 23: sys.stderr.write('Network must be at least a /23 in size.\n') sys.exit(1) compute_subnet = '{}/24'.format(net) cluster_config = common.ecluster_config(args.econfig, args.cluster) conn = boto.connect_vpc( aws_access_key_id=cluster_config['cloud']['ec2_access_key'], aws_secret_access_key=cluster_config['cloud']['ec2_secret_key']) existing_vpcs = conn.get_all_vpcs(filters={'tag:Name': args.cluster}) if existing_vpcs: if args.recreate: raise NotImplementedError("bcbio does not currently remove VPCs. " "The easiest way is to do this manually in the console: " "https://console.aws.amazon.com/vpc/home") # FIXME: this doesn't automatically remove resources in the VPC # like the AWS management console does. conn.delete_vpc(existing_vpcs[0].id) else: print('VPC {} already exists. Skipping. Use --recreate to re-create if needed.'.format(args.cluster)) return vpc = conn.create_vpc(args.network) vpc.add_tag('Name', args.cluster) sg = conn.create_security_group( '{}_cluster_sg'.format(args.cluster), 'bcbio cluster nodes', vpc.id) sg.authorize(ip_protocol='tcp', from_port=22, to_port=22, cidr_ip='0.0.0.0/0') sg.authorize(ip_protocol='-1', src_group=sg) igw = conn.create_internet_gateway() igw.add_tag('Name', '{}_gw'.format(args.cluster)) conn.attach_internet_gateway(igw.id, vpc.id) rtb = conn.create_route_table(vpc.id) rtb.add_tag('Name', '{}_rtable'.format(args.cluster)) conn.create_route(rtb.id, '0.0.0.0/0', igw.id) subnet = conn.create_subnet(vpc.id, compute_subnet) subnet.add_tag('Name', '{}_cluster'.format(args.cluster)) conn.associate_route_table(rtb.id, subnet.id) print("Created VPC: %s" % args.cluster) return args.cluster
def _bootstrap_baseline(args, ansible_base): """Install required tools -- docker and gof3r on system. """ cluster = common.ecluster_config(args.econfig).load_cluster(args.cluster) inventory_path = os.path.join(cluster.repository.storage_path, 'ansible-inventory.{}'.format(args.cluster)) docker_pb = os.path.join(ansible_base, "roles", "docker", "tasks", "main.yml") common.run_ansible_pb(inventory_path, docker_pb, args) gof3r_pb = os.path.join(ansible_base, "roles", "gof3r", "tasks", "main.yml") common.run_ansible_pb(inventory_path, gof3r_pb, args)
def _bootstrap_baseline(args, ansible_base): """Install required tools -- docker and gof3r on system. """ cluster = common.ecluster_config(args.econfig).load_cluster(args.cluster) inventory_path = os.path.join( cluster.repository.storage_path, 'ansible-inventory.{}'.format(args.cluster)) docker_pb = os.path.join( ansible_base, "roles", "docker", "tasks", "main.yml") common.run_ansible_pb(inventory_path, docker_pb, args) gof3r_pb = os.path.join( ansible_base, "roles", "gof3r", "tasks", "main.yml") common.run_ansible_pb(inventory_path, gof3r_pb, args)
def _setup_placment_group(args, new_vpc): cluster_config = common.ecluster_config(args.econfig, args.cluster) conn = boto.connect_vpc( aws_access_key_id=cluster_config['cloud']['ec2_access_key'], aws_secret_access_key=cluster_config['cloud']['ec2_secret_key']) pgname = "{}_cluster_pg".format(args.cluster) pgs = conn.get_all_placement_groups() if new_vpc or pgname not in [x.name for x in pgs]: if pgname in [x.name for x in pgs]: print("Refreshing placement group %s." % pgname) conn.delete_placement_group(pgname) conn.create_placement_group(pgname) print("Placement group %s created." % pgname) else: print("Placement group %s already exists. Skipping" % pgname)
def mount_or_unmount(args, mount=True): cluster = common.ecluster_config(args.econfig).load_cluster(args.cluster) inventory_path = os.path.join( cluster.repository.storage_path, 'ansible-inventory.{}'.format(args.cluster)) if mount: playbook_file = "mount.yml" else: playbook_file = "unmount.yml" playbook_path = os.path.join( common.ANSIBLE_BASE, "roles", "lustre_client", "tasks", playbook_file) def get_lustre_vars(args, cluster_config): return {'lustre_fs_spec': _get_fs_spec( args.stack_name, cluster_config['cloud'])} common.run_ansible_pb( inventory_path, playbook_path, args, get_lustre_vars)
def mount_or_unmount(args, mount=True): cluster = common.ecluster_config(args.econfig).load_cluster(args.cluster) inventory_path = os.path.join(cluster.repository.storage_path, 'ansible-inventory.{}'.format(args.cluster)) if mount: playbook_file = "mount.yml" else: playbook_file = "unmount.yml" playbook_path = os.path.join(common.ANSIBLE_BASE, "roles", "lustre_client", "tasks", playbook_file) def get_lustre_vars(args, cluster_config): return { 'lustre_fs_spec': _get_fs_spec(args.stack_name, cluster_config['cloud']) } common.run_ansible_pb(inventory_path, playbook_path, args, get_lustre_vars)
def _setup_placment_group(args, vpc_info): import boto import boto.ec2 cluster_config = common.ecluster_config(args.econfig, args.cluster) ec2_conn = boto.ec2.connect_to_region(args.region) conn = boto.connect_vpc( aws_access_key_id=cluster_config['cloud']['ec2_access_key'], aws_secret_access_key=cluster_config['cloud']['ec2_secret_key'], region=ec2_conn.region) pgname = "{}_cluster_pg".format(args.cluster) pgs = conn.get_all_placement_groups() if vpc_info.get("created") or pgname not in [x.name for x in pgs]: if pgname in [x.name for x in pgs]: print("Refreshing placement group %s." % pgname) conn.delete_placement_group(pgname) conn.create_placement_group(pgname) print("Placement group %s created." % pgname) else: print("Placement group %s already exists. Skipping" % pgname)
def _bootstrap_nfs(args, ansible_base): """Mount encrypted NFS volume on master node and expose across worker nodes. """ cluster = common.ecluster_config(args.econfig).load_cluster(args.cluster) inventory_path = os.path.join(cluster.repository.storage_path, 'ansible-inventory.{}'.format(args.cluster)) nfs_clients = [] with open(inventory_path) as in_handle: for line in in_handle: if line.startswith("frontend"): nfs_server = line.split()[0] elif line.startswith("compute"): nfs_clients.append(line.split()[0]) playbook_path = os.path.join(ansible_base, "roles", "encrypted_nfs", "tasks", "main.yml") def _extra_vars(args, cluster_config): return {"encrypted_mount": "/encrypted", "nfs_server": nfs_server, "nfs_clients": ",".join(nfs_clients), "login_user": tz.get_in(["nodes", "frontend", "login"], cluster_config), "encrypted_device": tz.get_in(["nodes", "frontend", "encrypted_volume_device"], cluster_config, "/dev/xvdf")} common.run_ansible_pb(inventory_path, playbook_path, args, _extra_vars)
def stop(args): cluster_config = common.ecluster_config(args.econfig, args.cluster) _delete_stack(args.stack_name, cluster_config)
def fs_spec(args): cluster_config = common.ecluster_config(args.econfig, args.cluster) print(_get_fs_spec(args.stack_name, cluster_config['cloud']))
def setup_vpc(args, region): import boto import boto.ec2 cidr_regex = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/\d{1,2}$' if not re.search(cidr_regex, args.network): raise ValueError( 'Network {} is not in CIDR (a.b.c.d/e) format.\n'.format( args.network)) net, mask = args.network.split('/') if int(mask) > 23: sys.stderr.write('Network must be at least a /23 in size.\n') sys.exit(1) compute_subnet = '{}/24'.format(net) ec2_conn = boto.ec2.connect_to_region(region) if hasattr(args, "econfig"): cluster_config = common.ecluster_config(args.econfig, args.cluster) conn = boto.connect_vpc( aws_access_key_id=cluster_config['cloud']['ec2_access_key'], aws_secret_access_key=cluster_config['cloud']['ec2_secret_key'], region=ec2_conn.region) else: conn = boto.connect_vpc(region=ec2_conn.region) out = {"vpc": args.cluster, "security_group": "%s_cluster_sg" % args.cluster} azone = args.zone if hasattr(args, "zone") else None existing_vpcs = conn.get_all_vpcs(filters={'tag:Name': args.cluster}) if existing_vpcs: if hasattr(args, "recreate") and args.recreate: raise NotImplementedError("bcbio does not currently remove VPCs. " "The easiest way is to do this manually in the console: " "https://console.aws.amazon.com/vpc/home") # FIXME: this doesn't automatically remove resources in the VPC # like the AWS management console does. conn.delete_vpc(existing_vpcs[0].id) else: print('VPC {} already exists. Skipping creation.'.format(args.cluster)) out["created"] = False out["subnet_id"] = _get_subnet_id(existing_vpcs[0], conn, azone) return out vpc = conn.create_vpc(args.network) while vpc.state != "available": vpc.update() vpc.add_tag('Name', args.cluster) conn.modify_vpc_attribute(vpc.id, enable_dns_support=True) conn.modify_vpc_attribute(vpc.id, enable_dns_hostnames=True) out["created"] = True sg = conn.create_security_group( '{}_cluster_sg'.format(args.cluster), 'bcbio cluster nodes', vpc.id) sg.authorize(ip_protocol='tcp', from_port=22, to_port=22, cidr_ip='0.0.0.0/0') sg.authorize(ip_protocol='-1', src_group=sg) igw = conn.create_internet_gateway() igw.add_tag('Name', '{}_gw'.format(args.cluster)) conn.attach_internet_gateway(igw.id, vpc.id) rtb = conn.create_route_table(vpc.id) rtb.add_tag('Name', '{}_rtable'.format(args.cluster)) conn.create_route(rtb.id, '0.0.0.0/0', igw.id) subnet = conn.create_subnet(vpc.id, compute_subnet, availability_zone=azone) subnet.add_tag('Name', '{}_cluster'.format(args.cluster)) conn.associate_route_table(rtb.id, subnet.id) print("Created VPC: %s" % args.cluster) out["subnet_id"] = _get_subnet_id(vpc, conn, azone) return out
def setup_vpc(args, region): import boto import boto.ec2 cidr_regex = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/\d{1,2}$' if not re.search(cidr_regex, args.network): raise ValueError( 'Network {} is not in CIDR (a.b.c.d/e) format.\n'.format( args.network)) net, mask = args.network.split('/') if int(mask) > 23: sys.stderr.write('Network must be at least a /23 in size.\n') sys.exit(1) compute_subnet = '{}/24'.format(net) ec2_conn = boto.ec2.connect_to_region(region) if hasattr(args, "econfig"): cluster_config = common.ecluster_config(args.econfig, args.cluster) conn = boto.connect_vpc( aws_access_key_id=cluster_config['cloud']['ec2_access_key'], aws_secret_access_key=cluster_config['cloud']['ec2_secret_key'], region=ec2_conn.region) else: conn = boto.connect_vpc(region=ec2_conn.region) out = { "vpc": args.cluster, "security_group": "%s_cluster_sg" % args.cluster } azone = args.zone if hasattr(args, "zone") else None existing_vpcs = conn.get_all_vpcs(filters={'tag:Name': args.cluster}) if existing_vpcs: if hasattr(args, "recreate") and args.recreate: raise NotImplementedError( "bcbio does not currently remove VPCs. " "The easiest way is to do this manually in the console: " "https://console.aws.amazon.com/vpc/home") # FIXME: this doesn't automatically remove resources in the VPC # like the AWS management console does. conn.delete_vpc(existing_vpcs[0].id) else: print('VPC {} already exists. Skipping creation.'.format( args.cluster)) out["created"] = False out["subnet_id"] = _get_subnet_id(existing_vpcs[0], conn, azone) return out vpc = conn.create_vpc(args.network) while vpc.state != "available": vpc.update() vpc.add_tag('Name', args.cluster) conn.modify_vpc_attribute(vpc.id, enable_dns_support=True) conn.modify_vpc_attribute(vpc.id, enable_dns_hostnames=True) out["created"] = True sg = conn.create_security_group('{}_cluster_sg'.format(args.cluster), 'bcbio cluster nodes', vpc.id) sg.authorize(ip_protocol='tcp', from_port=22, to_port=22, cidr_ip='0.0.0.0/0') sg.authorize(ip_protocol='-1', src_group=sg) igw = conn.create_internet_gateway() igw.add_tag('Name', '{}_gw'.format(args.cluster)) conn.attach_internet_gateway(igw.id, vpc.id) rtb = conn.create_route_table(vpc.id) rtb.add_tag('Name', '{}_rtable'.format(args.cluster)) conn.create_route(rtb.id, '0.0.0.0/0', igw.id) subnet = conn.create_subnet(vpc.id, compute_subnet, availability_zone=azone) subnet.add_tag('Name', '{}_cluster'.format(args.cluster)) conn.associate_route_table(rtb.id, subnet.id) print("Created VPC: %s" % args.cluster) out["subnet_id"] = _get_subnet_id(vpc, conn, azone) return out