def build_block_device_map(ephemeral,
number_ephemeral_disks=1,
ebs_size=None,
iops=None,
number_ebs_volumes=1):
bdm = blockdevicemapping.BlockDeviceMapping()
if ephemeral:
# The ephemeral disk
xvdb = BlockDeviceType()
xvdb.ephemeral_name = 'ephemeral0'
bdm['/dev/xvdb'] = xvdb
if number_ephemeral_disks == 2:
xvdc = BlockDeviceType()
xvdc.ephemeral_name = 'ephemeral1'
bdm['/dev/xvdc'] = xvdc
if ebs_size:
for disks in range(0, number_ebs_volumes):
xvd_n = blockdevicemapping.EBSBlockDeviceType(
delete_on_termination=True)
xvd_n.size = int(ebs_size) # size in Gigabytes
if iops:
xvd_n.iops = 500
xvd_n.volume_type = 'io1'
else:
xvd_n.volume_type = 'gp2'
last_char = chr(ord('f') + disks)
bdm['/dev/xvd' + last_char] = xvd_n
return bdm
def launch_instance(args, userdata=None):
"""Connect to AWS and launch instance using args from create_parser"""
# Make userdata unless given
userdata = userdata or create_userdata(args.mirrorurl, args.ksurl)
# Connect to EC2 endpoint for region
conn = connect_to_region(args.region)
# Choose first image ID that matches the given AMI name pattern
try:
id = conn.get_all_images(filters={'name': args.bootami})[0].id
except IndexError:
raise Error('ERROR: No matching AMIs found!')
# Connect to the given SubnetID or get a list of subnets in this region
if args.novpc:
subnets = None
else:
c = vpc.connect_to_region(args.region)
subnets = c.get_all_subnets(args.subnetid)
# Use a VPC if we can, unless told not to. Use first subnet in list.
if subnets:
grpfilt = {'group-name': args.secgroup, 'vpc_id': subnets[0].vpc_id}
subnetid = subnets[0].id
# Find the security group id from the name
group = conn.get_all_security_groups(filters=grpfilt)[0].id
# associate the instance with a VPC and give it a puclic IP address
interface = networkinterface.NetworkInterfaceSpecification(
subnet_id=subnetid,
groups=[group],
associate_public_ip_address=True)
interfaces = networkinterface.NetworkInterfaceCollection(interface)
groups = None
else:
interfaces = None
groups = [args.secgroup]
# Set disk mapping if needed
if args.disksize:
dev_xvda = blockdevicemapping.BlockDeviceType(
delete_on_termination=True)
dev_xvda.size = args.disksize
device_map = blockdevicemapping.BlockDeviceMapping()
device_map['/dev/xvda'] = dev_xvda
else:
device_map = None
# launch instance
res = conn.run_instances(id,
key_name=args.key,
instance_type=args.type,
network_interfaces=interfaces,
user_data=userdata,
security_groups=groups,
block_device_map=device_map)
return res.instances[0]
def create_launch_configuration(name, image_id, key_name=None,
security_groups=None, user_data=None,
instance_type='m1.small', kernel_id=None,
ramdisk_id=None, block_device_mappings=None,
instance_monitoring=False, spot_price=None,
instance_profile_name=None,
ebs_optimized=False,
associate_public_ip_address=None,
volume_type=None, delete_on_termination=True,
iops=None, use_block_device_types=False,
region=None, key=None, keyid=None,
profile=None):
'''
Create a launch configuration.
CLI example::
salt myminion boto_asg.create_launch_configuration mylc image_id=ami-0b9c9f62 key_name='mykey' security_groups='["mygroup"]' instance_type='c3.2xlarge'
'''
conn = _get_conn(region, key, keyid, profile)
if not conn:
return False
if isinstance(security_groups, six.string_types):
security_groups = json.loads(security_groups)
if isinstance(block_device_mappings, six.string_types):
block_device_mappings = json.loads(block_device_mappings)
_bdms = []
if block_device_mappings:
# Boto requires objects for the mappings and the devices.
_block_device_map = blockdevicemapping.BlockDeviceMapping()
for block_device_dict in block_device_mappings:
for block_device, attributes in six.iteritems(block_device_dict):
_block_device = blockdevicemapping.EBSBlockDeviceType()
for attribute, value in six.iteritems(attributes):
setattr(_block_device, attribute, value)
_block_device_map[block_device] = _block_device
_bdms = [_block_device_map]
lc = autoscale.LaunchConfiguration(
name=name, image_id=image_id, key_name=key_name,
security_groups=security_groups, user_data=user_data,
instance_type=instance_type, kernel_id=kernel_id,
ramdisk_id=ramdisk_id, block_device_mappings=_bdms,
instance_monitoring=instance_monitoring, spot_price=spot_price,
instance_profile_name=instance_profile_name,
ebs_optimized=ebs_optimized,
associate_public_ip_address=associate_public_ip_address,
volume_type=volume_type, delete_on_termination=delete_on_termination,
iops=iops, use_block_device_types=use_block_device_types)
try:
conn.create_launch_configuration(lc)
log.info('Created LC {0}'.format(name))
return True
except boto.exception.BotoServerError as e:
log.debug(e)
msg = 'Failed to create LC {0}'.format(name)
log.error(msg)
return False
def create_instance(ebs_size, ami_name):
"""
Create the AWS instance
:param ebs_size:
"""
puts('Creating the instance {1} with disk size {0} GB'.format(
ebs_size, ami_name))
# This relies on a ~/.boto file holding the '<aws access key>', '<aws secret key>'
ec2_connection = boto.connect_ec2()
dev_sda1 = blockdevicemapping.EBSBlockDeviceType(
delete_on_termination=True)
dev_sda1.size = int(ebs_size) # size in Gigabytes
bdm = blockdevicemapping.BlockDeviceMapping()
bdm['/dev/sda1'] = dev_sda1
reservations = ec2_connection.run_instances(
AMI_ID,
instance_type=INSTANCE_TYPE,
key_name=KEY_NAME,
security_groups=SECURITY_GROUPS,
block_device_map=bdm)
instance = reservations.instances[0]
# Sleep so Amazon recognizes the new instance
for i in range(4):
fastprint('.')
time.sleep(5)
# Are we running yet?
while not instance.update() == 'running':
fastprint('.')
time.sleep(5)
# Sleep a bit more Amazon recognizes the new instance
for i in range(4):
fastprint('.')
time.sleep(5)
puts('.')
ec2_connection.create_tags([instance.id], {'Name': '{0}'.format(ami_name)})
# The instance is started, but not useable (yet)
puts('Started the instance now waiting for the SSH daemon to start.')
for i in range(12):
fastprint('.')
time.sleep(5)
puts('.')
# Return the instance
return instance, ec2_connection
def get_bdm(ec2_conf):
def device(d):
dev = bdm.BlockDeviceType()
if d['VirtualName'].startswith('ephemeral'):
# Instance Storage
dev.ephemeral_name = d['VirtualName']
else:
# EBS
dev.size = d['Ebs.VolumeSize']
delete = d.get('Ebs.DeleteOnTermination', None)
if delete is not None:
dev.delete_on_termination = delete
return (d['DeviceName'], dev)
devices = map(device, ec2_conf['Block_Device_Mapping'])
device_mapping = bdm.BlockDeviceMapping()
for name, dev in devices:
device_mapping[name] = dev
return device_mapping
def create_launch_configuration(name,
image_id,
key_name=None,
vpc_id=None,
vpc_name=None,
security_groups=None,
user_data=None,
instance_type='m1.small',
kernel_id=None,
ramdisk_id=None,
block_device_mappings=None,
instance_monitoring=False,
spot_price=None,
instance_profile_name=None,
ebs_optimized=False,
associate_public_ip_address=None,
volume_type=None,
delete_on_termination=True,
iops=None,
use_block_device_types=False,
region=None,
key=None,
keyid=None,
profile=None):
'''
Create a launch configuration.
CLI example::
salt myminion boto_asg.create_launch_configuration mylc image_id=ami-0b9c9f62 key_name='mykey' security_groups='["mygroup"]' instance_type='c3.2xlarge'
'''
conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile)
if isinstance(security_groups, six.string_types):
security_groups = json.loads(security_groups)
if isinstance(block_device_mappings, six.string_types):
block_device_mappings = json.loads(block_device_mappings)
_bdms = []
if block_device_mappings:
# Boto requires objects for the mappings and the devices.
_block_device_map = blockdevicemapping.BlockDeviceMapping()
for block_device_dict in block_device_mappings:
for block_device, attributes in six.iteritems(block_device_dict):
_block_device = blockdevicemapping.EBSBlockDeviceType()
for attribute, value in six.iteritems(attributes):
setattr(_block_device, attribute, value)
_block_device_map[block_device] = _block_device
_bdms = [_block_device_map]
# If a VPC is specified, then determine the secgroup id's within that VPC, not
# within the default VPC. If a security group id is already part of the list,
# convert_to_group_ids leaves that entry without attempting a lookup on it.
if security_groups and (vpc_id or vpc_name):
security_groups = __salt__['boto_secgroup.convert_to_group_ids'](
security_groups,
vpc_id=vpc_id,
vpc_name=vpc_name,
region=region,
key=key,
keyid=keyid,
profile=profile)
lc = autoscale.LaunchConfiguration(
name=name,
image_id=image_id,
key_name=key_name,
security_groups=security_groups,
user_data=user_data,
instance_type=instance_type,
kernel_id=kernel_id,
ramdisk_id=ramdisk_id,
block_device_mappings=_bdms,
instance_monitoring=instance_monitoring,
spot_price=spot_price,
instance_profile_name=instance_profile_name,
ebs_optimized=ebs_optimized,
associate_public_ip_address=associate_public_ip_address,
volume_type=volume_type,
delete_on_termination=delete_on_termination,
iops=iops,
use_block_device_types=use_block_device_types)
retries = 30
while True:
try:
conn.create_launch_configuration(lc)
log.info('Created LC {0}'.format(name))
return True
except boto.exception.BotoServerError as e:
if retries and e.code == 'Throttling':
log.debug('Throttled by AWS API, retrying in 5 seconds...')
time.sleep(5)
retries -= 1
continue
log.error(e)
msg = 'Failed to create LC {0}'.format(name)
log.error(msg)
return False
def request_task_instance(ec2, autoscale, instance_type, lifespan, command, bucket, aws_sns_arn, patch_version, tempsize):
'''
'''
major_version = patch_version.split('.')[0]
group_name = 'CI Workers {0}.x'.format(major_version)
(group, ) = autoscale.get_all_groups([group_name])
(config, ) = autoscale.get_all_launch_configurations(names=[group.launch_config_name])
(image, ) = ec2.get_all_images(image_ids=[config.image_id])
keypair = [kp for kp in ec2.get_all_key_pairs() if kp.name.startswith('oa-')][0]
yyyymmdd = datetime.utcnow().strftime('%Y-%m-%d-%H-%M')
with open(first_file(userdata_paths)) as file:
userdata_kwargs = dict(
command = ' '.join(map(shlex.quote, command)),
lifespan = shlex.quote(str(lifespan)),
major_version = shlex.quote(major_version),
patch_version = shlex.quote(patch_version),
log_prefix = shlex.quote('logs/{}-{}'.format(yyyymmdd, command[0])),
bucket = shlex.quote(bucket or 'data.openaddresses.io'),
aws_sns_arn = '', aws_region = '',
command_name = command[0]
)
if aws_sns_arn:
try:
_, _, _, aws_region, _ = aws_sns_arn.split(':', 4)
except ValueError:
pass
else:
if aws_sns_arn.startswith('arn:aws:sns:'):
userdata_kwargs.update(aws_sns_arn = shlex.quote(aws_sns_arn),
aws_region = shlex.quote(aws_region))
device_map = blockdevicemapping.BlockDeviceMapping()
if tempsize:
dev_sdb = blockdevicemapping.BlockDeviceType(delete_on_termination=True)
dev_sdb.size = tempsize
device_map['/dev/sdb'] = dev_sdb
run_kwargs = dict(instance_type=instance_type, security_groups=['default'],
instance_initiated_shutdown_behavior='terminate',
# TODO: use current role from http://169.254.169.254/latest/meta-data/iam/info
instance_profile_name='machine-communication',
key_name=keypair.name, block_device_map=device_map)
print('Running with keyword args:', pprint.pformat(run_kwargs), sep='\n', file=sys.stderr)
run_kwargs.update(user_data=file.read().format(**userdata_kwargs))
reservation = image.run(**run_kwargs)
(instance, ) = reservation.instances
try:
instance.add_tag('Name', 'Scheduled {} {}'.format(yyyymmdd, command[0]))
instance.add_tag('Command', command[0])
instance.add_tag('Trigger', 'run-ec2-command')
except EC2ResponseError:
time.sleep(10)
try:
instance.add_tag('Name', 'Scheduled {} {}'.format(yyyymmdd, command[0]))
instance.add_tag('Command', command[0])
instance.add_tag('Trigger', 'run-ec2-command')
except EC2ResponseError:
time.sleep(30)
instance.add_tag('Name', 'Scheduled {} {}'.format(yyyymmdd, command[0]))
instance.add_tag('Command', command[0])
instance.add_tag('Trigger', 'run-ec2-command')
print('Started EC2 instance {} from AMI {}'.format(instance, image), file=sys.stderr)
return instance
def main():
print('Starting nightly build: {}'.format(strftime("%Y-%m-%d %H:%M:%S")))
print('Opening connection..')
access_key = os.environ.get('AWS_ACCESS_KEY_ID')
secret_key = os.environ.get('AWS_ACCESS_KEY')
conn = boto.ec2.connect_to_region(settings['region'],
aws_access_key_id=access_key,
aws_secret_access_key=secret_key)
RESOURCES.append(conn)
print('Running Packer..')
baked_ami_id = run_packer()
baked_ami = conn.get_image(baked_ami_id)
RESOURCES.append(baked_ami)
baked_snap = baked_ami.block_device_mapping['/dev/sda1'].snapshot_id
print('Launching worker machine..')
mapping = bdm.BlockDeviceMapping()
mapping['/dev/sda1'] = bdm.BlockDeviceType(size=10,
volume_type='gp2',
delete_on_termination=True)
mapping['/dev/sdf'] = bdm.BlockDeviceType(snapshot_id=baked_snap,
volume_type='gp2',
delete_on_termination=True)
kp_name = random_generator()
kp = conn.create_key_pair(kp_name)
kp.save(gettempdir())
print('Keypair created: {}'.format(kp_name))
sg_name = random_generator()
sg = conn.create_security_group(sg_name, 'vagrant nightly')
sg.authorize(ip_protocol='tcp',
from_port=22,
to_port=22,
cidr_ip='0.0.0.0/0')
print('Security Group created: {}'.format(sg_name))
reserv = conn.run_instances(
image_id=settings['factory_ami'],
key_name=kp_name,
instance_type=settings['instance_type'],
security_groups=[sg],
block_device_map=mapping,
instance_profile_name=settings['aws_iam_group'])
factory_instance = reserv.instances[0]
RESOURCES.append(factory_instance)
RESOURCES.append(kp)
RESOURCES.append(sg)
env.key_filename = os.path.join(gettempdir(), '{}.pem'.format(kp_name))
env.timeout = 10
env.connection_attempts = 12
while factory_instance.state != 'running':
sleep(5)
factory_instance.update()
print('machine state: {}'.format(factory_instance.state))
print('Executing script..')
execute(do_work,
host='{}@{}'.format(settings['username'],
factory_instance.ip_address))
def create_instance(ebs_size, ami_name, sydney=False):
"""
Create the AWS instance
:param ebs_size:
"""
puts('Creating the instance {1} with disk size {0} GB'.format(ebs_size, ami_name))
if exists(join(expanduser('~'), '.aws/credentials')):
# This relies on a ~/.aws/credentials file holding the '<aws access key>', '<aws secret key>'
puts("Using ~/.aws/credentials")
if sydney:
ec2_connection = boto.ec2.connect_to_region(SYDNEY_REGION, profile_name='theSkyNet')
else:
ec2_connection = boto.connect_ec2(profile_name='theSkyNet')
else:
# This relies on a ~/.boto or /etc/boto.cfg file holding the '<aws access key>', '<aws secret key>'
puts("Using ~/.boto or /etc/boto.cfg")
ec2_connection = boto.connect_ec2()
dev_xvda = blockdevicemapping.EBSBlockDeviceType(delete_on_termination=True)
dev_xvda.size = int(ebs_size) # size in Gigabytes
bdm = blockdevicemapping.BlockDeviceMapping()
bdm['/dev/xvda'] = dev_xvda
if sydney:
reservations = ec2_connection.run_instances(
SYDNEY_AMI_ID,
subnet_id=SYDNEY_SUBNET,
instance_type=INSTANCE_TYPE,
key_name=SYDNEY_KEY_NAME,
security_group_ids=SYDNEY_SECURITY_GROUPS,
block_device_map=bdm)
else:
reservations = ec2_connection.run_instances(AMI_ID, instance_type=INSTANCE_TYPE, key_name=KEY_NAME, security_groups=SECURITY_GROUPS, block_device_map=bdm)
instance = reservations.instances[0]
# Sleep so Amazon recognizes the new instance
for i in range(4):
fastprint('.')
time.sleep(5)
# Are we running yet?
while not instance.update() == 'running':
fastprint('.')
time.sleep(5)
# Sleep a bit more Amazon recognizes the new instance
for i in range(4):
fastprint('.')
time.sleep(5)
puts('.')
ec2_connection.create_tags([instance.id], {'Name': '{0}'.format(ami_name)})
# The instance is started, but not useable (yet)
puts('Started the instance now waiting for the SSH daemon to start.')
for i in range(12):
fastprint('.')
time.sleep(5)
puts('.')
# Return the instance
return instance, ec2_connection
def request_task_instance(ec2, autoscale, instance_type, chef_role, lifespan,
command, bucket, aws_sns_arn):
'''
'''
group_name = 'CI Workers {0}.x'.format(*get_version().split('.'))
(group, ) = autoscale.get_all_groups([group_name])
(config, ) = autoscale.get_all_launch_configurations(
names=[group.launch_config_name])
(image, ) = ec2.get_all_images(image_ids=[config.image_id])
keypair = [
kp for kp in ec2.get_all_key_pairs() if kp.name.startswith('oa-')
][0]
yyyymmdd = datetime.utcnow().strftime('%Y-%m-%d-%H-%M')
with open(join(dirname(__file__), 'templates',
'task-instance-userdata.sh')) as file:
userdata_kwargs = dict(role=shlex.quote(chef_role),
command=' '.join(map(shlex.quote, command)),
lifespan=shlex.quote(str(lifespan)),
version=shlex.quote(get_version()),
log_prefix=shlex.quote('logs/{}-{}'.format(
yyyymmdd, command[0])),
bucket=shlex.quote(bucket
or 'data.openaddresses.io'),
aws_sns_arn='',
aws_region='',
command_name=command[0])
if aws_sns_arn:
try:
_, _, _, aws_region, _ = aws_sns_arn.split(':', 4)
except ValueError:
pass
else:
if aws_sns_arn.startswith('arn:aws:sns:'):
userdata_kwargs.update(
aws_sns_arn=shlex.quote(aws_sns_arn),
aws_region=shlex.quote(aws_region))
device_map = blockdevicemapping.BlockDeviceMapping()
if instance_type in block_device_sizes:
device_map = blockdevicemapping.BlockDeviceMapping()
dev_sdb = blockdevicemapping.BlockDeviceType()
dev_sdb.size = block_device_sizes[instance_type]
device_map['/dev/sdb'] = dev_sdb
run_kwargs = dict(
instance_type=instance_type,
security_groups=['default'],
instance_initiated_shutdown_behavior='terminate',
user_data=file.read().format(**userdata_kwargs),
# TODO: use current role from http://169.254.169.254/latest/meta-data/iam/info
instance_profile_name='machine-communication',
key_name=keypair.name,
block_device_map=device_map)
reservation = image.run(**run_kwargs)
(instance, ) = reservation.instances
try:
instance.add_tag('Name',
'Scheduled {} {}'.format(yyyymmdd, command[0]))
except EC2ResponseError:
time.sleep(10)
try:
instance.add_tag('Name',
'Scheduled {} {}'.format(yyyymmdd, command[0]))
except EC2ResponseError:
time.sleep(10)
instance.add_tag('Name',
'Scheduled {} {}'.format(yyyymmdd, command[0]))
_L.info('Started EC2 instance {} from AMI {}'.format(instance, image))
return instance
