def launch_instance(args, userdata=None):
"""Connect to AWS and launch instance using args from create_parser"""
# Make userdata unless given
userdata = userdata or create_userdata(args.mirrorurl, args.ksurl)
# Connect to EC2 endpoint for region
conn = connect_to_region(args.region)
# Choose first image ID that matches the given AMI name pattern
try:
id = conn.get_all_images(filters={'name': args.bootami})[0].id
except IndexError:
raise Error('ERROR: No matching AMIs found!')
# Connect to the given SubnetID or get a list of subnets in this region
if args.novpc:
subnets = None
else:
c = vpc.connect_to_region(args.region)
subnets = c.get_all_subnets(args.subnetid)
# Use a VPC if we can, unless told not to. Use first subnet in list.
if subnets:
grpfilt = {'group-name': args.secgroup, 'vpc_id': subnets[0].vpc_id}
subnetid = subnets[0].id
# Find the security group id from the name
group = conn.get_all_security_groups(filters=grpfilt)[0].id
# associate the instance with a VPC and give it a puclic IP address
interface = networkinterface.NetworkInterfaceSpecification(
subnet_id=subnetid,
groups=[group],
associate_public_ip_address=True)
interfaces = networkinterface.NetworkInterfaceCollection(interface)
groups = None
else:
interfaces = None
groups = [args.secgroup]
# Set disk mapping if needed
if args.disksize:
dev_xvda = blockdevicemapping.BlockDeviceType(
delete_on_termination=True)
dev_xvda.size = args.disksize
device_map = blockdevicemapping.BlockDeviceMapping()
device_map['/dev/xvda'] = dev_xvda
else:
device_map = None
# launch instance
res = conn.run_instances(id,
key_name=args.key,
instance_type=args.type,
network_interfaces=interfaces,
user_data=userdata,
security_groups=groups,
block_device_map=device_map)
return res.instances[0]
def device(d):
dev = bdm.BlockDeviceType()
if d['VirtualName'].startswith('ephemeral'):
# Instance Storage
dev.ephemeral_name = d['VirtualName']
else:
# EBS
dev.size = d['Ebs.VolumeSize']
delete = d.get('Ebs.DeleteOnTermination', None)
if delete is not None:
dev.delete_on_termination = delete
return (d['DeviceName'], dev)
def request_task_instance(ec2, autoscale, instance_type, lifespan, command, bucket, aws_sns_arn, patch_version, tempsize):
'''
'''
major_version = patch_version.split('.')[0]
group_name = 'CI Workers {0}.x'.format(major_version)
(group, ) = autoscale.get_all_groups([group_name])
(config, ) = autoscale.get_all_launch_configurations(names=[group.launch_config_name])
(image, ) = ec2.get_all_images(image_ids=[config.image_id])
keypair = [kp for kp in ec2.get_all_key_pairs() if kp.name.startswith('oa-')][0]
yyyymmdd = datetime.utcnow().strftime('%Y-%m-%d-%H-%M')
with open(first_file(userdata_paths)) as file:
userdata_kwargs = dict(
command = ' '.join(map(shlex.quote, command)),
lifespan = shlex.quote(str(lifespan)),
major_version = shlex.quote(major_version),
patch_version = shlex.quote(patch_version),
log_prefix = shlex.quote('logs/{}-{}'.format(yyyymmdd, command[0])),
bucket = shlex.quote(bucket or 'data.openaddresses.io'),
aws_sns_arn = '', aws_region = '',
command_name = command[0]
)
if aws_sns_arn:
try:
_, _, _, aws_region, _ = aws_sns_arn.split(':', 4)
except ValueError:
pass
else:
if aws_sns_arn.startswith('arn:aws:sns:'):
userdata_kwargs.update(aws_sns_arn = shlex.quote(aws_sns_arn),
aws_region = shlex.quote(aws_region))
device_map = blockdevicemapping.BlockDeviceMapping()
if tempsize:
dev_sdb = blockdevicemapping.BlockDeviceType(delete_on_termination=True)
dev_sdb.size = tempsize
device_map['/dev/sdb'] = dev_sdb
run_kwargs = dict(instance_type=instance_type, security_groups=['default'],
instance_initiated_shutdown_behavior='terminate',
# TODO: use current role from http://169.254.169.254/latest/meta-data/iam/info
instance_profile_name='machine-communication',
key_name=keypair.name, block_device_map=device_map)
print('Running with keyword args:', pprint.pformat(run_kwargs), sep='\n', file=sys.stderr)
run_kwargs.update(user_data=file.read().format(**userdata_kwargs))
reservation = image.run(**run_kwargs)
(instance, ) = reservation.instances
try:
instance.add_tag('Name', 'Scheduled {} {}'.format(yyyymmdd, command[0]))
instance.add_tag('Command', command[0])
instance.add_tag('Trigger', 'run-ec2-command')
except EC2ResponseError:
time.sleep(10)
try:
instance.add_tag('Name', 'Scheduled {} {}'.format(yyyymmdd, command[0]))
instance.add_tag('Command', command[0])
instance.add_tag('Trigger', 'run-ec2-command')
except EC2ResponseError:
time.sleep(30)
instance.add_tag('Name', 'Scheduled {} {}'.format(yyyymmdd, command[0]))
instance.add_tag('Command', command[0])
instance.add_tag('Trigger', 'run-ec2-command')
print('Started EC2 instance {} from AMI {}'.format(instance, image), file=sys.stderr)
return instance
def main():
print('Starting nightly build: {}'.format(strftime("%Y-%m-%d %H:%M:%S")))
print('Opening connection..')
access_key = os.environ.get('AWS_ACCESS_KEY_ID')
secret_key = os.environ.get('AWS_ACCESS_KEY')
conn = boto.ec2.connect_to_region(settings['region'],
aws_access_key_id=access_key,
aws_secret_access_key=secret_key)
RESOURCES.append(conn)
print('Running Packer..')
baked_ami_id = run_packer()
baked_ami = conn.get_image(baked_ami_id)
RESOURCES.append(baked_ami)
baked_snap = baked_ami.block_device_mapping['/dev/sda1'].snapshot_id
print('Launching worker machine..')
mapping = bdm.BlockDeviceMapping()
mapping['/dev/sda1'] = bdm.BlockDeviceType(size=10,
volume_type='gp2',
delete_on_termination=True)
mapping['/dev/sdf'] = bdm.BlockDeviceType(snapshot_id=baked_snap,
volume_type='gp2',
delete_on_termination=True)
kp_name = random_generator()
kp = conn.create_key_pair(kp_name)
kp.save(gettempdir())
print('Keypair created: {}'.format(kp_name))
sg_name = random_generator()
sg = conn.create_security_group(sg_name, 'vagrant nightly')
sg.authorize(ip_protocol='tcp',
from_port=22,
to_port=22,
cidr_ip='0.0.0.0/0')
print('Security Group created: {}'.format(sg_name))
reserv = conn.run_instances(
image_id=settings['factory_ami'],
key_name=kp_name,
instance_type=settings['instance_type'],
security_groups=[sg],
block_device_map=mapping,
instance_profile_name=settings['aws_iam_group'])
factory_instance = reserv.instances[0]
RESOURCES.append(factory_instance)
RESOURCES.append(kp)
RESOURCES.append(sg)
env.key_filename = os.path.join(gettempdir(), '{}.pem'.format(kp_name))
env.timeout = 10
env.connection_attempts = 12
while factory_instance.state != 'running':
sleep(5)
factory_instance.update()
print('machine state: {}'.format(factory_instance.state))
print('Executing script..')
execute(do_work,
host='{}@{}'.format(settings['username'],
factory_instance.ip_address))
def request_task_instance(ec2, autoscale, instance_type, chef_role, lifespan,
command, bucket, aws_sns_arn):
'''
'''
group_name = 'CI Workers {0}.x'.format(*get_version().split('.'))
(group, ) = autoscale.get_all_groups([group_name])
(config, ) = autoscale.get_all_launch_configurations(
names=[group.launch_config_name])
(image, ) = ec2.get_all_images(image_ids=[config.image_id])
keypair = [
kp for kp in ec2.get_all_key_pairs() if kp.name.startswith('oa-')
][0]
yyyymmdd = datetime.utcnow().strftime('%Y-%m-%d-%H-%M')
with open(join(dirname(__file__), 'templates',
'task-instance-userdata.sh')) as file:
userdata_kwargs = dict(role=shlex.quote(chef_role),
command=' '.join(map(shlex.quote, command)),
lifespan=shlex.quote(str(lifespan)),
version=shlex.quote(get_version()),
log_prefix=shlex.quote('logs/{}-{}'.format(
yyyymmdd, command[0])),
bucket=shlex.quote(bucket
or 'data.openaddresses.io'),
aws_sns_arn='',
aws_region='',
command_name=command[0])
if aws_sns_arn:
try:
_, _, _, aws_region, _ = aws_sns_arn.split(':', 4)
except ValueError:
pass
else:
if aws_sns_arn.startswith('arn:aws:sns:'):
userdata_kwargs.update(
aws_sns_arn=shlex.quote(aws_sns_arn),
aws_region=shlex.quote(aws_region))
device_map = blockdevicemapping.BlockDeviceMapping()
if instance_type in block_device_sizes:
device_map = blockdevicemapping.BlockDeviceMapping()
dev_sdb = blockdevicemapping.BlockDeviceType()
dev_sdb.size = block_device_sizes[instance_type]
device_map['/dev/sdb'] = dev_sdb
run_kwargs = dict(
instance_type=instance_type,
security_groups=['default'],
instance_initiated_shutdown_behavior='terminate',
user_data=file.read().format(**userdata_kwargs),
# TODO: use current role from http://169.254.169.254/latest/meta-data/iam/info
instance_profile_name='machine-communication',
key_name=keypair.name,
block_device_map=device_map)
reservation = image.run(**run_kwargs)
(instance, ) = reservation.instances
try:
instance.add_tag('Name',
'Scheduled {} {}'.format(yyyymmdd, command[0]))
except EC2ResponseError:
time.sleep(10)
try:
instance.add_tag('Name',
'Scheduled {} {}'.format(yyyymmdd, command[0]))
except EC2ResponseError:
time.sleep(10)
instance.add_tag('Name',
'Scheduled {} {}'.format(yyyymmdd, command[0]))
_L.info('Started EC2 instance {} from AMI {}'.format(instance, image))
return instance