def test_set_desired_capacity_the_same():
conn = boto.connect_autoscale()
config = LaunchConfiguration(
name='tester',
image_id='ami-abcd1234',
instance_type='t2.medium',
)
conn.create_launch_configuration(config)
group = AutoScalingGroup(
name='tester_group',
availability_zones=['us-east-1c', 'us-east-1b'],
desired_capacity=2,
max_size=2,
min_size=2,
launch_config=config,
vpc_zone_identifier='subnet-1234abcd',
)
conn.create_auto_scaling_group(group)
group = conn.get_all_groups()[0]
group.desired_capacity.should.equal(2)
instances = list(conn.get_all_autoscaling_instances())
instances.should.have.length_of(2)
conn.set_desired_capacity("tester_group", 2)
group = conn.get_all_groups()[0]
group.desired_capacity.should.equal(2)
instances = list(conn.get_all_autoscaling_instances())
instances.should.have.length_of(2)
def test_launch_config_edit(sys, user_input):
add_launch_config("web", user_data="echo 'web_machine' > /etc/config")
sys.argv = [
'autoscaler_launch_config',
'edit',
'web',
]
# "image_id", "key_name", "security_groups", "user_data", "instance_type",
# "kernel_id", "ramdisk_id", "block_device_mappings", "instance_monitoring"
user_input.side_effect = [
"",
"",
"",
"echo 'other_machine' > /etc/config",
"",
"",
"",
"",
"yes",
"arn:aws:iam::123456789012:instance-profile/tester",
"",
]
# Simulate CLI call
launch_config()
conn = boto.connect_autoscale()
configs = conn.get_all_launch_configurations(names=['web'])
configs.should.have.length_of(1)
web_config = configs[0]
web_config.user_data.should.equal("echo 'other_machine' > /etc/config")
def find_one_resource(stack, resource_type, only_id=False):
stackresources = stack.describe_resources()
resources = [r for r in stackresources if r.resource_type == resource_type]
if len(resources) == 0:
raise ValueError("This stack contains no AutoScale")
if len(resources) > 1:
raise ValueError("This stack contains more than one AutoScale")
phy_id = resources[0].physical_resource_id
if resource_type == RES_TYPE_ASG:
if only_id:
return phy_id
else:
try:
return boto.connect_autoscale().get_all_groups([phy_id])[0]
except IndexError:
raise ValueError("The AutoScale physical id doesn't exist")
elif resource_type == RES_TYPE_ELB:
if only_id:
return phy_id
else:
try:
return boto.connect_elb().get_all_load_balancers([phy_id])[0]
except IndexError:
raise ValueError("The ELB physical id doesn't exist")
else:
raise NotImplementedError("Unkown resource type")
def test_autoscaling_group_describe_instances():
mocked_networking = setup_networking_deprecated()
conn = boto.connect_autoscale()
config = LaunchConfiguration(
name='tester',
image_id='ami-abcd1234',
instance_type='t2.medium',
)
conn.create_launch_configuration(config)
group = AutoScalingGroup(
name='tester_group',
max_size=2,
min_size=2,
launch_config=config,
vpc_zone_identifier=mocked_networking['subnet1'],
)
conn.create_auto_scaling_group(group)
instances = list(conn.get_all_autoscaling_instances())
instances.should.have.length_of(2)
instances[0].launch_config_name.should.equal('tester')
instances[0].health_status.should.equal('Healthy')
autoscale_instance_ids = [instance.instance_id for instance in instances]
ec2_conn = boto.connect_ec2()
reservations = ec2_conn.get_all_instances()
instances = reservations[0].instances
instances.should.have.length_of(2)
instance_ids = [instance.id for instance in instances]
set(autoscale_instance_ids).should.equal(set(instance_ids))
instances[0].instance_type.should.equal("t2.medium")
def main():
parser = argparse.ArgumentParser(description='Update the launch config for an autoscale group to use a new AMI')
parser.add_argument('template_instance', action='store')
parser.add_argument('autoscale_group', action='store')
parser.add_argument('new_lc_name', action='store')
args = parser.parse_args()
ec2conn = boto.connect_ec2()
image_id = ec2conn.create_image(args.template_instance, args.new_lc_name)
time.sleep(3)
image = ec2conn.get_all_images([image_id])[0]
print "Creating AMI: " + image_id
while (image.state != 'available'):
time.sleep(3)
image = ec2conn.get_all_images([image_id])[0]
print "Done"
asconn = boto.connect_autoscale()
group = asconn.get_all_groups([args.autoscale_group], 1)[0]
old_lc_name = group.launch_config_name
print 'Old launch configuration: ' + old_lc_name
launch_config = asconn.get_all_launch_configurations(names=[old_lc_name],
max_records=1)[0]
print 'Old AMI: ' + launch_config.image_id
launch_config.name = args.new_lc_name
launch_config.image_id = image_id
print 'Creating launch configuration: ' + args.new_lc_name
asconn.create_launch_configuration(launch_config)
print 'Updating autoscale group with new launch configuration'
group.launch_config_name = launch_config.name
# ugh fix this.
# currently, boto does not get the group's max size. fill it in manually.
# will be fixed in 2.6.0 or further patches to 2.5.
group.max_size = 100
group.update()
print 'Group ' + group.name + ' configured to use launch configuration ' + launch_config.name + ' with AMI ' + launch_config.image_id
def test_autoscaling_group_describe_filter():
mocked_networking = setup_networking_deprecated()
conn = boto.connect_autoscale()
config = LaunchConfiguration(
name='tester',
image_id='ami-abcd1234',
instance_type='t2.medium',
)
conn.create_launch_configuration(config)
group = AutoScalingGroup(
name='tester_group',
max_size=2,
min_size=2,
launch_config=config,
vpc_zone_identifier=mocked_networking['subnet1'],
)
conn.create_auto_scaling_group(group)
group.name = 'tester_group2'
conn.create_auto_scaling_group(group)
group.name = 'tester_group3'
conn.create_auto_scaling_group(group)
conn.get_all_groups(
names=['tester_group', 'tester_group2']).should.have.length_of(2)
conn.get_all_groups().should.have.length_of(3)
def test_create_autoscaling_groups_defaults():
""" Test with the minimum inputs and check that all of the proper defaults
are assigned for the other attributes """
conn = boto.connect_autoscale()
config = LaunchConfiguration(
name='tester',
image_id='ami-abcd1234',
instance_type='m1.small',
)
conn.create_launch_configuration(config)
group = AutoScalingGroup(
name='tester_group',
max_size=2,
min_size=2,
launch_config=config,
)
conn.create_auto_scaling_group(group)
group = conn.get_all_groups()[0]
group.name.should.equal('tester_group')
group.max_size.should.equal(2)
group.min_size.should.equal(2)
group.launch_config_name.should.equal('tester')
# Defaults
list(group.availability_zones).should.equal([])
group.desired_capacity.should.equal(2)
group.vpc_zone_identifier.should.equal('')
group.default_cooldown.should.equal(300)
group.health_check_period.should.equal(None)
group.health_check_type.should.equal("EC2")
list(group.load_balancers).should.equal([])
group.placement_group.should.equal(None)
list(group.termination_policies).should.equal([])
def get_map(ec2):
""" Map the data from each available connection """
# Get extra connections
elb = boto.connect_elb(ec2.access_key, ec2.secret_key)
asg = boto.connect_autoscale(ec2.access_key, ec2.secret_key)
s3b = boto.connect_s3(ec2.access_key,ec2.secret_key)
# EC2 Keypairs
keys = {}
for k in ec2.get_all_key_pairs():
keys[k.name] = k.fingerprint
# EC2 Security Groups
security_groups = {}
for s in ec2.get_all_security_groups():
rules = {}
for r in s.rules:
g = str(r.grants)
if g not in rules: rules[g] = []
rules[g].append('%s:[%s%s]' % (r.ip_protocol, r.from_port,
r.to_port != r.from_port and '-'+r.to_port or ''))
security_groups[s.name] = rules
# Elastic Load Balancers
elbs = {}
for lb in elb.get_all_load_balancers():
info = {}
info['instances'] = lb.instances
info['dns_name'] = lb.dns_name
elbs[lb.name] = info
# Need to map out 'asg'
# * Launch Configurations
# * AutoScaling Groups
# * AutoScaling Triggers and Instances
# S3 Buckets
buckets = {}
for b in s3b.get_all_buckets():
buckets[b.name] = b
# EC2 Instances
instances = {}
for r in ec2.get_all_instances():
for i in r.instances:
if i.image_id not in instances:
instances[i.image_id] = {}
if i.state not in instances[i.image_id]:
instances[i.image_id][i.state] = []
instances[i.image_id][i.state].append(i)
data = {
'asgs': {},
'elbs': elbs,
'instances': instances,
'keys': keys,
's3bs': buckets,
'security_groups': security_groups,
}
return data
def test_launch_config_edit(sys, read_input):
add_launch_config(
"web",
user_data="echo 'web_machine' > /etc/config",
spot_price=0.2,
instance_monitoring=True,
)
sys.argv = [
'autoscaler_launch_config',
'edit',
'web',
]
# "image_id", "key_name", "security_groups", "user_data", "instance_type",
# "kernel_id", "ramdisk_id", "block_device_mappings", "instance_monitoring"
# "instance_profile_name", "spot_price", "ebs_optimized", "associate_public_ip_address"
read_input.side_effect = [
"",
"",
"",
"echo 'other_machine' > /etc/config",
"",
"",
"",
"",
"yes",
"arn:aws:iam::123456789012:instance-profile/tester",
"0.1",
"yes",
"yes",
]
# Simulate CLI call
launch_config()
list(read_input.mock_calls).should.equal([
call('What image_id?', u'None'),
call('What key_name?', ''),
call('What security_groups?', ''),
call('What user_data?', "echo 'web_machine' > /etc/config"),
call('What instance_type?', u'm1.small'),
call('What kernel_id?', ''),
call('What ramdisk_id?', ''),
call('What block_device_mappings?', []),
call('What instance_monitoring?', "yes"),
call('What instance_profile_name?', None),
call('What spot_price?', 0.2),
call('What ebs_optimized?', "no"),
call('What associate_public_ip_address?', False),
])
conn = boto.connect_autoscale()
configs = conn.get_all_launch_configurations(names=['web'])
configs.should.have.length_of(1)
web_config = configs[0]
web_config.user_data.should.equal("echo 'other_machine' > /etc/config")
web_config.spot_price.should.equal(0.1)
web_config.ebs_optimized.should.equal(True)
web_config.associate_public_ip_address.should.equal(True)
def get_group_attributes_or_defaults(group_name):
conn = boto.connect_autoscale()
groups = conn.get_all_groups(names=[group_name])
if groups:
return attrs_from_group(groups[0])
else:
return empty_group_attrs
def test_add_launch_configuration_with_named_base():
conn = boto.connect_autoscale()
config = LaunchConfiguration(
name='named_default_copy_from',
image_id='ami-1234abcd',
key_name='tester',
security_groups=["default"],
user_data="echo 'default_machine' > /etc/config",
instance_type='m1.large',
instance_monitoring=True,
instance_profile_name='arn:aws:iam::123456789012:instance-profile/tester',
spot_price=0.1,
)
conn.create_launch_configuration(config)
conn.get_all_launch_configurations().should.have.length_of(1)
add_launch_config("web", base='named_default_copy_from', user_data="echo 'web_machine' > /etc/config")
configs = conn.get_all_launch_configurations(names=['web'])
configs.should.have.length_of(1)
web_config = configs[0]
web_config.user_data.should.equal("echo 'web_machine' > /etc/config")
web_config.image_id.should.equal('ami-1234abcd')
web_config.key_name.should.equal('tester')
web_config.instance_profile_name.should.equal('arn:aws:iam::123456789012:instance-profile/tester')
web_config.spot_price.should.equal(0.1)
def main():
tag_Name = sys.argv[1]
autoscale = boto.connect_autoscale()
tags = autoscale.get_all_tags()
for tag in tags:
if tag.key == 'Name' and tag.value == tag_Name:
print tag.resource_id
def setUpAutoScaleGroup(self, configurations, env="stg"):
conn = boto.connect_autoscale()
for configuration in configurations:
config = LaunchConfiguration(
name=configuration[self.launch_configuration_name],
image_id='ami-abcd1234',
instance_type='m1.medium',
)
load_balancer_name = 'servergmsextenderELB{0}'.format(env)
group = AutoScalingGroup(
name=configuration[self.autoscaling_group_name],
availability_zones=['us-east-1a'],
default_cooldown=300,
desired_capacity=2,
health_check_period='0',
health_check_type="EC2",
max_size=10,
min_size=2,
launch_config=config,
load_balancers=[load_balancer_name],
vpc_zone_identifier='subnet-1234abcd',
termination_policies=["Default"],
)
conn.create_launch_configuration(config)
conn.create_auto_scaling_group(group)
def test_set_desired_capacity_down():
mocked_networking = setup_networking_deprecated()
conn = boto.connect_autoscale()
config = LaunchConfiguration(
name='tester',
image_id='ami-abcd1234',
instance_type='t2.medium',
)
conn.create_launch_configuration(config)
group = AutoScalingGroup(
name='tester_group',
availability_zones=['us-east-1a'],
desired_capacity=2,
max_size=2,
min_size=2,
launch_config=config,
vpc_zone_identifier=mocked_networking['subnet1'],
)
conn.create_auto_scaling_group(group)
group = conn.get_all_groups()[0]
group.desired_capacity.should.equal(2)
instances = list(conn.get_all_autoscaling_instances())
instances.should.have.length_of(2)
conn.set_desired_capacity("tester_group", 1)
group = conn.get_all_groups()[0]
group.desired_capacity.should.equal(1)
instances = list(conn.get_all_autoscaling_instances())
instances.should.have.length_of(1)
def test_create_launch_configuration():
conn = boto.connect_autoscale()
config = LaunchConfiguration(
name='tester',
image_id='ami-abcd1234',
instance_type='m1.small',
key_name='the_keys',
security_groups=["default", "default2"],
user_data="This is some user_data",
instance_monitoring=True,
instance_profile_name='arn:aws:iam::123456789012:instance-profile/testing',
spot_price=0.1,
)
conn.create_launch_configuration(config)
launch_config = conn.get_all_launch_configurations()[0]
launch_config.name.should.equal('tester')
launch_config.image_id.should.equal('ami-abcd1234')
launch_config.instance_type.should.equal('m1.small')
launch_config.key_name.should.equal('the_keys')
set(launch_config.security_groups).should.equal(set(['default', 'default2']))
launch_config.user_data.should.equal("This is some user_data")
launch_config.instance_monitoring.enabled.should.equal('true')
launch_config.instance_profile_name.should.equal('arn:aws:iam::123456789012:instance-profile/testing')
launch_config.spot_price.should.equal(0.1)
def _get_autoscale_group(name):
conn = boto.connect_autoscale()
groups = conn.get_all_groups(names=[name])
if len(groups) == 1:
return groups[0]
raise ValueError('Autoscale group %s not found' % name)
def _setup_test_asg_to_be_deleted(self):
"""
Setup a test ASG that is tagged to be deleted.
"""
# pylint: disable=attribute-defined-outside-init
self.test_asg_name = "test-asg-random-tags"
self.test_autoscale = boto.connect_autoscale()
launch_config = boto.ec2.autoscale.LaunchConfiguration(
name='my-launch_config',
image_id='my-ami',
key_name='my_key_name',
security_groups=['my_security_groups']
)
self.test_autoscale.create_launch_configuration(launch_config)
asg = boto.ec2.autoscale.AutoScalingGroup(
group_name=self.test_asg_name,
load_balancers=['my-lb'],
availability_zones=['us-east-1a', 'us-east-1b'],
launch_config=launch_config,
min_size=4,
max_size=8,
connection=self.test_autoscale
)
create_elb('my-lb')
self.test_autoscale.create_auto_scaling_group(asg)
ec2.tag_asg_for_deletion(self.test_asg_name, 0)
self.test_asg = self.test_autoscale.get_all_groups([self.test_asg_name])[0]
def test_autoscaling_update():
conn = boto.connect_autoscale()
config = LaunchConfiguration(
name='tester',
image_id='ami-abcd1234',
instance_type='t2.medium',
)
conn.create_launch_configuration(config)
group = AutoScalingGroup(
name='tester_group',
availability_zones=['us-east-1c', 'us-east-1b'],
desired_capacity=2,
max_size=2,
min_size=2,
launch_config=config,
vpc_zone_identifier='subnet-1234abcd',
)
conn.create_auto_scaling_group(group)
group = conn.get_all_groups()[0]
group.vpc_zone_identifier.should.equal('subnet-1234abcd')
group.vpc_zone_identifier = 'subnet-5678efgh'
group.update()
group = conn.get_all_groups()[0]
group.vpc_zone_identifier.should.equal('subnet-5678efgh')
def test_autoscaling_update():
mocked_networking = setup_networking_deprecated()
conn = boto.connect_autoscale()
config = LaunchConfiguration(
name='tester',
image_id='ami-abcd1234',
instance_type='t2.medium',
)
conn.create_launch_configuration(config)
group = AutoScalingGroup(
name='tester_group',
desired_capacity=2,
max_size=2,
min_size=2,
launch_config=config,
vpc_zone_identifier=mocked_networking['subnet1'],
)
conn.create_auto_scaling_group(group)
group = conn.get_all_groups()[0]
group.availability_zones.should.equal(['us-east-1a'])
group.vpc_zone_identifier.should.equal(mocked_networking['subnet1'])
group.availability_zones = ['us-east-1b']
group.vpc_zone_identifier = mocked_networking['subnet2']
group.update()
group = conn.get_all_groups()[0]
group.availability_zones.should.equal(['us-east-1b'])
group.vpc_zone_identifier.should.equal(mocked_networking['subnet2'])
def test_autoscaling_group_describe_instances():
conn = boto.connect_autoscale()
config = LaunchConfiguration(
name='tester',
image_id='ami-abcd1234',
instance_type='t2.medium',
)
conn.create_launch_configuration(config)
group = AutoScalingGroup(
name='tester_group',
max_size=2,
min_size=2,
launch_config=config,
)
conn.create_auto_scaling_group(group)
instances = list(conn.get_all_autoscaling_instances())
instances.should.have.length_of(2)
instances[0].launch_config_name.should.equal('tester')
autoscale_instance_ids = [instance.instance_id for instance in instances]
ec2_conn = boto.connect_ec2()
reservations = ec2_conn.get_all_instances()
instances = reservations[0].instances
instances.should.have.length_of(2)
instance_ids = [instance.id for instance in instances]
set(autoscale_instance_ids).should.equal(set(instance_ids))
instances[0].instance_type.should.equal("t2.medium")
def edit_launch_config(name, **kwargs):
conn = boto.connect_autoscale()
configs = conn.get_all_launch_configurations(names=[name])
if not configs:
raise AutoScalerException("No launch configuration could be found for %s", name)
config = configs[0]
temp_name = "{}-autoscaler-temp".format(name)
config_attrs = attrs_from_config(config)
config_attrs.update(kwargs)
# Create temp config and reassign groups to it
add_launch_config(temp_name, **config_attrs)
update_all_groups(name, temp_name)
# Delete the old config
conn.delete_launch_configuration(name)
# Create new config with the original name and reassign groups
new_config = add_launch_config(name, **config_attrs)
update_all_groups(temp_name, name)
# Delete the temp config
conn.delete_launch_configuration(temp_name)
return new_config
def test_create_launch_configuration_with_block_device_mappings():
block_device_mapping = BlockDeviceMapping()
ephemeral_drive = BlockDeviceType()
ephemeral_drive.ephemeral_name = 'ephemeral0'
block_device_mapping['/dev/xvdb'] = ephemeral_drive
snapshot_drive = BlockDeviceType()
snapshot_drive.snapshot_id = "snap-1234abcd"
snapshot_drive.volume_type = "standard"
block_device_mapping['/dev/xvdp'] = snapshot_drive
ebs_drive = BlockDeviceType()
ebs_drive.volume_type = "io1"
ebs_drive.size = 100
ebs_drive.iops = 1000
ebs_drive.delete_on_termination = False
block_device_mapping['/dev/xvdh'] = ebs_drive
conn = boto.connect_autoscale(use_block_device_types=True)
config = LaunchConfiguration(
name='tester',
image_id='ami-abcd1234',
instance_type='m1.small',
key_name='the_keys',
security_groups=["default", "default2"],
user_data="This is some user_data",
instance_monitoring=True,
instance_profile_name='arn:aws:iam::123456789012:instance-profile/testing',
spot_price=0.1,
block_device_mappings=[block_device_mapping]
)
conn.create_launch_configuration(config)
launch_config = conn.get_all_launch_configurations()[0]
launch_config.name.should.equal('tester')
launch_config.image_id.should.equal('ami-abcd1234')
launch_config.instance_type.should.equal('m1.small')
launch_config.key_name.should.equal('the_keys')
set(launch_config.security_groups).should.equal(set(['default', 'default2']))
launch_config.user_data.should.equal("This is some user_data")
launch_config.instance_monitoring.enabled.should.equal('true')
launch_config.instance_profile_name.should.equal('arn:aws:iam::123456789012:instance-profile/testing')
launch_config.spot_price.should.equal(0.1)
len(launch_config.block_device_mappings).should.equal(3)
returned_mapping = launch_config.block_device_mappings
set(returned_mapping.keys()).should.equal(set(['/dev/xvdb', '/dev/xvdp', '/dev/xvdh']))
returned_mapping['/dev/xvdh'].iops.should.equal(1000)
returned_mapping['/dev/xvdh'].size.should.equal(100)
returned_mapping['/dev/xvdh'].volume_type.should.equal("io1")
returned_mapping['/dev/xvdh'].delete_on_termination.should.be.false
returned_mapping['/dev/xvdp'].snapshot_id.should.equal("snap-1234abcd")
returned_mapping['/dev/xvdp'].volume_type.should.equal("standard")
returned_mapping['/dev/xvdb'].ephemeral_name.should.equal('ephemeral0')
def add_launch_config(name, base=DEFAULT_CONFIG_NAME, **kwargs):
conn = boto.connect_autoscale()
attributes = get_config_values(base)
attributes.update(kwargs)
attributes['name'] = name
config = LaunchConfiguration(**attributes)
conn.create_launch_configuration(config)
return config
def delete_autoscaling_notifications(asg_name, topicArn):
asgconn = boto.connect_autoscale()
params = {
"AutoScalingGroupName":asg_name,
"TopicARN":topicArn,
}
status = asgconn.get_status("DeleteNotificationConfiguration", params)
assert status, "error during DeleteNotificationConfiguration"
def connect(self):
"""Connect to API if not already connected; set self.conn."""
if self.conn is not None:
return
elif self.region:
self.conn = self.connect_via(boto.ec2.autoscale.connect_to_region)
else:
self.conn = boto.connect_autoscale()
def test_create_launch_configuration_using_ip_association_should_default_to_false():
conn = boto.connect_autoscale()
config = LaunchConfiguration(
name='tester',
image_id='ami-abcd1234', )
conn.create_launch_configuration(config)
launch_config = conn.get_all_launch_configurations()[0]
launch_config.associate_public_ip_address.should.equal(False)
def edit_auto_scaling_group(name, **kwargs):
conn = boto.connect_autoscale()
groups = conn.get_all_groups(names=[name])
if not groups:
raise AutoScalerException("No autoscaling groups could be found for %s", name)
group = groups[0]
for attr_name, attr_value in kwargs.items():
setattr(group, attr_name, attr_value)
group.update()
def groups_for_token(token):
groups = []
conn = boto.connect_autoscale()
groups_iter = conn.get_all_groups(next_token=token)
groups.extend(groups_iter)
if groups_iter.next_token:
groups.extend(groups_for_token(groups_iter.next_token))
return groups
def test_add_launch_configuration():
add_launch_config("web", user_data="echo 'web_machine' > /etc/config")
conn = boto.connect_autoscale()
configs = conn.get_all_launch_configurations(names=['web'])
configs.should.have.length_of(1)
web_config = configs[0]
web_config.user_data.should.equal("echo 'web_machine' > /etc/config")
def test_wait_for_in_service_health_failure(self):
asg_name = "unhealthy_asg"
create_asg_with_tags(asg_name, {"foo": "bar"})
autoscale = boto.connect_autoscale()
asgs = autoscale.get_all_groups([asg_name])
asg = asgs[0]
asg.instances[0].health_status = "Unhealthy"
with mock.patch("boto.ec2.autoscale.AutoScaleConnection.get_all_groups", return_value=asgs):
self.assertRaises(TimeoutException, ec2.wait_for_in_service, [asg_name], 2)
def test_launch_configuration_delete():
conn = boto.connect_autoscale()
config = LaunchConfiguration(
name='tester',
image_id='ami-abcd1234',
instance_type='m1.small',
)
conn.create_launch_configuration(config)
conn.get_all_launch_configurations().should.have.length_of(1)
conn.delete_launch_configuration('tester')
conn.get_all_launch_configurations().should.have.length_of(0)
def main():
''' Single threaded worker to serve the job queue.
'''
args = parser.parse_args()
setup_logger(args.access_key, args.secret_key, args.sns_arn, log_level=args.loglevel)
s3 = S3(args.access_key, args.secret_key, args.bucket)
autoscale = connect_autoscale(args.access_key, args.secret_key)
cloudwatch = connect_cloudwatch(args.access_key, args.secret_key)
github_auth = args.github_token, 'x-oauth-basic'
next_queue_interval, next_autoscale_interval = 60, 43200
try:
with db_connect(args.database_url) as conn:
task_Q = db_queue(conn, TASK_QUEUE)
next_queue_report = time() + next_queue_interval
next_autoscale_grow = time() + next_autoscale_interval
minimum_capacity = count(1)
with task_Q as db:
run_times = get_batch_run_times(db, args.owner, args.repository)
sources = find_batch_sources(args.owner, args.repository, github_auth, run_times)
with task_Q as db:
new_set = add_set(db, args.owner, args.repository)
for expected_count in enqueue_sources(task_Q, new_set, sources):
if time() >= next_queue_report:
next_queue_report, n = time() + next_queue_interval, len(task_Q)
args = n, 's' if n != 1 else '', expected_count
_L.debug('Task queue has {} item{}, {} sources expected'.format(*args))
try:
if time() >= next_autoscale_grow:
next_autoscale_grow = time() + next_autoscale_interval
set_autoscale_capacity(autoscale, cloudwatch, next(minimum_capacity))
except Exception as e:
_L.error('Problem during autoscale', exc_info=True)
if expected_count:
sleep(5)
with task_Q as db:
_L.debug('Rendering that shit')
render_set_maps(s3, db, new_set)
except:
_L.error('Error in worker main()', exc_info=True)
return 1
else:
return 0
def test_autoscaling_group_add(sys, user_input):
sys.argv = [
'autoscaler_auto_scaling_group',
'add',
'web',
]
# INPUTS
# "availability_zones", "default_cooldown", "desired_capacity",
# "health_check_period", "health_check_type", "launch_config_name",
# "load_balancers", "max_size", "min_size", "placement_group",
# "vpc_zone_identifier", "termination_policies"
user_input.side_effect = [
'us-east-1b,us-east-1c',
'60',
"2",
"",
"",
"web_config",
"",
"2",
"2",
"",
"",
"",
]
# Create the launch config
add_launch_config("web_config")
# Simulate CLI call
autoscaling_group()
conn = boto.connect_autoscale(use_block_device_types=True)
configs = conn.get_all_groups()
configs.should.have.length_of(1)
config = configs[0]
config.name.should.equal("web")
set(config.availability_zones).should.equal(
set(["us-east-1b", "us-east-1c"]))
config.default_cooldown.should.equal(60)
config.desired_capacity.should.equal(2)
config.health_check_period.should.equal(None)
config.health_check_type.should.equal("EC2")
config.launch_config_name.should.equal("web_config")
list(config.load_balancers).should.equal([])
config.max_size.should.equal(2)
config.min_size.should.equal(2)
config.placement_group.should.equal(None)
config.vpc_zone_identifier.should.equal("")
list(config.termination_policies).should.equal([])
def test_launch_configuration_describe_filter():
conn = boto.connect_autoscale()
config = LaunchConfiguration(name="tester",
image_id="ami-abcd1234",
instance_type="m1.small")
conn.create_launch_configuration(config)
config.name = "tester2"
conn.create_launch_configuration(config)
config.name = "tester3"
conn.create_launch_configuration(config)
conn.get_all_launch_configurations(
names=["tester", "tester2"]).should.have.length_of(2)
conn.get_all_launch_configurations().should.have.length_of(3)
def test_autoscaling_group_with_elb():
mocked_networking = setup_networking_deprecated()
elb_conn = boto.connect_elb()
zones = ['us-east-1a', 'us-east-1b']
ports = [(80, 8080, 'http'), (443, 8443, 'tcp')]
lb = elb_conn.create_load_balancer('my-lb', zones, ports)
instances_health = elb_conn.describe_instance_health('my-lb')
instances_health.should.be.empty
conn = boto.connect_autoscale()
config = LaunchConfiguration(
name='tester',
image_id='ami-abcd1234',
instance_type='t2.medium',
)
conn.create_launch_configuration(config)
group = AutoScalingGroup(
name='tester_group',
max_size=2,
min_size=2,
launch_config=config,
load_balancers=["my-lb"],
vpc_zone_identifier=mocked_networking['subnet1'],
)
conn.create_auto_scaling_group(group)
group = conn.get_all_groups()[0]
elb = elb_conn.get_all_load_balancers()[0]
group.desired_capacity.should.equal(2)
elb.instances.should.have.length_of(2)
autoscale_instance_ids = set(
instance.instance_id for instance in group.instances)
elb_instace_ids = set(instance.id for instance in elb.instances)
autoscale_instance_ids.should.equal(elb_instace_ids)
conn.set_desired_capacity("tester_group", 3)
group = conn.get_all_groups()[0]
elb = elb_conn.get_all_load_balancers()[0]
group.desired_capacity.should.equal(3)
elb.instances.should.have.length_of(3)
autoscale_instance_ids = set(
instance.instance_id for instance in group.instances)
elb_instace_ids = set(instance.id for instance in elb.instances)
autoscale_instance_ids.should.equal(elb_instace_ids)
conn.delete_auto_scaling_group('tester_group')
conn.get_all_groups().should.have.length_of(0)
elb = elb_conn.get_all_load_balancers()[0]
elb.instances.should.have.length_of(0)
def test_execute_policy_percent_change_in_capacity():
setup_autoscale_group()
conn = boto.connect_autoscale()
policy = ScalingPolicy(
name="ScaleUp",
adjustment_type="PercentChangeInCapacity",
as_name="tester_group",
scaling_adjustment=50,
)
conn.create_scaling_policy(policy)
conn.execute_policy("ScaleUp")
instances = list(conn.get_all_autoscaling_instances())
instances.should.have.length_of(3)
def test_delete_policy():
setup_autoscale_group()
conn = boto.connect_autoscale()
policy = ScalingPolicy(
name="ScaleUp",
adjustment_type="ExactCapacity",
as_name="tester_group",
scaling_adjustment=3,
)
conn.create_scaling_policy(policy)
conn.get_all_policies().should.have.length_of(1)
conn.delete_policy("ScaleUp")
conn.get_all_policies().should.have.length_of(0)
def test_execute_policy_positive_change_in_capacity():
setup_autoscale_group()
conn = boto.connect_autoscale()
policy = ScalingPolicy(
name='ScaleUp',
adjustment_type='ChangeInCapacity',
as_name='tester_group',
scaling_adjustment=3,
)
conn.create_scaling_policy(policy)
conn.execute_policy("ScaleUp")
instances = list(conn.get_all_autoscaling_instances())
instances.should.have.length_of(5)
def main():
'''
'''
args = parser.parse_args()
instance, deadline, lifespan = False, time() + (args.hours +
1) * 3600, int(args.hours *
3600)
setup_logger(None, None, args.sns_arn, log_level=args.loglevel)
ec2 = connect_ec2(None, None)
autoscale = connect_autoscale(None, None)
instance = request_task_instance(ec2, autoscale, args.instance_type,
args.role, lifespan, args.command,
args.bucket, args.sns_arn)
_L.info('instance {} is off to the races.'.format(instance))
def test_create_policy_default_values():
setup_autoscale_group()
conn = boto.connect_autoscale()
policy = ScalingPolicy(
name="ScaleUp",
adjustment_type="ExactCapacity",
as_name="tester_group",
scaling_adjustment=3,
)
conn.create_scaling_policy(policy)
policy = conn.get_all_policies()[0]
policy.name.should.equal("ScaleUp")
# Defaults
policy.cooldown.should.equal(300)
def test_launch_configuration_describe_filter():
conn = boto.connect_autoscale()
config = LaunchConfiguration(
name='tester',
image_id='ami-abcd1234',
instance_type='m1.small',
)
conn.create_launch_configuration(config)
config.name = 'tester2'
conn.create_launch_configuration(config)
config.name = 'tester3'
conn.create_launch_configuration(config)
conn.get_all_launch_configurations(
names=['tester', 'tester2']).should.have.length_of(2)
conn.get_all_launch_configurations().should.have.length_of(3)
def test_autoscaling_group_edit(sys, user_input):
add_launch_config("web_config",
user_data="echo 'web_machine' > /etc/config")
add_auto_scaling_group(
"web",
availability_zones=['us-east-1c'],
max_size=2,
min_size=2,
launch_config='web_config',
)
sys.argv = [
'autoscaler_auto_scaling_group',
'edit',
'web',
]
# INPUTS
# "availability_zones", "default_cooldown", "desired_capacity",
# "health_check_period", "health_check_type", "launch_config_name",
# "load_balancers", "max_size", "min_size", "placement_group",
# "vpc_zone_identifier", "termination_policies"
user_input.side_effect = [
'us-east-1b,us-east-1c',
'60',
"2",
"",
"",
"web_config",
"",
"1",
"2",
"",
"",
"",
]
# Create the launch config
add_launch_config("web_config")
# Simulate CLI call
autoscaling_group()
conn = boto.connect_autoscale(use_block_device_types=True)
configs = conn.get_all_groups(names=['web'])
configs.should.have.length_of(1)
web_config = configs[0]
web_config.max_size.should.equal(1)
def setup_autoscale_group():
conn = boto.connect_autoscale()
config = LaunchConfiguration(
name='tester',
image_id='ami-abcd1234',
instance_type='m1.small',
)
conn.create_launch_configuration(config)
group = AutoScalingGroup(
name='tester_group',
max_size=2,
min_size=2,
launch_config=config,
)
conn.create_auto_scaling_group(group)
return group
def setup_autoscale_group():
mocked_networking = setup_networking_deprecated()
conn = boto.connect_autoscale()
config = LaunchConfiguration(name="tester",
image_id=EXAMPLE_AMI_ID,
instance_type="m1.small")
conn.create_launch_configuration(config)
group = AutoScalingGroup(
name="tester_group",
max_size=2,
min_size=2,
launch_config=config,
vpc_zone_identifier=mocked_networking["subnet1"],
)
conn.create_auto_scaling_group(group)
return group
def tag_asg_for_deletion(asg_name, seconds_until_delete_delta=600):
"""
Tag an asg with a tag named ASG_DELETE_TAG_KEY with a value of the MS since epoch UTC + ms_until_delete_delta
that an ASG may be deleted.
Arguments:
asg_name (str): the name of the autoscale group to tag
Returns:
None
"""
tag = create_tag_for_asg_deletion(asg_name, seconds_until_delete_delta)
autoscale = boto.connect_autoscale()
if len(get_all_autoscale_groups([asg_name])) < 1:
LOG.info("ASG {} no longer exists, will not tag".format(asg_name))
else:
autoscale.create_or_update_tags([tag])
def test_launch_config_add(sys, user_input):
sys.argv = [
'autoscaler_launch_config',
'add',
'web',
]
# "image_id", "key_name", "security_groups", "user_data", "instance_type",
# "kernel_id", "ramdisk_id", "block_device_mappings", "instance_monitoring",
# "instance_profile_name", "spot_price", "ebs_optimized", "associate_public_ip_address"
user_input.side_effect = [
'ami-1234abcd',
'the_key',
"default,web",
"echo 'web' > /etc/config",
"m1.small",
"",
"",
"",
"yes",
"arn:aws:iam::123456789012:instance-profile/tester",
"0.2",
"yes",
"",
]
# Simulate CLI call
launch_config()
conn = boto.connect_autoscale(use_block_device_types=True)
configs = conn.get_all_launch_configurations()
configs.should.have.length_of(1)
config = configs[0]
config.name.should.equal("web")
config.image_id.should.equal("ami-1234abcd")
config.key_name.should.equal("the_key")
set(config.security_groups).should.equal(set(["web", "default"]))
config.user_data.should.equal("echo 'web' > /etc/config")
config.instance_type.should.equal("m1.small")
config.kernel_id.should.equal("")
config.ramdisk_id.should.equal("")
config.block_device_mappings.should.equal(BlockDeviceMapping())
config.instance_monitoring.enabled.should.equal('true')
config.spot_price.should.equal(0.2)
config.ebs_optimized.should.equal(True)
config.associate_public_ip_address.should.equal(False)
def test_autoscaling_tags_update():
mocked_networking = setup_networking_deprecated()
conn = boto.connect_autoscale()
config = LaunchConfiguration(
name="tester", image_id="ami-abcd1234", instance_type="t2.medium"
)
conn.create_launch_configuration(config)
group = AutoScalingGroup(
name="tester_group",
availability_zones=["us-east-1a"],
desired_capacity=2,
max_size=2,
min_size=2,
launch_config=config,
tags=[
Tag(
resource_id="tester_group",
key="test_key",
value="test_value",
propagate_at_launch=True,
)
],
vpc_zone_identifier=mocked_networking["subnet1"],
)
conn.create_auto_scaling_group(group)
conn.create_or_update_tags(
tags=[
Tag(
resource_id="tester_group",
key="test_key",
value="new_test_value",
propagate_at_launch=True,
),
Tag(
resource_id="tester_group",
key="test_key2",
value="test_value2",
propagate_at_launch=True,
),
]
)
group = conn.get_all_groups()[0]
group.tags.should.have.length_of(2)
def run(self):
autoscale = boto.connect_autoscale(profile_name=self.profile)
groups = autoscale.get_all_groups()
instances = self.get_instance_dict()
inventory = defaultdict(list)
for group in groups:
for instance in group.instances:
private_ip_address = instances[instance.instance_id].private_ip_address
if private_ip_address:
inventory[group.name].append(private_ip_address)
inventory[group.name + "_" + instance.lifecycle_state].append(private_ip_address)
inventory[instance.lifecycle_state.replace(":","_")].append(private_ip_address)
print json.dumps(inventory, sort_keys=True, indent=2)
def test_execute_policy_small_percent_change_in_capacity():
"""http://docs.aws.amazon.com/AutoScaling/latest/DeveloperGuide/as-scale-based-on-demand.html
If PercentChangeInCapacity returns a value between 0 and 1,
Auto Scaling will round it off to 1."""
setup_autoscale_group()
conn = boto.connect_autoscale()
policy = ScalingPolicy(
name="ScaleUp",
adjustment_type="PercentChangeInCapacity",
as_name="tester_group",
scaling_adjustment=1,
)
conn.create_scaling_policy(policy)
conn.execute_policy("ScaleUp")
instances = list(conn.get_all_autoscaling_instances())
instances.should.have.length_of(3)
def test_create_policy():
setup_autoscale_group()
conn = boto.connect_autoscale()
policy = ScalingPolicy(
name="ScaleUp",
adjustment_type="ExactCapacity",
as_name="tester_group",
scaling_adjustment=3,
cooldown=60,
)
conn.create_scaling_policy(policy)
policy = conn.get_all_policies()[0]
policy.name.should.equal("ScaleUp")
policy.adjustment_type.should.equal("ExactCapacity")
policy.as_name.should.equal("tester_group")
policy.scaling_adjustment.should.equal(3)
policy.cooldown.should.equal(60)
def test_create_autoscaling_group():
conn = boto.connect_autoscale()
config = LaunchConfiguration(
name='tester',
image_id='ami-abcd1234',
instance_type='t2.medium',
)
conn.create_launch_configuration(config)
group = AutoScalingGroup(
name='tester_group',
availability_zones=['us-east-1c', 'us-east-1b'],
default_cooldown=60,
desired_capacity=2,
health_check_period=100,
health_check_type="EC2",
max_size=2,
min_size=2,
launch_config=config,
load_balancers=["test_lb"],
placement_group="test_placement",
vpc_zone_identifier='subnet-1234abcd',
termination_policies=["OldestInstance", "NewestInstance"],
)
conn.create_auto_scaling_group(group)
group = conn.get_all_groups()[0]
group.name.should.equal('tester_group')
set(group.availability_zones).should.equal(
set(['us-east-1c', 'us-east-1b']))
group.desired_capacity.should.equal(2)
group.max_size.should.equal(2)
group.min_size.should.equal(2)
group.instances.should.have.length_of(2)
group.vpc_zone_identifier.should.equal('subnet-1234abcd')
group.launch_config_name.should.equal('tester')
group.default_cooldown.should.equal(60)
group.health_check_period.should.equal(100)
group.health_check_type.should.equal("EC2")
list(group.load_balancers).should.equal(["test_lb"])
group.placement_group.should.equal("test_placement")
list(group.termination_policies).should.equal(
["OldestInstance", "NewestInstance"])
def test_editing_launch_configuration_update_AS_groups():
conn = boto.connect_autoscale(use_block_device_types=True)
config = add_launch_config("web", user_data="echo 'web_machine' > /etc/config")
group = AutoScalingGroup(
name='web',
launch_config=config,
max_size=2,
min_size=2,
)
conn.create_auto_scaling_group(group)
# Now edit the user data
edit_launch_config("web", user_data="echo 'other_machine' > /etc/config")
group = conn.get_all_groups()[0]
config_name = group.launch_config_name
config_name.should.equal("web")
config = conn.get_all_launch_configurations(names=[config_name])[0]
config.user_data.should.equal("echo 'other_machine' > /etc/config")
def setup_autoscale_group():
mocked_networking = setup_networking_deprecated()
conn = boto.connect_autoscale()
config = LaunchConfiguration(
name='tester',
image_id='ami-abcd1234',
instance_type='m1.small',
)
conn.create_launch_configuration(config)
group = AutoScalingGroup(
name='tester_group',
max_size=2,
min_size=2,
launch_config=config,
vpc_zone_identifier=mocked_networking['subnet1'],
)
conn.create_auto_scaling_group(group)
return group
def main():
'''
'''
instance, deadline = False, time() + 12 * 3600
args = parser.parse_args()
setup_logger(args.access_key,
args.secret_key,
args.sns_arn,
log_level=args.loglevel)
try:
ec2 = connect_ec2(args.access_key, args.secret_key)
autoscale = connect_autoscale(args.access_key, args.secret_key)
instance = request_task_instance(ec2, autoscale, args.instance_type,
args.role, args.command)
while True:
instance.update()
_L.debug('{:.0f} seconds to go, instance is {}...'.format(
deadline - time(), instance.state))
if instance.state == 'terminated':
break
if time() > deadline:
log_instance_log(instance)
_L.warning('Stopping instance {} at deadline'.format(instance))
raise RuntimeError('Out of time')
sleep(60)
log_instance_log(instance)
except:
_L.error('Error in worker main()', exc_info=True)
if instance:
instance.terminate()
return 1
else:
return 0
def test_edit_launch_configuration():
add_launch_config("web", user_data="echo 'web_machine' > /etc/config")
conn = boto.connect_autoscale()
configs = conn.get_all_launch_configurations(names=['web'])
configs.should.have.length_of(1)
web_config = configs[0]
web_config.user_data.should.equal("echo 'web_machine' > /etc/config")
# Now edit the user data
edit_launch_config("web", user_data="echo 'other_machine' > /etc/config")
configs = conn.get_all_launch_configurations(names=['web'])
configs.should.have.length_of(1)
web_config = configs[0]
web_config.user_data.should.equal("echo 'other_machine' > /etc/config")
# And there should only be a single launch configuration
conn.get_all_launch_configurations().should.have.length_of(1)
def lambda_func(event, context):
''' Request a task instance inside AWS Lambda context.
'''
ec2, autoscale = boto.connect_ec2(), boto.connect_autoscale()
kwargs = dict(
instance_type = event.get('instance-type', 'm3.medium'),
lifespan = int(event.get('hours', 12)) * 3600,
command = event.get('command', ['sleep', '300']),
bucket = event.get('bucket', os.environ.get('AWS_S3_BUCKET')),
aws_sns_arn = event.get('sns-arn', os.environ.get('AWS_SNS_ARN')),
patch_version = event.get('version', get_version()),
tempsize = event.get('temp-size', None),
)
try:
print('Requesting with keyword args:', pprint.pformat(kwargs), sep='\n', file=sys.stderr)
return str(request_task_instance(ec2, autoscale, **kwargs))
except Exception as e:
print('Failed with exception:', e, sep='\n', file=sys.stderr)
return None
def get_all_autoscale_groups(names=None):
"""
Get all the autoscale groups
Arguments:
names (list) - A list of ASG names as strings
Returns:
List of :class:`boto.ec2.autoscale.group.AutoScalingGroup` instances.
"""
autoscale_conn = boto.connect_autoscale()
fetched_asgs = autoscale_conn.get_all_groups(names=names)
total_asgs = []
while True:
total_asgs.extend([asg for asg in fetched_asgs])
if fetched_asgs.next_token:
fetched_asgs = autoscale_conn.get_all_groups(
names=names, next_token=fetched_asgs.next_token)
else:
break
return total_asgs
def test_autoscaling_group_delete():
mocked_networking = setup_networking_deprecated()
conn = boto.connect_autoscale()
config = LaunchConfiguration(name="tester",
image_id="ami-abcd1234",
instance_type="t2.medium")
conn.create_launch_configuration(config)
group = AutoScalingGroup(
name="tester_group",
max_size=2,
min_size=2,
launch_config=config,
vpc_zone_identifier=mocked_networking["subnet1"],
)
conn.create_auto_scaling_group(group)
conn.get_all_groups().should.have.length_of(1)
conn.delete_auto_scaling_group("tester_group")
conn.get_all_groups().should.have.length_of(0)
def test_create_launch_configuration_defaults():
"""Test with the minimum inputs and check that all of the proper defaults
are assigned for the other attributes"""
conn = boto.connect_autoscale()
config = LaunchConfiguration(
name="tester", image_id="ami-abcd1234", instance_type="m1.small"
)
conn.create_launch_configuration(config)
launch_config = conn.get_all_launch_configurations()[0]
launch_config.name.should.equal("tester")
launch_config.image_id.should.equal("ami-abcd1234")
launch_config.instance_type.should.equal("m1.small")
# Defaults
launch_config.key_name.should.equal("")
list(launch_config.security_groups).should.equal([])
launch_config.user_data.should.equal(b"")
launch_config.instance_monitoring.enabled.should.equal("false")
launch_config.instance_profile_name.should.equal(None)
launch_config.spot_price.should.equal(None)
def test_autoscaling_group_delete():
conn = boto.connect_autoscale()
config = LaunchConfiguration(
name='tester',
image_id='ami-abcd1234',
instance_type='t2.medium',
)
conn.create_launch_configuration(config)
group = AutoScalingGroup(
name='tester_group',
max_size=2,
min_size=2,
launch_config=config,
)
conn.create_auto_scaling_group(group)
conn.get_all_groups().should.have.length_of(1)
conn.delete_auto_scaling_group('tester_group')
conn.get_all_groups().should.have.length_of(0)
