def run(self):
all_lcs = self.as_conn.get_all_launch_configurations()
lc_by_group = defaultdict(list)
lc_max_num_by_group = defaultdict(int)
for lc in all_lcs:
name, num = lc.name.split('-')
num = int(num)
lc_by_group[name].append(lc)
if num > lc_max_num_by_group[name]:
lc_max_num_by_group[name] = num
all_ags = self.as_conn.get_all_groups()
ag_by_name = {}
for ag in all_ags:
ag_by_name[ag.name] = ag
for group_name, config in self.as_config["groups"].iteritems():
print "Configuring %s" % group_name
use_lc = None
lc_to_delete = []
for lc in lc_by_group[group_name]:
if use_lc is None and \
lc.image_id == config['ami'] and \
lc.key_name == config['ssh_key'] and \
lc.instance_type == config['instance_type'] and \
lc.security_groups == [config['security_group']] and \
lc.user_data == self.user_data:
print " Found LaunchConfig %s that matches profile" % \
lc.name
use_lc = lc
else:
lc_to_delete.append(lc)
print " Found %d LaunchConfigurations to delete" % len(
lc_to_delete)
if not use_lc:
print " Making LaunchConfiguration for %s" % group_name
lc_num = lc_max_num_by_group[group_name] + 1
use_lc = LaunchConfiguration(
name="%s-%d" % (group_name, lc_num),
image_id=config['ami'],
key_name=config['ssh_key'],
instance_type=config['instance_type'],
security_groups=[config['security_group']],
user_data=self.user_data)
self.as_conn.create_launch_configuration(use_lc)
if group_name in ag_by_name:
print " Found existing AutoScalingGroup, updating"
ag = ag_by_name[group_name]
ag_exists = True
else:
print " Making new AutoScalingGroup"
ag = AutoScalingGroup()
ag_exists = False
# config ASG as we want it
ag.name = group_name
ag.launch_config_name = use_lc.name
ag.availability_zones = config['zones']
ag.desired_capacity = config['capacity']
ag.min_size = config['min_size']
ag.max_size = config['max_size']
# create or update as appropriate
if ag_exists:
ag.update()
else:
self.as_conn.create_auto_scaling_group(ag)
# make it send e-mail whenever it does something
if 'notification_topic' in self.as_config:
# NOTE [adam Sept/18/12]: this is a hack designed to work
# around that boto support for this isn't in a release yet.
# when the next release is out, we should uncomment the
# code below.
params = {
'AutoScalingGroupName': ag.name,
'TopicARN': self.as_config['notification_topic']
}
self.as_conn.build_list_params(params, self.AS_NOTIFICATIONS,
'NotificationTypes')
self.as_conn.get_status('PutNotificationConfiguration', params)
#as_conn.put_notification_configuration(
# ag.name,
# self.as_config['notification_topic'],
# self.AS_NOTIFICATIONS)
tags = []
for tag_name, tag_value in config.get('tags', {}).iteritems():
print " Adding tag %s = %s" % (tag_name, tag_value)
tags.append(
Tag(key=tag_name,
value=tag_value,
propagate_at_launch=True,
resource_id=ag.name))
self.as_conn.create_or_update_tags(tags)
for lc in lc_to_delete:
print " Deleting old LaunchConfiguration %s" % lc.name
lc.delete()
for alarm_name, alarm_cfg in config.get('alarms', {}).iteritems():
alarm_policy_arn = self.make_policy(group_name,
alarm_cfg['policy'])
alarm_name = '%s|%s|%s' % (group_name, alarm_cfg['policy'],
alarm_cfg['metric'])
alarm = MetricAlarm(
name=alarm_name,
namespace=alarm_cfg['namespace'],
metric=alarm_cfg['metric'],
statistic='Average',
dimensions={'AutoScalingGroupName': group_name},
comparison=alarm_cfg['comparison'],
threshold=alarm_cfg['threshold'],
period=alarm_cfg['period'],
evaluation_periods=alarm_cfg.get('evaluation_periods', 1),
alarm_actions=[alarm_policy_arn])
self.cw_conn.put_metric_alarm(alarm)
def deploy_latest_app_ami(app_name, env_name):
global aws
aws = AWS.from_config(config)
aws.connect()
lb_group_ids=[aws.get_security_group_id('riker-load-balancer')]
inst_group_ids=[aws.get_security_group_id('riker-instance')]
app = App(env_name, app_name)
health_check_target = app.config.get('health_check', 'TCP:80')
name = re.sub('[^A-Za-z0-9\-]', '-', app.name)
app_image = LatestAppImage(app).get()
print '-----> Connecting to ELB'
elb_conn = boto.connect_elb()
log('info', 'Load balancer', show_header=True)
load_balancer_name = name
try:
elb_result = elb_conn.get_all_load_balancers(load_balancer_names=[load_balancer_name])
lb = elb_result[0]
log('info', 'Found {}'.format(load_balancer_name))
except boto.exception.BotoServerError:
log('info', 'Not found, creating load balancer')
listeners = [(80, 80, 'HTTP', 'HTTP')]
lb = elb_conn.create_load_balancer(name=load_balancer_name,
zones=None,
complex_listeners=listeners,
security_groups=lb_group_ids,
subnets=[aws.subnet_id])
hc = HealthCheck(target=health_check_target)
lb.configure_health_check(hc)
cda = ConnectionDrainingAttribute()
cda.enabled = True
cda.timeout = 300
elb_conn.modify_lb_attribute(load_balancer_name=load_balancer_name,
attribute='connectionDraining',
value=cda)
print '-----> Connecting to AutoScale'
as_conn = boto.connect_autoscale()
log('info', 'Launch configuration', show_header=True)
launch_config_name = "{}-{}".format(name, app_image.tags['deploy-id'])
lc_result = as_conn.get_all_launch_configurations(names=[launch_config_name])
if len(lc_result) == 0:
log('info', 'Not found, creating LaunchConfiguration')
lc = LaunchConfiguration(name=launch_config_name,
image_id=app_image.id,
key_name=aws.key_pair_name,
security_groups=inst_group_ids,
instance_type=aws.instance_type)
as_conn.create_launch_configuration(lc)
else:
log('info', 'Found {}'.format(launch_config_name))
lc = lc_result[0]
existing_group = None
deploy_id = int(app_image.tags['deploy-id'] or 0)
log('info', 'Getting previous auto-scaling group', show_header=True)
for did in xrange(deploy_id-1, 0, -1):
existing_group_name = "{}-{}".format(name, did)
log('info', '{} ?'.format(existing_group_name))
ag_result = as_conn.get_all_groups(names=[existing_group_name])
if len(ag_result) > 0:
existing_group = ag_result[0]
log('info', 'Found {}'.format(existing_group.name))
break
else:
log('info', 'No')
if existing_group is not None:
existing_healthy_instances = [inst for inst in existing_group.instances if inst.lifecycle_state == 'InService' and inst.health_status == 'Healthy']
existing_healthy_instance_count = len(existing_healthy_instances)
desired_capacity = existing_group.desired_capacity
min_size = existing_group.min_size
max_size = existing_group.max_size
if existing_healthy_instance_count == 0 and desired_capacity == 0:
print '-----> WARNING: existing auto-scaling group {} has no healthy instances and a desired capacity of 0. New auto-scaling group will launch 1 instance.'.format(existing_group)
desired_capacity = 1
min_size = 1
max_size = max_size if max_size > 0 else 1
else:
existing_healthy_instance_count = 0
desired_capacity = 1
min_size = 1
max_size = 1
log('info', '{} existing instance(s) found'.format(existing_healthy_instance_count), show_header=True)
log('info', 'Existing auto-scale properties: desired_capacity={}, min_size={}, max_size={}'.format(desired_capacity, min_size, max_size))
log('info', 'Auto-scaling group', show_header=True)
group_name = "{}-{}".format(name, app_image.tags['deploy-id'])
ag_result = as_conn.get_all_groups(names=[group_name])
if len(ag_result) == 0:
log('info', 'Not found, creating autoscale group')
ag = AutoScalingGroup(name=group_name,
load_balancers=[load_balancer_name], launch_config=lc,
desired_capacity=desired_capacity, min_size=min_size, max_size=max_size,
health_check_type='ELB', health_check_period='300',
vpc_zone_identifier=aws.subnet_id)
as_conn.create_auto_scaling_group(ag)
else:
log('info', 'Found {}'.format(group_name))
ag = ag_result[0]
ag.desired_capacity = desired_capacity
ag.max_size = max_size
ag.min_size = min_size
ag.launch_config_name = launch_config_name
ag.update()
log('info', 'Waiting for new instances to become healthy', show_header=True)
all_healthy = False
for i in xrange(60):
if i > 0:
print ' ---'
time.sleep(10)
elb_result = elb_conn.get_all_load_balancers(load_balancer_names=[load_balancer_name])
lb = elb_result[0]
lb_insts = lb.get_instance_health()
print ' Load-balancer instances: {}'.format(lb_insts)
# NOTE: re-get auto-scaling group to get updated instance info.
ag = as_conn.get_all_groups(names=[group_name])[0]
ag_insts = [inst for inst in ag.instances]
log('info', 'Auto-scaling group Instances: {}'.format(ag_insts))
if len(ag_insts) < desired_capacity:
not_yet_launched_count = desired_capacity - len(ag_insts)
log('info', '{} new instance(s) not yet launched'.format(not_yet_launched_count))
continue
ag_inst_ids = set(inst.instance_id for inst in ag_insts)
lb_inst_ids = set(inst.instance_id for inst in lb_insts)
asg_insts_not_in_lb = ag_inst_ids.difference(lb_inst_ids)
if len(asg_insts_not_in_lb) > 0:
log('info', '{} new instance(s) not yet in load balancer'.format(len(asg_insts_not_in_lb)))
continue
new_lb_insts = [inst for inst in lb_insts if inst.instance_id in ag_inst_ids]
healthy_new_lb_insts = [inst for inst in new_lb_insts if inst.state == 'InService']
all_healthy = len(healthy_new_lb_insts) == len(ag_insts)
log('info', '{} new instance(s) are healthy'.format(len(healthy_new_lb_insts)))
diff = existing_healthy_instance_count - len(healthy_new_lb_insts)
if existing_group is not None and diff >= 0:
change = False
if existing_group.desired_capacity != diff:
existing_group.desired_capacity = diff
change = True
if existing_group.max_size != diff:
existing_group.max_size = diff
change = True
if diff < existing_group.min_size:
existing_group.min_size = diff
change = True
if change:
existing_group.update()
log('info', 'Change previous auto-scale group {} properties: desired_capacity={}, min_size={}, max_size={}'.format(existing_group, existing_group.desired_capacity, existing_group.min_size, existing_group.max_size))
if all_healthy:
log('info', 'All new instances healthy!', show_header=True)
healthy_lb_inst_ids = [inst.instance_id for inst in lb_insts if inst.state == 'InService']
previous_healthy_inst_ids = [inst.instance_id for inst in existing_healthy_instances] if existing_group else []
not_yet_out_of_service = set(previous_healthy_inst_ids).intersection(healthy_lb_inst_ids)
if len(not_yet_out_of_service) > 0:
log('info', 'Waiting to remove previous instances ({}) from load balancer'.format(not_yet_out_of_service))
else:
log('info', 'All previous instances ({}) have been removed from load balancer'.format(previous_healthy_inst_ids), show_header=True)
if all_healthy and len(not_yet_out_of_service) == 0:
break
else:
raise Exception("Timeout")
elb_result = elb_conn.get_all_load_balancers(load_balancer_names=[load_balancer_name])
lb = elb_result[0]
lb_insts = [inst for inst in lb.get_instance_health() if inst.state == 'InService']
print '-----> Deployed {} instance(s) of {} to {}'.format(lb_insts, app.name, lb.dns_name)
print '-----> DONE!'
def run(self):
all_lcs = self.as_conn.get_all_launch_configurations()
lc_by_group = defaultdict(list)
lc_max_num_by_group = defaultdict(int)
for lc in all_lcs:
name, num = lc.name.split('-')
num = int(num)
lc_by_group[name].append(lc)
if num > lc_max_num_by_group[name]:
lc_max_num_by_group[name] = num
all_ags = self.as_conn.get_all_groups()
ag_by_name = {}
for ag in all_ags:
ag_by_name[ag.name] = ag
for group_name, config in self.as_config["groups"].iteritems():
print "Configuring %s" % group_name
use_lc = None
lc_to_delete = []
for lc in lc_by_group[group_name]:
if use_lc is None and \
lc.image_id == config['ami'] and \
lc.key_name == config['ssh_key'] and \
lc.instance_type == config['instance_type'] and \
lc.security_groups == [config['security_group']] and \
lc.user_data == self.user_data:
print " Found LaunchConfig %s that matches profile" % \
lc.name
use_lc = lc
else:
lc_to_delete.append(lc)
print " Found %d LaunchConfigurations to delete" % len(lc_to_delete)
if not use_lc:
print " Making LaunchConfiguration for %s" % group_name
lc_num = lc_max_num_by_group[group_name] + 1
use_lc = LaunchConfiguration(
name="%s-%d" % (group_name, lc_num),
image_id=config['ami'],
key_name=config['ssh_key'],
instance_type=config['instance_type'],
security_groups=[config['security_group']],
user_data=self.user_data)
self.as_conn.create_launch_configuration(use_lc)
if group_name in ag_by_name:
print " Found existing AutoScalingGroup, updating"
ag = ag_by_name[group_name]
ag_exists = True
else:
print " Making new AutoScalingGroup"
ag = AutoScalingGroup()
ag_exists = False
# config ASG as we want it
ag.name = group_name
ag.launch_config_name = use_lc.name
ag.availability_zones = config['zones']
ag.desired_capacity = config['capacity']
ag.min_size = config['min_size']
ag.max_size = config['max_size']
# create or update as appropriate
if ag_exists:
ag.update()
else:
self.as_conn.create_auto_scaling_group(ag)
# make it send e-mail whenever it does something
if 'notification_topic' in self.as_config:
# NOTE [adam Sept/18/12]: this is a hack designed to work
# around that boto support for this isn't in a release yet.
# when the next release is out, we should uncomment the
# code below.
params = {'AutoScalingGroupName': ag.name,
'TopicARN': self.as_config['notification_topic'] }
self.as_conn.build_list_params(params, self.AS_NOTIFICATIONS,
'NotificationTypes')
self.as_conn.get_status('PutNotificationConfiguration', params)
#as_conn.put_notification_configuration(
# ag.name,
# self.as_config['notification_topic'],
# self.AS_NOTIFICATIONS)
tags = []
for tag_name, tag_value in config.get('tags', {}).iteritems():
print " Adding tag %s = %s" % (tag_name, tag_value)
tags.append(Tag(key=tag_name, value=tag_value,
propagate_at_launch=True,
resource_id=ag.name))
self.as_conn.create_or_update_tags(tags)
for lc in lc_to_delete:
print " Deleting old LaunchConfiguration %s" % lc.name
lc.delete()
for alarm_name, alarm_cfg in config.get('alarms', {}).iteritems():
alarm_policy_arn = self.make_policy(group_name,
alarm_cfg['policy'])
alarm_name = '%s|%s|%s' % (group_name,
alarm_cfg['policy'],
alarm_cfg['metric'])
alarm = MetricAlarm(
name=alarm_name,
namespace=alarm_cfg['namespace'],
metric=alarm_cfg['metric'],
statistic='Average',
dimensions={'AutoScalingGroupName': group_name},
comparison=alarm_cfg['comparison'],
threshold=alarm_cfg['threshold'],
period=alarm_cfg['period'],
evaluation_periods=alarm_cfg.get('evaluation_periods', 1),
alarm_actions=[alarm_policy_arn])
self.cw_conn.put_metric_alarm(alarm)
def deploy_latest_app_ami(app_name, env_name):
global aws
aws = AWS.from_config(config)
aws.connect()
lb_group_ids = [aws.get_security_group_id('riker-load-balancer')]
inst_group_ids = [aws.get_security_group_id('riker-instance')]
app = App(env_name, app_name)
health_check_target = app.config.get('health_check', 'TCP:80')
name = re.sub('[^A-Za-z0-9\-]', '-', app.name)
app_image = LatestAppImage(app).get()
print '-----> Connecting to ELB'
elb_conn = boto.connect_elb()
log('info', 'Load balancer', show_header=True)
load_balancer_name = name
try:
elb_result = elb_conn.get_all_load_balancers(
load_balancer_names=[load_balancer_name])
lb = elb_result[0]
log('info', 'Found {}'.format(load_balancer_name))
except boto.exception.BotoServerError:
log('info', 'Not found, creating load balancer')
listeners = [(80, 80, 'HTTP', 'HTTP')]
lb = elb_conn.create_load_balancer(name=load_balancer_name,
zones=None,
complex_listeners=listeners,
security_groups=lb_group_ids,
subnets=[aws.subnet_id])
hc = HealthCheck(target=health_check_target)
lb.configure_health_check(hc)
cda = ConnectionDrainingAttribute()
cda.enabled = True
cda.timeout = 300
elb_conn.modify_lb_attribute(load_balancer_name=load_balancer_name,
attribute='connectionDraining',
value=cda)
print '-----> Connecting to AutoScale'
as_conn = boto.connect_autoscale()
log('info', 'Launch configuration', show_header=True)
launch_config_name = "{}-{}".format(name, app_image.tags['deploy-id'])
lc_result = as_conn.get_all_launch_configurations(
names=[launch_config_name])
if len(lc_result) == 0:
log('info', 'Not found, creating LaunchConfiguration')
lc = LaunchConfiguration(name=launch_config_name,
image_id=app_image.id,
key_name=aws.key_pair_name,
security_groups=inst_group_ids,
instance_type=aws.instance_type)
as_conn.create_launch_configuration(lc)
else:
log('info', 'Found {}'.format(launch_config_name))
lc = lc_result[0]
existing_group = None
deploy_id = int(app_image.tags['deploy-id'] or 0)
log('info', 'Getting previous auto-scaling group', show_header=True)
for did in xrange(deploy_id - 1, 0, -1):
existing_group_name = "{}-{}".format(name, did)
log('info', '{} ?'.format(existing_group_name))
ag_result = as_conn.get_all_groups(names=[existing_group_name])
if len(ag_result) > 0:
existing_group = ag_result[0]
log('info', 'Found {}'.format(existing_group.name))
break
else:
log('info', 'No')
if existing_group is not None:
existing_healthy_instances = [
inst for inst in existing_group.instances
if inst.lifecycle_state == 'InService'
and inst.health_status == 'Healthy'
]
existing_healthy_instance_count = len(existing_healthy_instances)
desired_capacity = existing_group.desired_capacity
min_size = existing_group.min_size
max_size = existing_group.max_size
if existing_healthy_instance_count == 0 and desired_capacity == 0:
print '-----> WARNING: existing auto-scaling group {} has no healthy instances and a desired capacity of 0. New auto-scaling group will launch 1 instance.'.format(
existing_group)
desired_capacity = 1
min_size = 1
max_size = max_size if max_size > 0 else 1
else:
existing_healthy_instance_count = 0
desired_capacity = 1
min_size = 1
max_size = 1
log('info',
'{} existing instance(s) found'.format(
existing_healthy_instance_count),
show_header=True)
log(
'info',
'Existing auto-scale properties: desired_capacity={}, min_size={}, max_size={}'
.format(desired_capacity, min_size, max_size))
log('info', 'Auto-scaling group', show_header=True)
group_name = "{}-{}".format(name, app_image.tags['deploy-id'])
ag_result = as_conn.get_all_groups(names=[group_name])
if len(ag_result) == 0:
log('info', 'Not found, creating autoscale group')
ag = AutoScalingGroup(name=group_name,
load_balancers=[load_balancer_name],
launch_config=lc,
desired_capacity=desired_capacity,
min_size=min_size,
max_size=max_size,
health_check_type='ELB',
health_check_period='300',
vpc_zone_identifier=aws.subnet_id)
as_conn.create_auto_scaling_group(ag)
else:
log('info', 'Found {}'.format(group_name))
ag = ag_result[0]
ag.desired_capacity = desired_capacity
ag.max_size = max_size
ag.min_size = min_size
ag.launch_config_name = launch_config_name
ag.update()
log('info',
'Waiting for new instances to become healthy',
show_header=True)
all_healthy = False
for i in xrange(60):
if i > 0:
print ' ---'
time.sleep(10)
elb_result = elb_conn.get_all_load_balancers(
load_balancer_names=[load_balancer_name])
lb = elb_result[0]
lb_insts = lb.get_instance_health()
print ' Load-balancer instances: {}'.format(lb_insts)
# NOTE: re-get auto-scaling group to get updated instance info.
ag = as_conn.get_all_groups(names=[group_name])[0]
ag_insts = [inst for inst in ag.instances]
log('info', 'Auto-scaling group Instances: {}'.format(ag_insts))
if len(ag_insts) < desired_capacity:
not_yet_launched_count = desired_capacity - len(ag_insts)
log(
'info', '{} new instance(s) not yet launched'.format(
not_yet_launched_count))
continue
ag_inst_ids = set(inst.instance_id for inst in ag_insts)
lb_inst_ids = set(inst.instance_id for inst in lb_insts)
asg_insts_not_in_lb = ag_inst_ids.difference(lb_inst_ids)
if len(asg_insts_not_in_lb) > 0:
log(
'info', '{} new instance(s) not yet in load balancer'.format(
len(asg_insts_not_in_lb)))
continue
new_lb_insts = [
inst for inst in lb_insts if inst.instance_id in ag_inst_ids
]
healthy_new_lb_insts = [
inst for inst in new_lb_insts if inst.state == 'InService'
]
all_healthy = len(healthy_new_lb_insts) == len(ag_insts)
log('info',
'{} new instance(s) are healthy'.format(len(healthy_new_lb_insts)))
diff = existing_healthy_instance_count - len(healthy_new_lb_insts)
if existing_group is not None and diff >= 0:
change = False
if existing_group.desired_capacity != diff:
existing_group.desired_capacity = diff
change = True
if existing_group.max_size != diff:
existing_group.max_size = diff
change = True
if diff < existing_group.min_size:
existing_group.min_size = diff
change = True
if change:
existing_group.update()
log(
'info',
'Change previous auto-scale group {} properties: desired_capacity={}, min_size={}, max_size={}'
.format(existing_group, existing_group.desired_capacity,
existing_group.min_size, existing_group.max_size))
if all_healthy:
log('info', 'All new instances healthy!', show_header=True)
healthy_lb_inst_ids = [
inst.instance_id for inst in lb_insts
if inst.state == 'InService'
]
previous_healthy_inst_ids = [
inst.instance_id for inst in existing_healthy_instances
] if existing_group else []
not_yet_out_of_service = set(
previous_healthy_inst_ids).intersection(healthy_lb_inst_ids)
if len(not_yet_out_of_service) > 0:
log(
'info',
'Waiting to remove previous instances ({}) from load balancer'
.format(not_yet_out_of_service))
else:
log('info',
'All previous instances ({}) have been removed from load balancer'
.format(previous_healthy_inst_ids),
show_header=True)
if all_healthy and len(not_yet_out_of_service) == 0:
break
else:
raise Exception("Timeout")
elb_result = elb_conn.get_all_load_balancers(
load_balancer_names=[load_balancer_name])
lb = elb_result[0]
lb_insts = [
inst for inst in lb.get_instance_health() if inst.state == 'InService'
]
print '-----> Deployed {} instance(s) of {} to {}'.format(
lb_insts, app.name, lb.dns_name)
print '-----> DONE!'