def run(self): all_lcs = self.as_conn.get_all_launch_configurations() lc_by_group = defaultdict(list) lc_max_num_by_group = defaultdict(int) for lc in all_lcs: name, num = lc.name.split('-') num = int(num) lc_by_group[name].append(lc) if num > lc_max_num_by_group[name]: lc_max_num_by_group[name] = num all_ags = self.as_conn.get_all_groups() ag_by_name = {} for ag in all_ags: ag_by_name[ag.name] = ag for group_name, config in self.as_config["groups"].iteritems(): print "Configuring %s" % group_name use_lc = None lc_to_delete = [] for lc in lc_by_group[group_name]: if use_lc is None and \ lc.image_id == config['ami'] and \ lc.key_name == config['ssh_key'] and \ lc.instance_type == config['instance_type'] and \ lc.security_groups == [config['security_group']] and \ lc.user_data == self.user_data: print " Found LaunchConfig %s that matches profile" % \ lc.name use_lc = lc else: lc_to_delete.append(lc) print " Found %d LaunchConfigurations to delete" % len( lc_to_delete) if not use_lc: print " Making LaunchConfiguration for %s" % group_name lc_num = lc_max_num_by_group[group_name] + 1 use_lc = LaunchConfiguration( name="%s-%d" % (group_name, lc_num), image_id=config['ami'], key_name=config['ssh_key'], instance_type=config['instance_type'], security_groups=[config['security_group']], user_data=self.user_data) self.as_conn.create_launch_configuration(use_lc) if group_name in ag_by_name: print " Found existing AutoScalingGroup, updating" ag = ag_by_name[group_name] ag_exists = True else: print " Making new AutoScalingGroup" ag = AutoScalingGroup() ag_exists = False # config ASG as we want it ag.name = group_name ag.launch_config_name = use_lc.name ag.availability_zones = config['zones'] ag.desired_capacity = config['capacity'] ag.min_size = config['min_size'] ag.max_size = config['max_size'] # create or update as appropriate if ag_exists: ag.update() else: self.as_conn.create_auto_scaling_group(ag) # make it send e-mail whenever it does something if 'notification_topic' in self.as_config: # NOTE [adam Sept/18/12]: this is a hack designed to work # around that boto support for this isn't in a release yet. # when the next release is out, we should uncomment the # code below. params = { 'AutoScalingGroupName': ag.name, 'TopicARN': self.as_config['notification_topic'] } self.as_conn.build_list_params(params, self.AS_NOTIFICATIONS, 'NotificationTypes') self.as_conn.get_status('PutNotificationConfiguration', params) #as_conn.put_notification_configuration( # ag.name, # self.as_config['notification_topic'], # self.AS_NOTIFICATIONS) tags = [] for tag_name, tag_value in config.get('tags', {}).iteritems(): print " Adding tag %s = %s" % (tag_name, tag_value) tags.append( Tag(key=tag_name, value=tag_value, propagate_at_launch=True, resource_id=ag.name)) self.as_conn.create_or_update_tags(tags) for lc in lc_to_delete: print " Deleting old LaunchConfiguration %s" % lc.name lc.delete() for alarm_name, alarm_cfg in config.get('alarms', {}).iteritems(): alarm_policy_arn = self.make_policy(group_name, alarm_cfg['policy']) alarm_name = '%s|%s|%s' % (group_name, alarm_cfg['policy'], alarm_cfg['metric']) alarm = MetricAlarm( name=alarm_name, namespace=alarm_cfg['namespace'], metric=alarm_cfg['metric'], statistic='Average', dimensions={'AutoScalingGroupName': group_name}, comparison=alarm_cfg['comparison'], threshold=alarm_cfg['threshold'], period=alarm_cfg['period'], evaluation_periods=alarm_cfg.get('evaluation_periods', 1), alarm_actions=[alarm_policy_arn]) self.cw_conn.put_metric_alarm(alarm)
def deploy_latest_app_ami(app_name, env_name): global aws aws = AWS.from_config(config) aws.connect() lb_group_ids=[aws.get_security_group_id('riker-load-balancer')] inst_group_ids=[aws.get_security_group_id('riker-instance')] app = App(env_name, app_name) health_check_target = app.config.get('health_check', 'TCP:80') name = re.sub('[^A-Za-z0-9\-]', '-', app.name) app_image = LatestAppImage(app).get() print '-----> Connecting to ELB' elb_conn = boto.connect_elb() log('info', 'Load balancer', show_header=True) load_balancer_name = name try: elb_result = elb_conn.get_all_load_balancers(load_balancer_names=[load_balancer_name]) lb = elb_result[0] log('info', 'Found {}'.format(load_balancer_name)) except boto.exception.BotoServerError: log('info', 'Not found, creating load balancer') listeners = [(80, 80, 'HTTP', 'HTTP')] lb = elb_conn.create_load_balancer(name=load_balancer_name, zones=None, complex_listeners=listeners, security_groups=lb_group_ids, subnets=[aws.subnet_id]) hc = HealthCheck(target=health_check_target) lb.configure_health_check(hc) cda = ConnectionDrainingAttribute() cda.enabled = True cda.timeout = 300 elb_conn.modify_lb_attribute(load_balancer_name=load_balancer_name, attribute='connectionDraining', value=cda) print '-----> Connecting to AutoScale' as_conn = boto.connect_autoscale() log('info', 'Launch configuration', show_header=True) launch_config_name = "{}-{}".format(name, app_image.tags['deploy-id']) lc_result = as_conn.get_all_launch_configurations(names=[launch_config_name]) if len(lc_result) == 0: log('info', 'Not found, creating LaunchConfiguration') lc = LaunchConfiguration(name=launch_config_name, image_id=app_image.id, key_name=aws.key_pair_name, security_groups=inst_group_ids, instance_type=aws.instance_type) as_conn.create_launch_configuration(lc) else: log('info', 'Found {}'.format(launch_config_name)) lc = lc_result[0] existing_group = None deploy_id = int(app_image.tags['deploy-id'] or 0) log('info', 'Getting previous auto-scaling group', show_header=True) for did in xrange(deploy_id-1, 0, -1): existing_group_name = "{}-{}".format(name, did) log('info', '{} ?'.format(existing_group_name)) ag_result = as_conn.get_all_groups(names=[existing_group_name]) if len(ag_result) > 0: existing_group = ag_result[0] log('info', 'Found {}'.format(existing_group.name)) break else: log('info', 'No') if existing_group is not None: existing_healthy_instances = [inst for inst in existing_group.instances if inst.lifecycle_state == 'InService' and inst.health_status == 'Healthy'] existing_healthy_instance_count = len(existing_healthy_instances) desired_capacity = existing_group.desired_capacity min_size = existing_group.min_size max_size = existing_group.max_size if existing_healthy_instance_count == 0 and desired_capacity == 0: print '-----> WARNING: existing auto-scaling group {} has no healthy instances and a desired capacity of 0. New auto-scaling group will launch 1 instance.'.format(existing_group) desired_capacity = 1 min_size = 1 max_size = max_size if max_size > 0 else 1 else: existing_healthy_instance_count = 0 desired_capacity = 1 min_size = 1 max_size = 1 log('info', '{} existing instance(s) found'.format(existing_healthy_instance_count), show_header=True) log('info', 'Existing auto-scale properties: desired_capacity={}, min_size={}, max_size={}'.format(desired_capacity, min_size, max_size)) log('info', 'Auto-scaling group', show_header=True) group_name = "{}-{}".format(name, app_image.tags['deploy-id']) ag_result = as_conn.get_all_groups(names=[group_name]) if len(ag_result) == 0: log('info', 'Not found, creating autoscale group') ag = AutoScalingGroup(name=group_name, load_balancers=[load_balancer_name], launch_config=lc, desired_capacity=desired_capacity, min_size=min_size, max_size=max_size, health_check_type='ELB', health_check_period='300', vpc_zone_identifier=aws.subnet_id) as_conn.create_auto_scaling_group(ag) else: log('info', 'Found {}'.format(group_name)) ag = ag_result[0] ag.desired_capacity = desired_capacity ag.max_size = max_size ag.min_size = min_size ag.launch_config_name = launch_config_name ag.update() log('info', 'Waiting for new instances to become healthy', show_header=True) all_healthy = False for i in xrange(60): if i > 0: print ' ---' time.sleep(10) elb_result = elb_conn.get_all_load_balancers(load_balancer_names=[load_balancer_name]) lb = elb_result[0] lb_insts = lb.get_instance_health() print ' Load-balancer instances: {}'.format(lb_insts) # NOTE: re-get auto-scaling group to get updated instance info. ag = as_conn.get_all_groups(names=[group_name])[0] ag_insts = [inst for inst in ag.instances] log('info', 'Auto-scaling group Instances: {}'.format(ag_insts)) if len(ag_insts) < desired_capacity: not_yet_launched_count = desired_capacity - len(ag_insts) log('info', '{} new instance(s) not yet launched'.format(not_yet_launched_count)) continue ag_inst_ids = set(inst.instance_id for inst in ag_insts) lb_inst_ids = set(inst.instance_id for inst in lb_insts) asg_insts_not_in_lb = ag_inst_ids.difference(lb_inst_ids) if len(asg_insts_not_in_lb) > 0: log('info', '{} new instance(s) not yet in load balancer'.format(len(asg_insts_not_in_lb))) continue new_lb_insts = [inst for inst in lb_insts if inst.instance_id in ag_inst_ids] healthy_new_lb_insts = [inst for inst in new_lb_insts if inst.state == 'InService'] all_healthy = len(healthy_new_lb_insts) == len(ag_insts) log('info', '{} new instance(s) are healthy'.format(len(healthy_new_lb_insts))) diff = existing_healthy_instance_count - len(healthy_new_lb_insts) if existing_group is not None and diff >= 0: change = False if existing_group.desired_capacity != diff: existing_group.desired_capacity = diff change = True if existing_group.max_size != diff: existing_group.max_size = diff change = True if diff < existing_group.min_size: existing_group.min_size = diff change = True if change: existing_group.update() log('info', 'Change previous auto-scale group {} properties: desired_capacity={}, min_size={}, max_size={}'.format(existing_group, existing_group.desired_capacity, existing_group.min_size, existing_group.max_size)) if all_healthy: log('info', 'All new instances healthy!', show_header=True) healthy_lb_inst_ids = [inst.instance_id for inst in lb_insts if inst.state == 'InService'] previous_healthy_inst_ids = [inst.instance_id for inst in existing_healthy_instances] if existing_group else [] not_yet_out_of_service = set(previous_healthy_inst_ids).intersection(healthy_lb_inst_ids) if len(not_yet_out_of_service) > 0: log('info', 'Waiting to remove previous instances ({}) from load balancer'.format(not_yet_out_of_service)) else: log('info', 'All previous instances ({}) have been removed from load balancer'.format(previous_healthy_inst_ids), show_header=True) if all_healthy and len(not_yet_out_of_service) == 0: break else: raise Exception("Timeout") elb_result = elb_conn.get_all_load_balancers(load_balancer_names=[load_balancer_name]) lb = elb_result[0] lb_insts = [inst for inst in lb.get_instance_health() if inst.state == 'InService'] print '-----> Deployed {} instance(s) of {} to {}'.format(lb_insts, app.name, lb.dns_name) print '-----> DONE!'
def run(self): all_lcs = self.as_conn.get_all_launch_configurations() lc_by_group = defaultdict(list) lc_max_num_by_group = defaultdict(int) for lc in all_lcs: name, num = lc.name.split('-') num = int(num) lc_by_group[name].append(lc) if num > lc_max_num_by_group[name]: lc_max_num_by_group[name] = num all_ags = self.as_conn.get_all_groups() ag_by_name = {} for ag in all_ags: ag_by_name[ag.name] = ag for group_name, config in self.as_config["groups"].iteritems(): print "Configuring %s" % group_name use_lc = None lc_to_delete = [] for lc in lc_by_group[group_name]: if use_lc is None and \ lc.image_id == config['ami'] and \ lc.key_name == config['ssh_key'] and \ lc.instance_type == config['instance_type'] and \ lc.security_groups == [config['security_group']] and \ lc.user_data == self.user_data: print " Found LaunchConfig %s that matches profile" % \ lc.name use_lc = lc else: lc_to_delete.append(lc) print " Found %d LaunchConfigurations to delete" % len(lc_to_delete) if not use_lc: print " Making LaunchConfiguration for %s" % group_name lc_num = lc_max_num_by_group[group_name] + 1 use_lc = LaunchConfiguration( name="%s-%d" % (group_name, lc_num), image_id=config['ami'], key_name=config['ssh_key'], instance_type=config['instance_type'], security_groups=[config['security_group']], user_data=self.user_data) self.as_conn.create_launch_configuration(use_lc) if group_name in ag_by_name: print " Found existing AutoScalingGroup, updating" ag = ag_by_name[group_name] ag_exists = True else: print " Making new AutoScalingGroup" ag = AutoScalingGroup() ag_exists = False # config ASG as we want it ag.name = group_name ag.launch_config_name = use_lc.name ag.availability_zones = config['zones'] ag.desired_capacity = config['capacity'] ag.min_size = config['min_size'] ag.max_size = config['max_size'] # create or update as appropriate if ag_exists: ag.update() else: self.as_conn.create_auto_scaling_group(ag) # make it send e-mail whenever it does something if 'notification_topic' in self.as_config: # NOTE [adam Sept/18/12]: this is a hack designed to work # around that boto support for this isn't in a release yet. # when the next release is out, we should uncomment the # code below. params = {'AutoScalingGroupName': ag.name, 'TopicARN': self.as_config['notification_topic'] } self.as_conn.build_list_params(params, self.AS_NOTIFICATIONS, 'NotificationTypes') self.as_conn.get_status('PutNotificationConfiguration', params) #as_conn.put_notification_configuration( # ag.name, # self.as_config['notification_topic'], # self.AS_NOTIFICATIONS) tags = [] for tag_name, tag_value in config.get('tags', {}).iteritems(): print " Adding tag %s = %s" % (tag_name, tag_value) tags.append(Tag(key=tag_name, value=tag_value, propagate_at_launch=True, resource_id=ag.name)) self.as_conn.create_or_update_tags(tags) for lc in lc_to_delete: print " Deleting old LaunchConfiguration %s" % lc.name lc.delete() for alarm_name, alarm_cfg in config.get('alarms', {}).iteritems(): alarm_policy_arn = self.make_policy(group_name, alarm_cfg['policy']) alarm_name = '%s|%s|%s' % (group_name, alarm_cfg['policy'], alarm_cfg['metric']) alarm = MetricAlarm( name=alarm_name, namespace=alarm_cfg['namespace'], metric=alarm_cfg['metric'], statistic='Average', dimensions={'AutoScalingGroupName': group_name}, comparison=alarm_cfg['comparison'], threshold=alarm_cfg['threshold'], period=alarm_cfg['period'], evaluation_periods=alarm_cfg.get('evaluation_periods', 1), alarm_actions=[alarm_policy_arn]) self.cw_conn.put_metric_alarm(alarm)
def deploy_latest_app_ami(app_name, env_name): global aws aws = AWS.from_config(config) aws.connect() lb_group_ids = [aws.get_security_group_id('riker-load-balancer')] inst_group_ids = [aws.get_security_group_id('riker-instance')] app = App(env_name, app_name) health_check_target = app.config.get('health_check', 'TCP:80') name = re.sub('[^A-Za-z0-9\-]', '-', app.name) app_image = LatestAppImage(app).get() print '-----> Connecting to ELB' elb_conn = boto.connect_elb() log('info', 'Load balancer', show_header=True) load_balancer_name = name try: elb_result = elb_conn.get_all_load_balancers( load_balancer_names=[load_balancer_name]) lb = elb_result[0] log('info', 'Found {}'.format(load_balancer_name)) except boto.exception.BotoServerError: log('info', 'Not found, creating load balancer') listeners = [(80, 80, 'HTTP', 'HTTP')] lb = elb_conn.create_load_balancer(name=load_balancer_name, zones=None, complex_listeners=listeners, security_groups=lb_group_ids, subnets=[aws.subnet_id]) hc = HealthCheck(target=health_check_target) lb.configure_health_check(hc) cda = ConnectionDrainingAttribute() cda.enabled = True cda.timeout = 300 elb_conn.modify_lb_attribute(load_balancer_name=load_balancer_name, attribute='connectionDraining', value=cda) print '-----> Connecting to AutoScale' as_conn = boto.connect_autoscale() log('info', 'Launch configuration', show_header=True) launch_config_name = "{}-{}".format(name, app_image.tags['deploy-id']) lc_result = as_conn.get_all_launch_configurations( names=[launch_config_name]) if len(lc_result) == 0: log('info', 'Not found, creating LaunchConfiguration') lc = LaunchConfiguration(name=launch_config_name, image_id=app_image.id, key_name=aws.key_pair_name, security_groups=inst_group_ids, instance_type=aws.instance_type) as_conn.create_launch_configuration(lc) else: log('info', 'Found {}'.format(launch_config_name)) lc = lc_result[0] existing_group = None deploy_id = int(app_image.tags['deploy-id'] or 0) log('info', 'Getting previous auto-scaling group', show_header=True) for did in xrange(deploy_id - 1, 0, -1): existing_group_name = "{}-{}".format(name, did) log('info', '{} ?'.format(existing_group_name)) ag_result = as_conn.get_all_groups(names=[existing_group_name]) if len(ag_result) > 0: existing_group = ag_result[0] log('info', 'Found {}'.format(existing_group.name)) break else: log('info', 'No') if existing_group is not None: existing_healthy_instances = [ inst for inst in existing_group.instances if inst.lifecycle_state == 'InService' and inst.health_status == 'Healthy' ] existing_healthy_instance_count = len(existing_healthy_instances) desired_capacity = existing_group.desired_capacity min_size = existing_group.min_size max_size = existing_group.max_size if existing_healthy_instance_count == 0 and desired_capacity == 0: print '-----> WARNING: existing auto-scaling group {} has no healthy instances and a desired capacity of 0. New auto-scaling group will launch 1 instance.'.format( existing_group) desired_capacity = 1 min_size = 1 max_size = max_size if max_size > 0 else 1 else: existing_healthy_instance_count = 0 desired_capacity = 1 min_size = 1 max_size = 1 log('info', '{} existing instance(s) found'.format( existing_healthy_instance_count), show_header=True) log( 'info', 'Existing auto-scale properties: desired_capacity={}, min_size={}, max_size={}' .format(desired_capacity, min_size, max_size)) log('info', 'Auto-scaling group', show_header=True) group_name = "{}-{}".format(name, app_image.tags['deploy-id']) ag_result = as_conn.get_all_groups(names=[group_name]) if len(ag_result) == 0: log('info', 'Not found, creating autoscale group') ag = AutoScalingGroup(name=group_name, load_balancers=[load_balancer_name], launch_config=lc, desired_capacity=desired_capacity, min_size=min_size, max_size=max_size, health_check_type='ELB', health_check_period='300', vpc_zone_identifier=aws.subnet_id) as_conn.create_auto_scaling_group(ag) else: log('info', 'Found {}'.format(group_name)) ag = ag_result[0] ag.desired_capacity = desired_capacity ag.max_size = max_size ag.min_size = min_size ag.launch_config_name = launch_config_name ag.update() log('info', 'Waiting for new instances to become healthy', show_header=True) all_healthy = False for i in xrange(60): if i > 0: print ' ---' time.sleep(10) elb_result = elb_conn.get_all_load_balancers( load_balancer_names=[load_balancer_name]) lb = elb_result[0] lb_insts = lb.get_instance_health() print ' Load-balancer instances: {}'.format(lb_insts) # NOTE: re-get auto-scaling group to get updated instance info. ag = as_conn.get_all_groups(names=[group_name])[0] ag_insts = [inst for inst in ag.instances] log('info', 'Auto-scaling group Instances: {}'.format(ag_insts)) if len(ag_insts) < desired_capacity: not_yet_launched_count = desired_capacity - len(ag_insts) log( 'info', '{} new instance(s) not yet launched'.format( not_yet_launched_count)) continue ag_inst_ids = set(inst.instance_id for inst in ag_insts) lb_inst_ids = set(inst.instance_id for inst in lb_insts) asg_insts_not_in_lb = ag_inst_ids.difference(lb_inst_ids) if len(asg_insts_not_in_lb) > 0: log( 'info', '{} new instance(s) not yet in load balancer'.format( len(asg_insts_not_in_lb))) continue new_lb_insts = [ inst for inst in lb_insts if inst.instance_id in ag_inst_ids ] healthy_new_lb_insts = [ inst for inst in new_lb_insts if inst.state == 'InService' ] all_healthy = len(healthy_new_lb_insts) == len(ag_insts) log('info', '{} new instance(s) are healthy'.format(len(healthy_new_lb_insts))) diff = existing_healthy_instance_count - len(healthy_new_lb_insts) if existing_group is not None and diff >= 0: change = False if existing_group.desired_capacity != diff: existing_group.desired_capacity = diff change = True if existing_group.max_size != diff: existing_group.max_size = diff change = True if diff < existing_group.min_size: existing_group.min_size = diff change = True if change: existing_group.update() log( 'info', 'Change previous auto-scale group {} properties: desired_capacity={}, min_size={}, max_size={}' .format(existing_group, existing_group.desired_capacity, existing_group.min_size, existing_group.max_size)) if all_healthy: log('info', 'All new instances healthy!', show_header=True) healthy_lb_inst_ids = [ inst.instance_id for inst in lb_insts if inst.state == 'InService' ] previous_healthy_inst_ids = [ inst.instance_id for inst in existing_healthy_instances ] if existing_group else [] not_yet_out_of_service = set( previous_healthy_inst_ids).intersection(healthy_lb_inst_ids) if len(not_yet_out_of_service) > 0: log( 'info', 'Waiting to remove previous instances ({}) from load balancer' .format(not_yet_out_of_service)) else: log('info', 'All previous instances ({}) have been removed from load balancer' .format(previous_healthy_inst_ids), show_header=True) if all_healthy and len(not_yet_out_of_service) == 0: break else: raise Exception("Timeout") elb_result = elb_conn.get_all_load_balancers( load_balancer_names=[load_balancer_name]) lb = elb_result[0] lb_insts = [ inst for inst in lb.get_instance_health() if inst.state == 'InService' ] print '-----> Deployed {} instance(s) of {} to {}'.format( lb_insts, app.name, lb.dns_name) print '-----> DONE!'