Example #1
0
    def run(self):
        all_lcs = self.as_conn.get_all_launch_configurations()
        lc_by_group = defaultdict(list)
        lc_max_num_by_group = defaultdict(int)

        for lc in all_lcs:
            name, num = lc.name.split('-')
            num = int(num)

            lc_by_group[name].append(lc)

            if num > lc_max_num_by_group[name]:
                lc_max_num_by_group[name] = num

        all_ags = self.as_conn.get_all_groups()
        ag_by_name = {}
        for ag in all_ags:
            ag_by_name[ag.name] = ag

        for group_name, config in self.as_config["groups"].iteritems():
            print "Configuring %s" % group_name
            use_lc = None
            lc_to_delete = []
            for lc in lc_by_group[group_name]:
                if use_lc is None and \
                   lc.image_id == config['ami'] and \
                   lc.key_name == config['ssh_key'] and \
                   lc.instance_type == config['instance_type'] and \
                   lc.security_groups == [config['security_group']] and \
                   lc.user_data == self.user_data:
                    print " Found LaunchConfig %s that matches profile" % \
                            lc.name
                    use_lc = lc
                else:
                    lc_to_delete.append(lc)

            print " Found %d LaunchConfigurations to delete" % len(
                lc_to_delete)

            if not use_lc:
                print " Making LaunchConfiguration for %s" % group_name
                lc_num = lc_max_num_by_group[group_name] + 1
                use_lc = LaunchConfiguration(
                    name="%s-%d" % (group_name, lc_num),
                    image_id=config['ami'],
                    key_name=config['ssh_key'],
                    instance_type=config['instance_type'],
                    security_groups=[config['security_group']],
                    user_data=self.user_data)
                self.as_conn.create_launch_configuration(use_lc)

            if group_name in ag_by_name:
                print " Found existing AutoScalingGroup, updating"
                ag = ag_by_name[group_name]
                ag_exists = True
            else:
                print " Making new AutoScalingGroup"
                ag = AutoScalingGroup()
                ag_exists = False

            # config ASG as we want it
            ag.name = group_name
            ag.launch_config_name = use_lc.name
            ag.availability_zones = config['zones']
            ag.desired_capacity = config['capacity']
            ag.min_size = config['min_size']
            ag.max_size = config['max_size']

            # create or update as appropriate
            if ag_exists:
                ag.update()
            else:
                self.as_conn.create_auto_scaling_group(ag)

            # make it send e-mail whenever it does something
            if 'notification_topic' in self.as_config:
                # NOTE [adam Sept/18/12]: this is a hack designed to work
                #      around that boto support for this isn't in a release yet.
                #      when the next release is out, we should uncomment the
                #      code below.
                params = {
                    'AutoScalingGroupName': ag.name,
                    'TopicARN': self.as_config['notification_topic']
                }
                self.as_conn.build_list_params(params, self.AS_NOTIFICATIONS,
                                               'NotificationTypes')
                self.as_conn.get_status('PutNotificationConfiguration', params)
                #as_conn.put_notification_configuration(
                #       ag.name,
                #       self.as_config['notification_topic'],
                #       self.AS_NOTIFICATIONS)

            tags = []
            for tag_name, tag_value in config.get('tags', {}).iteritems():
                print " Adding tag %s = %s" % (tag_name, tag_value)
                tags.append(
                    Tag(key=tag_name,
                        value=tag_value,
                        propagate_at_launch=True,
                        resource_id=ag.name))

            self.as_conn.create_or_update_tags(tags)

            for lc in lc_to_delete:
                print " Deleting old LaunchConfiguration %s" % lc.name
                lc.delete()

            for alarm_name, alarm_cfg in config.get('alarms', {}).iteritems():
                alarm_policy_arn = self.make_policy(group_name,
                                                    alarm_cfg['policy'])

                alarm_name = '%s|%s|%s' % (group_name, alarm_cfg['policy'],
                                           alarm_cfg['metric'])
                alarm = MetricAlarm(
                    name=alarm_name,
                    namespace=alarm_cfg['namespace'],
                    metric=alarm_cfg['metric'],
                    statistic='Average',
                    dimensions={'AutoScalingGroupName': group_name},
                    comparison=alarm_cfg['comparison'],
                    threshold=alarm_cfg['threshold'],
                    period=alarm_cfg['period'],
                    evaluation_periods=alarm_cfg.get('evaluation_periods', 1),
                    alarm_actions=[alarm_policy_arn])

                self.cw_conn.put_metric_alarm(alarm)
Example #2
0
File: api.py Project: gimlids/riker
def deploy_latest_app_ami(app_name, env_name):

    global aws
    aws = AWS.from_config(config)

    aws.connect()

    lb_group_ids=[aws.get_security_group_id('riker-load-balancer')]
    inst_group_ids=[aws.get_security_group_id('riker-instance')]

    app = App(env_name, app_name)

    health_check_target = app.config.get('health_check', 'TCP:80')

    name = re.sub('[^A-Za-z0-9\-]', '-', app.name)

    app_image = LatestAppImage(app).get()

    print '-----> Connecting to ELB'
    elb_conn = boto.connect_elb()

    log('info', 'Load balancer', show_header=True)
    load_balancer_name = name
    try:
        elb_result = elb_conn.get_all_load_balancers(load_balancer_names=[load_balancer_name])
        lb = elb_result[0]
        log('info', 'Found {}'.format(load_balancer_name))
    except boto.exception.BotoServerError:
        log('info', 'Not found, creating load balancer')
        listeners = [(80, 80, 'HTTP', 'HTTP')]
        lb = elb_conn.create_load_balancer(name=load_balancer_name,
                                           zones=None,
                                           complex_listeners=listeners,
                                           security_groups=lb_group_ids,
                                           subnets=[aws.subnet_id])
    hc = HealthCheck(target=health_check_target)
    lb.configure_health_check(hc)
    cda = ConnectionDrainingAttribute()
    cda.enabled = True
    cda.timeout = 300
    elb_conn.modify_lb_attribute(load_balancer_name=load_balancer_name,
                                  attribute='connectionDraining',
                                  value=cda)

    print '-----> Connecting to AutoScale'
    as_conn = boto.connect_autoscale()

    log('info', 'Launch configuration', show_header=True)
    launch_config_name = "{}-{}".format(name, app_image.tags['deploy-id'])
    lc_result = as_conn.get_all_launch_configurations(names=[launch_config_name])
    if len(lc_result) == 0:
        log('info', 'Not found, creating LaunchConfiguration')
        lc = LaunchConfiguration(name=launch_config_name,
                                 image_id=app_image.id,
                                 key_name=aws.key_pair_name,
                                 security_groups=inst_group_ids,
                                 instance_type=aws.instance_type)
        as_conn.create_launch_configuration(lc)
    else:
        log('info', 'Found {}'.format(launch_config_name))
        lc = lc_result[0]

    existing_group = None
    deploy_id = int(app_image.tags['deploy-id'] or 0)
    log('info', 'Getting previous auto-scaling group', show_header=True)
    for did in xrange(deploy_id-1, 0, -1):
        existing_group_name = "{}-{}".format(name, did)
        log('info', '{} ?'.format(existing_group_name))
        ag_result = as_conn.get_all_groups(names=[existing_group_name])
        if len(ag_result) > 0:
            existing_group = ag_result[0]
            log('info', 'Found {}'.format(existing_group.name))
            break
        else:
            log('info', 'No')

    if existing_group is not None:
        existing_healthy_instances = [inst for inst in existing_group.instances if inst.lifecycle_state == 'InService' and inst.health_status == 'Healthy']
        existing_healthy_instance_count = len(existing_healthy_instances)
        desired_capacity = existing_group.desired_capacity
        min_size = existing_group.min_size
        max_size = existing_group.max_size
        if existing_healthy_instance_count == 0 and desired_capacity == 0:
            print '-----> WARNING: existing auto-scaling group {} has no healthy instances and a desired capacity of 0. New auto-scaling group will launch 1 instance.'.format(existing_group)
            desired_capacity = 1
            min_size = 1
            max_size = max_size if max_size > 0 else 1
    else:
        existing_healthy_instance_count = 0
        desired_capacity = 1
        min_size = 1
        max_size = 1

    log('info', '{} existing instance(s) found'.format(existing_healthy_instance_count), show_header=True)

    log('info', 'Existing auto-scale properties: desired_capacity={}, min_size={}, max_size={}'.format(desired_capacity, min_size, max_size))

    log('info', 'Auto-scaling group', show_header=True)
    group_name = "{}-{}".format(name, app_image.tags['deploy-id'])
    ag_result = as_conn.get_all_groups(names=[group_name])
    if len(ag_result) == 0:
        log('info', 'Not found, creating autoscale group')
        ag = AutoScalingGroup(name=group_name,
                              load_balancers=[load_balancer_name], launch_config=lc,
                              desired_capacity=desired_capacity, min_size=min_size, max_size=max_size,
                              health_check_type='ELB', health_check_period='300',
                              vpc_zone_identifier=aws.subnet_id)
        as_conn.create_auto_scaling_group(ag)
    else:
        log('info', 'Found {}'.format(group_name))
        ag = ag_result[0]
        ag.desired_capacity = desired_capacity
        ag.max_size = max_size
        ag.min_size = min_size
        ag.launch_config_name = launch_config_name
        ag.update()

    log('info', 'Waiting for new instances to become healthy', show_header=True)
    all_healthy = False
    for i in xrange(60):
        if i > 0:
            print '       ---'
            time.sleep(10)
        elb_result = elb_conn.get_all_load_balancers(load_balancer_names=[load_balancer_name])
        lb = elb_result[0]
        lb_insts = lb.get_instance_health()
        print '       Load-balancer instances: {}'.format(lb_insts)
        # NOTE: re-get auto-scaling group to get updated instance info.
        ag = as_conn.get_all_groups(names=[group_name])[0]
        ag_insts = [inst for inst in ag.instances]
        log('info', 'Auto-scaling group Instances: {}'.format(ag_insts))
        if len(ag_insts) < desired_capacity:
            not_yet_launched_count = desired_capacity - len(ag_insts)
            log('info', '{} new instance(s) not yet launched'.format(not_yet_launched_count))
            continue
        ag_inst_ids = set(inst.instance_id for inst in ag_insts)
        lb_inst_ids = set(inst.instance_id for inst in lb_insts)
        asg_insts_not_in_lb = ag_inst_ids.difference(lb_inst_ids)
        if len(asg_insts_not_in_lb) > 0:
            log('info', '{} new instance(s) not yet in load balancer'.format(len(asg_insts_not_in_lb)))
            continue
        new_lb_insts = [inst for inst in lb_insts if inst.instance_id in ag_inst_ids]
        healthy_new_lb_insts = [inst for inst in new_lb_insts if inst.state == 'InService']
        all_healthy = len(healthy_new_lb_insts) == len(ag_insts)
        log('info', '{} new instance(s) are healthy'.format(len(healthy_new_lb_insts)))
        diff = existing_healthy_instance_count - len(healthy_new_lb_insts)
        if existing_group is not None and diff >= 0:
            change = False
            if existing_group.desired_capacity != diff:
                existing_group.desired_capacity = diff
                change = True
            if existing_group.max_size != diff:
                existing_group.max_size = diff
                change = True
            if diff < existing_group.min_size:
                existing_group.min_size = diff
                change = True
            if change:
                existing_group.update()
                log('info', 'Change previous auto-scale group {} properties: desired_capacity={}, min_size={}, max_size={}'.format(existing_group, existing_group.desired_capacity, existing_group.min_size, existing_group.max_size))
        if all_healthy:
            log('info', 'All new instances healthy!', show_header=True)
            healthy_lb_inst_ids = [inst.instance_id for inst in lb_insts if inst.state == 'InService']
            previous_healthy_inst_ids = [inst.instance_id for inst in existing_healthy_instances] if existing_group else []
            not_yet_out_of_service = set(previous_healthy_inst_ids).intersection(healthy_lb_inst_ids)
            if len(not_yet_out_of_service) > 0:
                log('info', 'Waiting to remove previous instances ({}) from load balancer'.format(not_yet_out_of_service))
            else:
                log('info', 'All previous instances ({}) have been removed from load balancer'.format(previous_healthy_inst_ids), show_header=True)
        if all_healthy and len(not_yet_out_of_service) == 0:
            break
    else:
        raise Exception("Timeout")

    elb_result = elb_conn.get_all_load_balancers(load_balancer_names=[load_balancer_name])
    lb = elb_result[0]
    lb_insts = [inst for inst in lb.get_instance_health() if inst.state == 'InService']
    print '-----> Deployed {} instance(s) of {} to {}'.format(lb_insts, app.name, lb.dns_name)

    print '-----> DONE!'
Example #3
0
    def run(self):
        all_lcs = self.as_conn.get_all_launch_configurations()
        lc_by_group = defaultdict(list)
        lc_max_num_by_group = defaultdict(int)

        for lc in all_lcs:
            name, num = lc.name.split('-')
            num = int(num)

            lc_by_group[name].append(lc)

            if num > lc_max_num_by_group[name]:
                lc_max_num_by_group[name] = num

        all_ags = self.as_conn.get_all_groups()
        ag_by_name = {}
        for ag in all_ags:
            ag_by_name[ag.name] = ag

        for group_name, config in self.as_config["groups"].iteritems():
            print "Configuring %s" % group_name
            use_lc = None
            lc_to_delete = []
            for lc in lc_by_group[group_name]:
                if use_lc is None and \
                   lc.image_id == config['ami'] and \
                   lc.key_name == config['ssh_key'] and \
                   lc.instance_type == config['instance_type'] and \
                   lc.security_groups == [config['security_group']] and \
                   lc.user_data == self.user_data:
                    print " Found LaunchConfig %s that matches profile" % \
                            lc.name
                    use_lc = lc
                else:
                    lc_to_delete.append(lc)

            print " Found %d LaunchConfigurations to delete" % len(lc_to_delete)

            if not use_lc:
                print " Making LaunchConfiguration for %s" % group_name
                lc_num = lc_max_num_by_group[group_name] + 1
                use_lc = LaunchConfiguration(
                                 name="%s-%d" % (group_name, lc_num),
                                 image_id=config['ami'],
                                 key_name=config['ssh_key'],
                                 instance_type=config['instance_type'],
                                 security_groups=[config['security_group']],
                                 user_data=self.user_data)
                self.as_conn.create_launch_configuration(use_lc)

            if group_name in ag_by_name:
                print " Found existing AutoScalingGroup, updating"
                ag = ag_by_name[group_name]
                ag_exists = True
            else:
                print " Making new AutoScalingGroup"
                ag = AutoScalingGroup()
                ag_exists = False

            # config ASG as we want it
            ag.name = group_name
            ag.launch_config_name = use_lc.name
            ag.availability_zones = config['zones']
            ag.desired_capacity = config['capacity']
            ag.min_size = config['min_size']
            ag.max_size = config['max_size']

            # create or update as appropriate
            if ag_exists:
                ag.update()
            else:
                self.as_conn.create_auto_scaling_group(ag)

            # make it send e-mail whenever it does something
            if 'notification_topic' in self.as_config:
                # NOTE [adam Sept/18/12]: this is a hack designed to work
                #      around that boto support for this isn't in a release yet.
                #      when the next release is out, we should uncomment the
                #      code below.
                params = {'AutoScalingGroupName': ag.name,
                          'TopicARN': self.as_config['notification_topic'] }
                self.as_conn.build_list_params(params, self.AS_NOTIFICATIONS,
                                               'NotificationTypes')
                self.as_conn.get_status('PutNotificationConfiguration', params)
                #as_conn.put_notification_configuration(
                #       ag.name,
                #       self.as_config['notification_topic'],
                #       self.AS_NOTIFICATIONS)

            tags = []
            for tag_name, tag_value in config.get('tags', {}).iteritems():
                print " Adding tag %s = %s" % (tag_name, tag_value)
                tags.append(Tag(key=tag_name, value=tag_value,
                                propagate_at_launch=True,
                                resource_id=ag.name))

            self.as_conn.create_or_update_tags(tags)

            for lc in lc_to_delete:
                print " Deleting old LaunchConfiguration %s" % lc.name
                lc.delete()

            for alarm_name, alarm_cfg in config.get('alarms', {}).iteritems():
                alarm_policy_arn = self.make_policy(group_name,
                                          alarm_cfg['policy'])

                alarm_name = '%s|%s|%s' % (group_name,
                                           alarm_cfg['policy'],
                                           alarm_cfg['metric'])
                alarm = MetricAlarm(
                    name=alarm_name,
                    namespace=alarm_cfg['namespace'],
                    metric=alarm_cfg['metric'],
                    statistic='Average',
                    dimensions={'AutoScalingGroupName': group_name},
                    comparison=alarm_cfg['comparison'],
                    threshold=alarm_cfg['threshold'],
                    period=alarm_cfg['period'],
                    evaluation_periods=alarm_cfg.get('evaluation_periods', 1),
                    alarm_actions=[alarm_policy_arn])

                self.cw_conn.put_metric_alarm(alarm)
Example #4
0
File: api.py Project: ddnn55/riker
def deploy_latest_app_ami(app_name, env_name):

    global aws
    aws = AWS.from_config(config)

    aws.connect()

    lb_group_ids = [aws.get_security_group_id('riker-load-balancer')]
    inst_group_ids = [aws.get_security_group_id('riker-instance')]

    app = App(env_name, app_name)

    health_check_target = app.config.get('health_check', 'TCP:80')

    name = re.sub('[^A-Za-z0-9\-]', '-', app.name)

    app_image = LatestAppImage(app).get()

    print '-----> Connecting to ELB'
    elb_conn = boto.connect_elb()

    log('info', 'Load balancer', show_header=True)
    load_balancer_name = name
    try:
        elb_result = elb_conn.get_all_load_balancers(
            load_balancer_names=[load_balancer_name])
        lb = elb_result[0]
        log('info', 'Found {}'.format(load_balancer_name))
    except boto.exception.BotoServerError:
        log('info', 'Not found, creating load balancer')
        listeners = [(80, 80, 'HTTP', 'HTTP')]
        lb = elb_conn.create_load_balancer(name=load_balancer_name,
                                           zones=None,
                                           complex_listeners=listeners,
                                           security_groups=lb_group_ids,
                                           subnets=[aws.subnet_id])
    hc = HealthCheck(target=health_check_target)
    lb.configure_health_check(hc)
    cda = ConnectionDrainingAttribute()
    cda.enabled = True
    cda.timeout = 300
    elb_conn.modify_lb_attribute(load_balancer_name=load_balancer_name,
                                 attribute='connectionDraining',
                                 value=cda)

    print '-----> Connecting to AutoScale'
    as_conn = boto.connect_autoscale()

    log('info', 'Launch configuration', show_header=True)
    launch_config_name = "{}-{}".format(name, app_image.tags['deploy-id'])
    lc_result = as_conn.get_all_launch_configurations(
        names=[launch_config_name])
    if len(lc_result) == 0:
        log('info', 'Not found, creating LaunchConfiguration')
        lc = LaunchConfiguration(name=launch_config_name,
                                 image_id=app_image.id,
                                 key_name=aws.key_pair_name,
                                 security_groups=inst_group_ids,
                                 instance_type=aws.instance_type)
        as_conn.create_launch_configuration(lc)
    else:
        log('info', 'Found {}'.format(launch_config_name))
        lc = lc_result[0]

    existing_group = None
    deploy_id = int(app_image.tags['deploy-id'] or 0)
    log('info', 'Getting previous auto-scaling group', show_header=True)
    for did in xrange(deploy_id - 1, 0, -1):
        existing_group_name = "{}-{}".format(name, did)
        log('info', '{} ?'.format(existing_group_name))
        ag_result = as_conn.get_all_groups(names=[existing_group_name])
        if len(ag_result) > 0:
            existing_group = ag_result[0]
            log('info', 'Found {}'.format(existing_group.name))
            break
        else:
            log('info', 'No')

    if existing_group is not None:
        existing_healthy_instances = [
            inst for inst in existing_group.instances
            if inst.lifecycle_state == 'InService'
            and inst.health_status == 'Healthy'
        ]
        existing_healthy_instance_count = len(existing_healthy_instances)
        desired_capacity = existing_group.desired_capacity
        min_size = existing_group.min_size
        max_size = existing_group.max_size
        if existing_healthy_instance_count == 0 and desired_capacity == 0:
            print '-----> WARNING: existing auto-scaling group {} has no healthy instances and a desired capacity of 0. New auto-scaling group will launch 1 instance.'.format(
                existing_group)
            desired_capacity = 1
            min_size = 1
            max_size = max_size if max_size > 0 else 1
    else:
        existing_healthy_instance_count = 0
        desired_capacity = 1
        min_size = 1
        max_size = 1

    log('info',
        '{} existing instance(s) found'.format(
            existing_healthy_instance_count),
        show_header=True)

    log(
        'info',
        'Existing auto-scale properties: desired_capacity={}, min_size={}, max_size={}'
        .format(desired_capacity, min_size, max_size))

    log('info', 'Auto-scaling group', show_header=True)
    group_name = "{}-{}".format(name, app_image.tags['deploy-id'])
    ag_result = as_conn.get_all_groups(names=[group_name])
    if len(ag_result) == 0:
        log('info', 'Not found, creating autoscale group')
        ag = AutoScalingGroup(name=group_name,
                              load_balancers=[load_balancer_name],
                              launch_config=lc,
                              desired_capacity=desired_capacity,
                              min_size=min_size,
                              max_size=max_size,
                              health_check_type='ELB',
                              health_check_period='300',
                              vpc_zone_identifier=aws.subnet_id)
        as_conn.create_auto_scaling_group(ag)
    else:
        log('info', 'Found {}'.format(group_name))
        ag = ag_result[0]
        ag.desired_capacity = desired_capacity
        ag.max_size = max_size
        ag.min_size = min_size
        ag.launch_config_name = launch_config_name
        ag.update()

    log('info',
        'Waiting for new instances to become healthy',
        show_header=True)
    all_healthy = False
    for i in xrange(60):
        if i > 0:
            print '       ---'
            time.sleep(10)
        elb_result = elb_conn.get_all_load_balancers(
            load_balancer_names=[load_balancer_name])
        lb = elb_result[0]
        lb_insts = lb.get_instance_health()
        print '       Load-balancer instances: {}'.format(lb_insts)
        # NOTE: re-get auto-scaling group to get updated instance info.
        ag = as_conn.get_all_groups(names=[group_name])[0]
        ag_insts = [inst for inst in ag.instances]
        log('info', 'Auto-scaling group Instances: {}'.format(ag_insts))
        if len(ag_insts) < desired_capacity:
            not_yet_launched_count = desired_capacity - len(ag_insts)
            log(
                'info', '{} new instance(s) not yet launched'.format(
                    not_yet_launched_count))
            continue
        ag_inst_ids = set(inst.instance_id for inst in ag_insts)
        lb_inst_ids = set(inst.instance_id for inst in lb_insts)
        asg_insts_not_in_lb = ag_inst_ids.difference(lb_inst_ids)
        if len(asg_insts_not_in_lb) > 0:
            log(
                'info', '{} new instance(s) not yet in load balancer'.format(
                    len(asg_insts_not_in_lb)))
            continue
        new_lb_insts = [
            inst for inst in lb_insts if inst.instance_id in ag_inst_ids
        ]
        healthy_new_lb_insts = [
            inst for inst in new_lb_insts if inst.state == 'InService'
        ]
        all_healthy = len(healthy_new_lb_insts) == len(ag_insts)
        log('info',
            '{} new instance(s) are healthy'.format(len(healthy_new_lb_insts)))
        diff = existing_healthy_instance_count - len(healthy_new_lb_insts)
        if existing_group is not None and diff >= 0:
            change = False
            if existing_group.desired_capacity != diff:
                existing_group.desired_capacity = diff
                change = True
            if existing_group.max_size != diff:
                existing_group.max_size = diff
                change = True
            if diff < existing_group.min_size:
                existing_group.min_size = diff
                change = True
            if change:
                existing_group.update()
                log(
                    'info',
                    'Change previous auto-scale group {} properties: desired_capacity={}, min_size={}, max_size={}'
                    .format(existing_group, existing_group.desired_capacity,
                            existing_group.min_size, existing_group.max_size))
        if all_healthy:
            log('info', 'All new instances healthy!', show_header=True)
            healthy_lb_inst_ids = [
                inst.instance_id for inst in lb_insts
                if inst.state == 'InService'
            ]
            previous_healthy_inst_ids = [
                inst.instance_id for inst in existing_healthy_instances
            ] if existing_group else []
            not_yet_out_of_service = set(
                previous_healthy_inst_ids).intersection(healthy_lb_inst_ids)
            if len(not_yet_out_of_service) > 0:
                log(
                    'info',
                    'Waiting to remove previous instances ({}) from load balancer'
                    .format(not_yet_out_of_service))
            else:
                log('info',
                    'All previous instances ({}) have been removed from load balancer'
                    .format(previous_healthy_inst_ids),
                    show_header=True)
        if all_healthy and len(not_yet_out_of_service) == 0:
            break
    else:
        raise Exception("Timeout")

    elb_result = elb_conn.get_all_load_balancers(
        load_balancer_names=[load_balancer_name])
    lb = elb_result[0]
    lb_insts = [
        inst for inst in lb.get_instance_health() if inst.state == 'InService'
    ]
    print '-----> Deployed {} instance(s) of {} to {}'.format(
        lb_insts, app.name, lb.dns_name)

    print '-----> DONE!'