def verify_tag_dict_keys(as_group, tag_name, key_list): if not get_tag_dict_value(as_group, tag_name) or not all(key in get_tag_dict_value(as_group, tag_name).keys() for key in key_list): print_verbose(os.path.basename( __file__), 'warn', "Tag not found or not all expected keys found for %s. Initializing." % tag_name) return False else: print_verbose( os.path.basename(__file__), 'info', "Expected keys found for %s." % tag_name) return True
def verify_tag_dict_keys(as_group, tag_name, key_list): if not get_tag_dict_value(as_group, tag_name) or not all( key in get_tag_dict_value(as_group, tag_name).keys() for key in key_list): print_verbose( os.path.basename(__file__), 'warn', "Tag not found or not all expected keys found for %s. Initializing." % tag_name) return False else: print_verbose(os.path.basename(__file__), 'info', "Expected keys found for %s." % tag_name) return True
def get_healthy_zones(as_group, min_health_threshold): az_status = get_tag_dict_value(as_group, 'AZ_status') zone_prefix = as_group.availability_zones[0][:-1] return [ zone_prefix + t for t in az_status if az_status[t]['use'] and az_status[t]['health'].count(0) >= min_health_threshold ]
def get_max_bid(as_group): try: demand_price = get_ondemand_price(get_launch_config(as_group)) original_bid = get_tag_dict_value( as_group, 'ssr_config')['original_bid'] if get_rounded_price(demand_price) <= get_rounded_price(original_bid): return original_bid else: return demand_price except Exception as e: handle_exception(e) sys.exit(1)
def get_max_bid(as_group): try: demand_price = get_ondemand_price(get_launch_config(as_group)) original_bid = get_tag_dict_value(as_group, 'ssr_config')['original_bid'] if get_rounded_price(demand_price) <= get_rounded_price(original_bid): return original_bid else: return demand_price except Exception as e: handle_exception(e) sys.exit(1)
def set_tag_dict_value(as_group, tag_key, val_key, value): as_group = reload_as_group(as_group) tag_val = get_tag_dict_value(as_group, tag_key) tag_val[val_key] = value create_tag(as_group, tag_key, tag_val)
def main(args): global verbose global dry_run (verbose, dry_run) = dry_run_necessaries(args.dry_run, args.verbose) for region in [r.name for r in boto.ec2.regions() if r.name not in args.excluded_regions]: try: print_verbose( os.path.basename(__file__), 'info', 'Starting pass on %s' % region) as_conn = boto.ec2.autoscale.connect_to_region(region) as_groups = get_ssr_groups(as_conn) elb_conn = boto.ec2.elb.connect_to_region(as_conn.region.name) minutes_multiplier = 60 for as_group in as_groups: as_group = reload_as_group(as_group) print_verbose( os.path.basename(__file__), 'info', "Checking %s" % as_group.name) if as_group.load_balancers: maximize_elb_azs(elb_conn, as_group, dry_run) demand_expiration = get_tag_dict_value( as_group, 'ssr_config')['demand_expiration'] healthy_zones = get_healthy_zones( as_group, args.min_health_threshold) if demand_expiration is not False: if demand_expiration < int(time.time()): if len(healthy_zones) >= get_min_azs(as_group): print_verbose(os.path.basename( __file__), 'info', 'Woot! We can move back to spots at original bid price.') modify_as_group_azs( as_group, healthy_zones, dry_run) modify_price( as_group, get_tag_dict_value(as_group, 'ssr_config')['original_bid'], dry_run) set_tag_dict_value( as_group, 'ssr_config', 'demand_expiration', False) # kill all demand instances that were created ec2_conn = boto.ec2.connect_to_region( as_group.connection.region.name) all_ec2_instances = ec2_conn.get_all_instances() print_verbose(os.path.basename( __file__), 'info', "Looking at %s instances for potential termination" % str(len(as_group.instances))) for instance in as_group.instances: if not [i for i in all_ec2_instances if i.instances[0].id == instance.instance_id][0].instances[0].spot_instance_request_id and \ not dry_run: terminate_instance(instance) else: print_verbose(os.path.basename( __file__), 'info', 'Extending the life of demand instances as we cant fulfill with spots still') set_tag_dict_value(as_group, 'ssr_config', 'demand_expiration', int( time.time()) + (args.demand_expiration * minutes_multiplier)) elif sorted(as_group.availability_zones) != sorted(healthy_zones): as_group = reload_as_group(as_group) print_verbose( os.path.basename(__file__), 'info', "Healthy zones and zones in use dont match") if len(healthy_zones) >= get_min_azs(as_group): print_verbose( os.path.basename(__file__), 'info', 'Modifying zones accordingly.') modify_as_group_azs(as_group, healthy_zones, dry_run) else: print_verbose(os.path.basename( __file__), 'info', "Bid will need to be modified as we can't meet AZ minimum of %s" % str(get_min_azs(as_group))) best_bid = find_best_bid_price(as_group) print_verbose(os.path.basename( __file__), 'info', "Best possible bid given AZ minimum is %s" % str(best_bid)) if best_bid: modify_price(as_group, best_bid, dry_run) else: print_verbose( os.path.basename(__file__), 'info', "Moving to ondemand.") modify_price( as_group, None, dry_run, minutes_multiplier, args.demand_expiration) set_tag_dict_value(as_group, 'ssr_config', 'demand_expiration', int( time.time()) + (args.demand_expiration * minutes_multiplier)) modify_as_group_azs( as_group, get_usable_zones(as_group), dry_run) else: print_verbose( os.path.basename(__file__), 'info', 'No further actions to take on this ASG.') print_verbose( os.path.basename(__file__), 'info', 'Done with pass on %s' % region) except EC2ResponseError as e: handle_exception(e) except Exception as e: handle_exception(e) return 1 print_verbose(os.path.basename(__file__), 'info', "All regions complete")
def get_usable_zones(as_group): az_status = get_tag_dict_value(as_group, 'AZ_status') zone_prefix = as_group.availability_zones[0][:-1] return [zone_prefix + t for t in az_status if az_status[t]['use']]
def get_healthy_zones(as_group, min_health_threshold): az_status = get_tag_dict_value(as_group, 'AZ_status') zone_prefix = as_group.availability_zones[0][:-1] return [zone_prefix + t for t in az_status if az_status[t]['use'] and az_status[t]['health'].count(0) >= min_health_threshold]
def get_min_azs(as_group): return int(get_tag_dict_value(as_group, 'ssr_config')['min_AZs'])
def main(args): global verbose global dry_run (verbose, dry_run) = dry_run_necessaries(args.dry_run, args.verbose) for region in [r.name for r in boto.ec2.regions() if r.name not in args.excluded_regions]: try: print_verbose( os.path.basename(__file__), 'info', 'Starting pass on %s' % region) ec2_conn = boto.ec2.connect_to_region(region) as_conn = boto.ec2.autoscale.connect_to_region(region) all_groups = as_conn.get_all_groups() spot_lcs = [ e for e in as_conn.get_all_launch_configurations() if e.spot_price] # these need to be pulled from the same all_groups list or # duplicate objects will be seen as distinct. spot_lc_groups = [ g for g in all_groups if g.launch_config_name in [s.name for s in spot_lcs]] previously_ssr_managed_groups = [g for g in all_groups if get_tag_dict_value( g, 'ssr_config') and get_tag_dict_value(g, 'ssr_config')['enabled'] is True] all_groups = list( set(spot_lc_groups + previously_ssr_managed_groups)) for as_group in all_groups: print_verbose( os.path.basename(__file__), 'info', "Evaluating %s" % as_group.name) # this latter condition can happen when tag value (a dict) # can't be interpreted by ast.literal_eval() if args.reset_tags or not [t for t in as_group.tags if t.key == 'ssr_config'] or not get_tag_dict_value(as_group, 'ssr_config'): print_verbose(os.path.basename( __file__), 'info', 'Tags not found or reset tags option flagged. Adding all tags anew now.') init_ssr_config_tag(as_group, args.min_healthy_AZs) init_az_status_tag(as_group) elif [t for t in as_group.tags if t.key == 'ssr_config' and not get_tag_dict_value(as_group, 'ssr_config')['enabled']]: print_verbose( os.path.basename(__file__), 'info', 'ssr_config DISABLED. Doing nothing.') elif [t for t in as_group.tags if t.key == 'ssr_config' and get_tag_dict_value(as_group, 'ssr_config')['enabled']]: print_verbose(os.path.basename( __file__), 'info', 'ssr management enabled. Verifying all config values in place.') config_keys = [ 'enabled', 'original_bid', 'LC_name', 'min_AZs', 'demand_expiration'] if not verify_tag_dict_keys(as_group, 'ssr_config', config_keys) or \ not get_tag_dict_value(as_group, 'ssr_config')['LC_name'] == as_group.launch_config_name[-155:]: # this would indicate a change to the LC outside of ssr # scope. In that case, we need to disable ssr via tag # deletion. if not get_launch_config(as_group).spot_price: del_ssr_tags(as_group) continue else: init_ssr_config_tag(as_group, args.min_healthy_AZs) zones = [z.name[-1] for z in ec2_conn.get_all_zones()] if not verify_tag_dict_keys(as_group, 'AZ_status', zones): init_az_status_tag(as_group) else: raise Exception( "ssr_enabled tag found for %s but isn't a valid value." % (as_group.name,)) print_verbose( os.path.basename(__file__), 'info', 'Done with pass on %s' % region) except EC2ResponseError as e: handle_exception(e) except BotoServerError as e: handle_exception(e) except Exception as e: handle_exception(e) return 1 print_verbose(os.path.basename(__file__), 'info', "All regions complete")
def main(args): global verbose global dry_run (verbose, dry_run) = dry_run_necessaries(args.dry_run, args.verbose) for region in [ r.name for r in boto.ec2.regions() if r.name not in args.excluded_regions ]: try: print_verbose(os.path.basename(__file__), 'info', 'Starting pass on %s' % region) as_conn = boto.ec2.autoscale.connect_to_region(region) as_groups = get_ssr_groups(as_conn) elb_conn = boto.ec2.elb.connect_to_region(as_conn.region.name) minutes_multiplier = 60 for as_group in as_groups: as_group = reload_as_group(as_group) print_verbose(os.path.basename(__file__), 'info', "Checking %s" % as_group.name) if as_group.load_balancers: maximize_elb_azs(elb_conn, as_group, dry_run) demand_expiration = get_tag_dict_value( as_group, 'ssr_config')['demand_expiration'] healthy_zones = get_healthy_zones(as_group, args.min_health_threshold) if demand_expiration is not False: if demand_expiration < int(time.time()): if len(healthy_zones) >= get_min_azs(as_group): print_verbose( os.path.basename(__file__), 'info', 'Woot! We can move back to spots at original bid price.' ) modify_as_group_azs(as_group, healthy_zones, dry_run) modify_price( as_group, get_tag_dict_value( as_group, 'ssr_config')['original_bid'], dry_run) set_tag_dict_value(as_group, 'ssr_config', 'demand_expiration', False) # kill all demand instances that were created ec2_conn = boto.ec2.connect_to_region( as_group.connection.region.name) all_ec2_instances = ec2_conn.get_all_instances() print_verbose( os.path.basename(__file__), 'info', "Looking at %s instances for potential termination" % str(len(as_group.instances))) for instance in as_group.instances: if not [i for i in all_ec2_instances if i.instances[0].id == instance.instance_id][0].instances[0].spot_instance_request_id and \ not dry_run: terminate_instance(instance) else: print_verbose( os.path.basename(__file__), 'info', 'Extending the life of demand instances as we cant fulfill with spots still' ) set_tag_dict_value( as_group, 'ssr_config', 'demand_expiration', int(time.time()) + (args.demand_expiration * minutes_multiplier)) elif sorted( as_group.availability_zones) != sorted(healthy_zones): as_group = reload_as_group(as_group) print_verbose(os.path.basename(__file__), 'info', "Healthy zones and zones in use dont match") if len(healthy_zones) >= get_min_azs(as_group): print_verbose(os.path.basename(__file__), 'info', 'Modifying zones accordingly.') modify_as_group_azs(as_group, healthy_zones, dry_run) else: print_verbose( os.path.basename(__file__), 'info', "Bid will need to be modified as we can't meet AZ minimum of %s" % str(get_min_azs(as_group))) best_bid = find_best_bid_price(as_group) print_verbose( os.path.basename(__file__), 'info', "Best possible bid given AZ minimum is %s" % str(best_bid)) if best_bid: modify_price(as_group, best_bid, dry_run) else: print_verbose(os.path.basename(__file__), 'info', "Moving to ondemand.") modify_price(as_group, None, dry_run, minutes_multiplier, args.demand_expiration) set_tag_dict_value( as_group, 'ssr_config', 'demand_expiration', int(time.time()) + (args.demand_expiration * minutes_multiplier)) modify_as_group_azs(as_group, get_usable_zones(as_group), dry_run) else: print_verbose(os.path.basename(__file__), 'info', 'No further actions to take on this ASG.') print_verbose(os.path.basename(__file__), 'info', 'Done with pass on %s' % region) except EC2ResponseError as e: handle_exception(e) except Exception as e: handle_exception(e) return 1 print_verbose(os.path.basename(__file__), 'info', "All regions complete")
def main(args): global verbose global dry_run (verbose, dry_run) = dry_run_necessaries(args.dry_run, args.verbose) for region in [ r.name for r in boto.ec2.regions() if r.name not in args.excluded_regions ]: try: print_verbose(os.path.basename(__file__), 'info', 'Starting pass on %s' % region) ec2_conn = boto.ec2.connect_to_region(region) as_conn = boto.ec2.autoscale.connect_to_region(region) all_groups = as_conn.get_all_groups() spot_lcs = [ e for e in as_conn.get_all_launch_configurations() if e.spot_price ] # these need to be pulled from the same all_groups list or # duplicate objects will be seen as distinct. spot_lc_groups = [ g for g in all_groups if g.launch_config_name in [s.name for s in spot_lcs] ] previously_ssr_managed_groups = [ g for g in all_groups if get_tag_dict_value(g, 'ssr_config') and get_tag_dict_value(g, 'ssr_config')['enabled'] is True ] all_groups = list( set(spot_lc_groups + previously_ssr_managed_groups)) for as_group in all_groups: print_verbose(os.path.basename(__file__), 'info', "Evaluating %s" % as_group.name) # this latter condition can happen when tag value (a dict) # can't be interpreted by ast.literal_eval() if args.reset_tags or not [ t for t in as_group.tags if t.key == 'ssr_config' ] or not get_tag_dict_value(as_group, 'ssr_config'): print_verbose( os.path.basename(__file__), 'info', 'Tags not found or reset tags option flagged. Adding all tags anew now.' ) init_ssr_config_tag(as_group, args.min_healthy_AZs) init_az_status_tag(as_group) elif [ t for t in as_group.tags if t.key == 'ssr_config' and not get_tag_dict_value( as_group, 'ssr_config')['enabled'] ]: print_verbose(os.path.basename(__file__), 'info', 'ssr_config DISABLED. Doing nothing.') elif [ t for t in as_group.tags if t.key == 'ssr_config' and get_tag_dict_value( as_group, 'ssr_config')['enabled'] ]: print_verbose( os.path.basename(__file__), 'info', 'ssr management enabled. Verifying all config values in place.' ) config_keys = [ 'enabled', 'original_bid', 'LC_name', 'min_AZs', 'demand_expiration' ] if not verify_tag_dict_keys(as_group, 'ssr_config', config_keys) or \ not get_tag_dict_value(as_group, 'ssr_config')['LC_name'] == as_group.launch_config_name[-155:]: # this would indicate a change to the LC outside of ssr # scope. In that case, we need to disable ssr via tag # deletion. if not get_launch_config(as_group).spot_price: del_ssr_tags(as_group) continue else: init_ssr_config_tag(as_group, args.min_healthy_AZs) zones = [z.name[-1] for z in ec2_conn.get_all_zones()] if not verify_tag_dict_keys(as_group, 'AZ_status', zones): init_az_status_tag(as_group) else: raise Exception( "ssr_enabled tag found for %s but isn't a valid value." % (as_group.name, )) print_verbose(os.path.basename(__file__), 'info', 'Done with pass on %s' % region) except EC2ResponseError as e: handle_exception(e) except BotoServerError as e: handle_exception(e) except Exception as e: handle_exception(e) return 1 print_verbose(os.path.basename(__file__), 'info', "All regions complete")