def test_generic(m): r1 = mock.Mock() r1.tags = {"moz-type": "tt1"} r2 = mock.Mock() r2.tags = {"moz-type": "tt2"} r3 = mock.Mock() m.return_value = [r1, r2, r3] assert get_spot_requests_for_moztype("r11", "tt1") == [r1]
def request_spot_instances(all_instances, moz_instance_type, start_count, regions, region_priorities, spot_config, dryrun, slaveset, latest_ami_percentage): started = 0 spot_rules = spot_config.get("rules", {}).get(moz_instance_type) if not spot_rules: log.warn("No spot rules found for %s", moz_instance_type) return 0 instance_config = load_instance_config(moz_instance_type) connections = [get_aws_connection(r) for r in regions] spot_choices = get_spot_choices(connections, spot_rules, "Linux/UNIX (Amazon VPC)") if not spot_choices: log.warn("No spot choices for %s", moz_instance_type) return 0 to_start = defaultdict(list) active_instance_ids = set(i.id for i in all_instances) # count the number of instances for each image id ami_distribution = defaultdict(int) for instance in all_instances: ami_distribution[instance.image_id] += 1 for region in regions: # Check if spots are enabled in this region for this type region_limit = spot_config.get("limits", {}).get(region, {}).get(moz_instance_type) if not region_limit: log.debug("No spot limits defined for %s in %s, skipping...", moz_instance_type, region) continue # check the limits active_requests = get_spot_requests_for_moztype( region=region, moz_instance_type=moz_instance_type) log.debug("%i active spot requests for %s %s", len(active_requests), region, moz_instance_type) # Filter out requests for instances that don't exist active_requests = [ r for r in active_requests if r.instance_id is not None and r.instance_id in active_instance_ids ] log.debug("%i real active spot requests for %s %s", len(active_requests), region, moz_instance_type) active_count = len(active_requests) can_be_started = region_limit - active_count if can_be_started < 1: log.debug( "Not starting. Active spot request count in %s region " "hit limit of %s. Active count: %s", region, region_limit, active_count) continue to_be_started_latest = min(can_be_started, start_count - started) spot_amis = get_spot_amis(region=region, tags={"moz-type": moz_instance_type}) ami_latest = spot_amis[-1] if len(spot_amis) > 1 and latest_ami_percentage < 100: # get the total number of running instances with both the latest and # prevous ami types, so that we can decide how many of each type to # launch. ami_prev = spot_amis[-2] prev_ami_count = ami_distribution[ami_prev.id] latest_ami_count = ami_distribution[ami_latest.id] ami_prev_to_start, ami_latest_to_start = find_prev_latest_amis_needed( latest_ami_percentage, prev_ami_count, latest_ami_count, to_be_started_latest) to_start[region].append({ "ami": ami_prev, "instances": ami_prev_to_start }) to_start[region].append({ "ami": ami_latest, "instances": ami_latest_to_start }) else: to_start[region].append({ "ami": ami_latest, "instances": to_be_started_latest }) if not to_start: log.debug("Nothing to start for %s", moz_instance_type) return 0 for choice in spot_choices: region = choice.region if region not in to_start: log.debug("Skipping %s for %s", choice, region) continue if not usable_spot_choice(choice): log.debug("Skipping %s for %s - unusable", choice, region) continue for to_start_entry in to_start[region]: need = min(to_start_entry["instances"], start_count - started) if need > 0: log.debug("Need %s of %s in %s", need, moz_instance_type, choice.availability_zone) log.debug("Using %s", choice) launched = do_request_spot_instances( amount=need, region=region, moz_instance_type=moz_instance_type, ami=to_start_entry["ami"], instance_config=instance_config, dryrun=dryrun, spot_choice=choice, slaveset=slaveset, all_instances=all_instances, ) started += launched if started >= start_count: break return started
def request_spot_instances(all_instances, moz_instance_type, start_count, regions, region_priorities, spot_config, dryrun, latest_ami_percentage): started = 0 spot_rules = spot_config.get("rules", {}).get(moz_instance_type) if not spot_rules: log.warn("No spot rules found for %s", moz_instance_type) return 0 instance_config = load_instance_config(moz_instance_type) connections = [get_aws_connection(r) for r in regions] product_description = get_product_description(moz_instance_type) spot_choices = get_spot_choices(connections, spot_rules, product_description) if not spot_choices: log.warn("No spot choices for %s", moz_instance_type) log.warn("%s: market price too expensive in all available regions; spot instances needed: %i", moz_instance_type, start_count) return 0 to_start = defaultdict(list) active_instance_ids = set(i.id for i in all_instances) # count the number of instances for each image id ami_distribution = defaultdict(int) for instance in all_instances: ami_distribution[instance.image_id] += 1 for region in regions: # Check if spots are enabled in this region for this type region_limit = spot_config.get("limits", {}).get(region, {}).get( moz_instance_type) if not region_limit: log.debug("No spot limits defined for %s in %s, skipping...", moz_instance_type, region) continue # check the limits active_requests = get_spot_requests_for_moztype( region=region, moz_instance_type=moz_instance_type) log.debug("%i active spot requests for %s %s", len(active_requests), region, moz_instance_type) # Filter out requests for instances that don't exist active_requests = [r for r in active_requests if r.instance_id is not None and r.instance_id in active_instance_ids] log.debug("%i real active spot requests for %s %s", len(active_requests), region, moz_instance_type) active_count = len(active_requests) can_be_started = region_limit - active_count if can_be_started < 1: log.debug("Not starting. Active spot request count in %s region " "hit limit of %s. Active count: %s", region, region_limit, active_count) continue to_be_started_latest = min(can_be_started, start_count - started) spot_amis = get_spot_amis(region=region, tags={"moz-type": moz_instance_type}) ami_latest = spot_amis[-1] if len(spot_amis) > 1 and latest_ami_percentage < 100: # get the total number of running instances with both the latest and # prevous ami types, so that we can decide how many of each type to # launch. ami_prev = spot_amis[-2] prev_ami_count = ami_distribution[ami_prev.id] latest_ami_count = ami_distribution[ami_latest.id] ami_prev_to_start, ami_latest_to_start = find_prev_latest_amis_needed( latest_ami_percentage, prev_ami_count, latest_ami_count, to_be_started_latest ) to_start[region].append({"ami": ami_prev, "instances": ami_prev_to_start}) to_start[region].append({"ami": ami_latest, "instances": ami_latest_to_start}) else: to_start[region].append({"ami": ami_latest, "instances": to_be_started_latest}) if not to_start: log.debug("Nothing to start for %s", moz_instance_type) return 0 for choice in spot_choices: region = choice.region if region not in to_start: log.debug("Skipping %s for %s", choice, region) continue if not usable_spot_choice(choice): log.debug("Skipping %s for %s - unusable", choice, region) continue for to_start_entry in to_start[region]: need = min(to_start_entry["instances"], start_count - started) if need > 0: log.debug("Need %s of %s in %s", need, moz_instance_type, choice.availability_zone) log.debug("Using %s", choice) launched = do_request_spot_instances( amount=need, region=region, moz_instance_type=moz_instance_type, ami=to_start_entry["ami"], instance_config=instance_config, dryrun=dryrun, spot_choice=choice, all_instances=all_instances, ) started += launched if started >= start_count: break return started