def aws_resume_instances(all_instances, moz_instance_type, start_count,
regions, region_priorities, dryrun, slaveset):
"""Create up to `start_count` on-demand instances"""
start_count_per_region = distribute_in_region(start_count, regions,
region_priorities)
started = 0
instance_config = load_instance_config(moz_instance_type)
for region, count in start_count_per_region.iteritems():
# TODO: check region limits
ami = get_ami(region=region, moz_instance_type=moz_instance_type)
for _ in range(count):
try:
r = do_request_instance(
region=region,
moz_instance_type=moz_instance_type,
price=None,
availability_zone=None,
ami=ami,
instance_config=instance_config,
instance_type=instance_config[region]["instance_type"],
slaveset=slaveset,
is_spot=False,
dryrun=dryrun,
all_instances=all_instances)
if r:
started += 1
except EC2ResponseError, e:
# TODO: Handle e.code
log.warn("On-demand failure: %s; giving up", e.code)
log.warn("Cannot start", exc_info=True)
break
except Exception:
log.warn("Cannot start", exc_info=True)
def aws_resume_instances(all_instances, moz_instance_type, start_count,
regions, region_priorities, dryrun):
"""Create up to `start_count` on-demand instances"""
start_count_per_region = distribute_in_region(start_count, regions,
region_priorities)
started = 0
instance_config = load_instance_config(moz_instance_type)
for region, count in start_count_per_region.iteritems():
# TODO: check region limits
ami = get_ami(region=region, moz_instance_type=moz_instance_type)
for _ in range(count):
try:
r = do_request_instance(
region=region,
moz_instance_type=moz_instance_type,
price=None, availability_zone=None,
ami=ami, instance_config=instance_config,
instance_type=instance_config[region]["instance_type"],
is_spot=False, dryrun=dryrun,
all_instances=all_instances)
if r:
started += 1
except EC2ResponseError, e:
# TODO: Handle e.code
log.warn("On-demand failure: %s; giving up", e.code)
log.warn("Cannot start", exc_info=True)
break
except Exception:
log.warn("Cannot start", exc_info=True)
def request_spot_instances(all_instances, moz_instance_type, start_count,
regions, region_priorities, spot_config, dryrun,
slaveset, latest_ami_percentage):
started = 0
spot_rules = spot_config.get("rules", {}).get(moz_instance_type)
if not spot_rules:
log.warn("No spot rules found for %s", moz_instance_type)
return 0
instance_config = load_instance_config(moz_instance_type)
connections = [get_aws_connection(r) for r in regions]
spot_choices = get_spot_choices(connections, spot_rules,
"Linux/UNIX (Amazon VPC)")
if not spot_choices:
log.warn("No spot choices for %s", moz_instance_type)
return 0
to_start = defaultdict(list)
active_instance_ids = set(i.id for i in all_instances)
# count the number of instances for each image id
ami_distribution = defaultdict(int)
for instance in all_instances:
ami_distribution[instance.image_id] += 1
for region in regions:
# Check if spots are enabled in this region for this type
region_limit = spot_config.get("limits",
{}).get(region,
{}).get(moz_instance_type)
if not region_limit:
log.debug("No spot limits defined for %s in %s, skipping...",
moz_instance_type, region)
continue
# check the limits
active_requests = get_spot_requests_for_moztype(
region=region, moz_instance_type=moz_instance_type)
log.debug("%i active spot requests for %s %s", len(active_requests),
region, moz_instance_type)
# Filter out requests for instances that don't exist
active_requests = [
r for r in active_requests if r.instance_id is not None
and r.instance_id in active_instance_ids
]
log.debug("%i real active spot requests for %s %s",
len(active_requests), region, moz_instance_type)
active_count = len(active_requests)
can_be_started = region_limit - active_count
if can_be_started < 1:
log.debug(
"Not starting. Active spot request count in %s region "
"hit limit of %s. Active count: %s", region, region_limit,
active_count)
continue
to_be_started_latest = min(can_be_started, start_count - started)
spot_amis = get_spot_amis(region=region,
tags={"moz-type": moz_instance_type})
ami_latest = spot_amis[-1]
if len(spot_amis) > 1 and latest_ami_percentage < 100:
# get the total number of running instances with both the latest and
# prevous ami types, so that we can decide how many of each type to
# launch.
ami_prev = spot_amis[-2]
prev_ami_count = ami_distribution[ami_prev.id]
latest_ami_count = ami_distribution[ami_latest.id]
ami_prev_to_start, ami_latest_to_start = find_prev_latest_amis_needed(
latest_ami_percentage, prev_ami_count, latest_ami_count,
to_be_started_latest)
to_start[region].append({
"ami": ami_prev,
"instances": ami_prev_to_start
})
to_start[region].append({
"ami": ami_latest,
"instances": ami_latest_to_start
})
else:
to_start[region].append({
"ami": ami_latest,
"instances": to_be_started_latest
})
if not to_start:
log.debug("Nothing to start for %s", moz_instance_type)
return 0
for choice in spot_choices:
region = choice.region
if region not in to_start:
log.debug("Skipping %s for %s", choice, region)
continue
if not usable_spot_choice(choice):
log.debug("Skipping %s for %s - unusable", choice, region)
continue
for to_start_entry in to_start[region]:
need = min(to_start_entry["instances"], start_count - started)
if need > 0:
log.debug("Need %s of %s in %s", need, moz_instance_type,
choice.availability_zone)
log.debug("Using %s", choice)
launched = do_request_spot_instances(
amount=need,
region=region,
moz_instance_type=moz_instance_type,
ami=to_start_entry["ami"],
instance_config=instance_config,
dryrun=dryrun,
spot_choice=choice,
slaveset=slaveset,
all_instances=all_instances,
)
started += launched
if started >= start_count:
break
return started
def request_spot_instances(all_instances, moz_instance_type, start_count,
regions, region_priorities, spot_config, dryrun,
latest_ami_percentage):
started = 0
spot_rules = spot_config.get("rules", {}).get(moz_instance_type)
if not spot_rules:
log.warn("No spot rules found for %s", moz_instance_type)
return 0
instance_config = load_instance_config(moz_instance_type)
connections = [get_aws_connection(r) for r in regions]
product_description = get_product_description(moz_instance_type)
spot_choices = get_spot_choices(connections, spot_rules, product_description)
if not spot_choices:
log.warn("No spot choices for %s", moz_instance_type)
log.warn("%s: market price too expensive in all available regions; spot instances needed: %i",
moz_instance_type, start_count)
return 0
to_start = defaultdict(list)
active_instance_ids = set(i.id for i in all_instances)
# count the number of instances for each image id
ami_distribution = defaultdict(int)
for instance in all_instances:
ami_distribution[instance.image_id] += 1
for region in regions:
# Check if spots are enabled in this region for this type
region_limit = spot_config.get("limits", {}).get(region, {}).get(
moz_instance_type)
if not region_limit:
log.debug("No spot limits defined for %s in %s, skipping...",
moz_instance_type, region)
continue
# check the limits
active_requests = get_spot_requests_for_moztype(
region=region, moz_instance_type=moz_instance_type)
log.debug("%i active spot requests for %s %s", len(active_requests),
region, moz_instance_type)
# Filter out requests for instances that don't exist
active_requests = [r for r in active_requests if r.instance_id is not
None and r.instance_id in active_instance_ids]
log.debug("%i real active spot requests for %s %s",
len(active_requests), region, moz_instance_type)
active_count = len(active_requests)
can_be_started = region_limit - active_count
if can_be_started < 1:
log.debug("Not starting. Active spot request count in %s region "
"hit limit of %s. Active count: %s", region,
region_limit, active_count)
continue
to_be_started_latest = min(can_be_started, start_count - started)
spot_amis = get_spot_amis(region=region, tags={"moz-type": moz_instance_type})
ami_latest = spot_amis[-1]
if len(spot_amis) > 1 and latest_ami_percentage < 100:
# get the total number of running instances with both the latest and
# prevous ami types, so that we can decide how many of each type to
# launch.
ami_prev = spot_amis[-2]
prev_ami_count = ami_distribution[ami_prev.id]
latest_ami_count = ami_distribution[ami_latest.id]
ami_prev_to_start, ami_latest_to_start = find_prev_latest_amis_needed(
latest_ami_percentage,
prev_ami_count,
latest_ami_count,
to_be_started_latest
)
to_start[region].append({"ami": ami_prev, "instances": ami_prev_to_start})
to_start[region].append({"ami": ami_latest, "instances": ami_latest_to_start})
else:
to_start[region].append({"ami": ami_latest, "instances": to_be_started_latest})
if not to_start:
log.debug("Nothing to start for %s", moz_instance_type)
return 0
for choice in spot_choices:
region = choice.region
if region not in to_start:
log.debug("Skipping %s for %s", choice, region)
continue
if not usable_spot_choice(choice):
log.debug("Skipping %s for %s - unusable", choice, region)
continue
for to_start_entry in to_start[region]:
need = min(to_start_entry["instances"], start_count - started)
if need > 0:
log.debug("Need %s of %s in %s", need, moz_instance_type,
choice.availability_zone)
log.debug("Using %s", choice)
launched = do_request_spot_instances(
amount=need,
region=region,
moz_instance_type=moz_instance_type,
ami=to_start_entry["ami"],
instance_config=instance_config, dryrun=dryrun,
spot_choice=choice,
all_instances=all_instances,
)
started += launched
if started >= start_count:
break
return started