def fulfill_pending(self, asgs, selectors_hash, pending, pods):
"""
selectors_hash - string repr of selectors
pending - KubeResources of all pending pods
pods - list of KubePods that are pending
"""
logger.info(
"========= Scaling for %s ========", selectors_hash)
logger.debug("pending: %s", pending)
groups = utils.get_groups_for_hash(asgs, selectors_hash)
groups = self._prioritize_groups(groups)
for group in groups:
logger.debug("group: %s", group)
# new desired # machines = # running nodes + # machines required to fit jobs that don't
# fit on running nodes. This scaling is conservative but won't create starving
units_needed = self._get_required_capacity(pending, group)
units_needed += self.over_provision
if self.autoscaling_groups.is_timed_out(group):
# if a machine is timed out, it cannot be scaled further
# just account for its current capacity (it may have more
# being launched, but we're being conservative)
unavailable_units = max(
0, units_needed - (group.desired_capacity - group.actual_capacity))
else:
unavailable_units = max(
0, units_needed - (group.max_size - group.actual_capacity))
units_requested = units_needed - unavailable_units
logger.debug("units_needed: %s", units_needed)
logger.debug("units_requested: %s", units_requested)
new_capacity = group.actual_capacity + units_requested
if not self.dry_run:
scaled = group.scale(new_capacity)
if scaled:
notify_scale(group, units_requested,
pods,
self.slack_hook)
else:
logger.info('[Dry run] Would have scaled up to %s', new_capacity)
pending -= units_requested * capacity.get_unit_capacity(group)
logger.debug("remining pending: %s", pending)
if not pending.possible:
break
if pending.possible:
logger.warn('Failed to scale sufficiently.')
notify_failed_to_scale(selectors_hash, pods, hook=self.slack_hook)
def _get_required_capacity(self, requested, group):
"""
returns the number of nodes within an autoscaling group that should
be provisioned to fit the requested amount of KubeResource.
TODO: bin packing would probably be better?
requested - KubeResource
group - AutoScalingGroup
"""
unit_capacity = capacity.get_unit_capacity(group)
return max(
# (peter) should 0.8 be configurable?
int(math.ceil(requested.get(field, 0.0) / unit_capacity.get(field, 1.0)))
for field in ('cpu', 'memory', 'pods')
)
def _get_required_capacity(self, requested, group):
"""
returns the number of nodes within an autoscaling group that should
be provisioned to fit the requested amount of KubeResource.
TODO: bin packing would probably be better?
requested - KubeResource
group - AutoScalingGroup
"""
unit_capacity = capacity.get_unit_capacity(group)
return max(
# (peter) should 0.8 be configurable?
int(math.ceil(requested.get(field, 0.0) / unit_capacity.get(field, 1.0)))
for field in ('cpu', 'memory', 'pods')
)
def fulfill_pending(self, asgs, selectors_hash, pods):
"""
selectors_hash - string repr of selectors
pods - list of KubePods that are pending
"""
logger.info(
"========= Scaling for %s ========", selectors_hash)
logger.debug("pending: %s", pods[:5])
accounted_pods = dict((p, False) for p in pods)
num_unaccounted = len(pods)
groups = utils.get_groups_for_hash(asgs, selectors_hash)
groups = self._prioritize_groups(groups)
async_operations = []
for group in groups:
logger.debug("group: %s", group)
if (self.autoscaling_timeouts.is_timed_out(group) or group.is_timed_out() or group.max_size == group.desired_capacity) \
and not group.unschedulable_nodes:
continue
unit_capacity = capacity.get_unit_capacity(group)
new_instance_resources = []
assigned_pods = []
for pod, acc in accounted_pods.items():
if acc or not (unit_capacity - pod.resources).possible or not group.is_taints_tolerated(pod):
continue
found_fit = False
for i, instance in enumerate(new_instance_resources):
if (instance - pod.resources).possible:
new_instance_resources[i] = instance - pod.resources
assigned_pods[i].append(pod)
found_fit = True
break
if not found_fit:
new_instance_resources.append(
unit_capacity - pod.resources)
assigned_pods.append([pod])
# new desired # machines = # running nodes + # machines required to fit jobs that don't
# fit on running nodes. This scaling is conservative but won't
# create starving
units_needed = len(new_instance_resources)
# The pods may not fit because of resource requests or taints. Don't scale in that case
if units_needed == 0:
continue
units_needed += self.over_provision
if self.autoscaling_timeouts.is_timed_out(group) or group.is_timed_out():
# if a machine is timed out, it cannot be scaled further
# just account for its current capacity (it may have more
# being launched, but we're being conservative)
unavailable_units = max(
0, units_needed - (group.desired_capacity - group.actual_capacity))
else:
unavailable_units = max(
0, units_needed - (group.max_size - group.actual_capacity))
units_requested = units_needed - unavailable_units
logger.debug("units_needed: %s", units_needed)
logger.debug("units_requested: %s", units_requested)
new_capacity = group.actual_capacity + units_requested
if not self.dry_run:
async_operation = group.scale(new_capacity)
async_operations.append(async_operation)
def notify_if_scaled(future):
if future.result():
flat_assigned_pods = []
for instance_pods in assigned_pods:
flat_assigned_pods.extend(instance_pods)
self.notifier.notify_scale(group, units_requested, flat_assigned_pods)
async_operation.add_done_callback(notify_if_scaled)
else:
logger.info(
'[Dry run] Would have scaled up (%s) to %s', group, new_capacity)
for i in range(min(len(assigned_pods), units_requested)):
for pod in assigned_pods[i]:
accounted_pods[pod] = True
num_unaccounted -= 1
logger.debug("remining pending: %s", num_unaccounted)
if not num_unaccounted:
break
if num_unaccounted:
logger.warn('Failed to scale sufficiently.')
self.notifier.notify_failed_to_scale(selectors_hash, pods)
for operation in async_operations:
try:
operation.result()
except CloudError as e:
logger.warn("Error while scaling Scale Set: {}".format(e.message))
except TimeoutError:
logger.warn("Timeout while scaling Scale Set")
def fulfill_pending(self, asgs, selectors_hash, pods):
"""
selectors_hash - string repr of selectors
pods - list of KubePods that are pending
"""
logger.info(
"========= Scaling for %s ========", selectors_hash)
logger.debug("pending: %s", pods[:5])
accounted_pods = dict((p, False) for p in pods)
num_unaccounted = len(pods)
groups = utils.get_groups_for_hash(asgs, selectors_hash)
groups = self._prioritize_groups(groups)
for group in groups:
logger.debug("group: %s", group)
unit_capacity = capacity.get_unit_capacity(group)
new_instance_resources = []
assigned_pods = []
for pod, acc in accounted_pods.items():
if acc or not (unit_capacity - pod.resources).possible:
continue
found_fit = False
for i, instance in enumerate(new_instance_resources):
if (instance - pod.resources).possible:
new_instance_resources[i] = instance - pod.resources
assigned_pods[i].append(pod)
found_fit = True
break
if not found_fit:
new_instance_resources.append(
unit_capacity - pod.resources)
assigned_pods.append([pod])
# new desired # machines = # running nodes + # machines required to fit jobs that don't
# fit on running nodes. This scaling is conservative but won't
# create starving
units_needed = len(new_instance_resources)
units_needed += self.over_provision
if self.autoscaling_timeouts.is_timed_out(group):
# if a machine is timed out, it cannot be scaled further
# just account for its current capacity (it may have more
# being launched, but we're being conservative)
unavailable_units = max(
0, units_needed - (group.desired_capacity - group.actual_capacity))
else:
unavailable_units = max(
0, units_needed - (group.max_size - group.actual_capacity))
units_requested = units_needed - unavailable_units
logger.debug("units_needed: %s", units_needed)
logger.debug("units_requested: %s", units_requested)
new_capacity = group.actual_capacity + units_requested
if not self.dry_run:
scaled = group.scale(new_capacity)
if scaled:
self.notifier.notify_scale(group, units_requested, pods)
else:
logger.info(
'[Dry run] Would have scaled up to %s', new_capacity)
for i in range(min(len(assigned_pods), units_requested)):
for pod in assigned_pods[i]:
accounted_pods[pod] = True
num_unaccounted -= 1
logger.debug("remining pending: %s", num_unaccounted)
if not num_unaccounted:
break
if num_unaccounted:
logger.warn('Failed to scale sufficiently.')
self.notifier.notify_failed_to_scale(selectors_hash, pods)
def fulfill_pending(self, asgs, selectors_hash, pods):
"""
selectors_hash - string repr of selectors
pods - list of KubePods that are pending
"""
logger.info(
"========= Scaling for %s ========", selectors_hash)
logger.debug("pending: %s", pods[:5])
accounted_pods = dict((p, False) for p in pods)
num_unaccounted = len(pods)
groups = utils.get_groups_for_hash(asgs, selectors_hash)
groups = self._prioritize_groups(groups)
for group in groups:
logger.debug("group: %s", group)
unit_capacity = capacity.get_unit_capacity(group)
new_instance_resources = []
assigned_pods = []
for pod, acc in accounted_pods.items():
if acc or not (unit_capacity - pod.resources).possible:
continue
found_fit = False
for i, instance in enumerate(new_instance_resources):
if (instance - pod.resources).possible:
new_instance_resources[i] = instance - pod.resources
assigned_pods[i].append(pod)
found_fit = True
break
if not found_fit:
new_instance_resources.append(unit_capacity - pod.resources)
assigned_pods.append([pod])
# new desired # machines = # running nodes + # machines required to fit jobs that don't
# fit on running nodes. This scaling is conservative but won't create starving
units_needed = len(new_instance_resources)
units_needed += self.over_provision
if self.autoscaling_groups.is_timed_out(group):
# if a machine is timed out, it cannot be scaled further
# just account for its current capacity (it may have more
# being launched, but we're being conservative)
unavailable_units = max(
0, units_needed - (group.desired_capacity - group.actual_capacity))
else:
unavailable_units = max(
0, units_needed - (group.max_size - group.actual_capacity))
units_requested = units_needed - unavailable_units
logger.debug("units_needed: %s", units_needed)
logger.debug("units_requested: %s", units_requested)
new_capacity = group.actual_capacity + units_requested
if not self.dry_run:
scaled = group.scale(new_capacity)
if scaled:
notify_scale(group, units_requested,
pods,
self.slack_hook)
else:
logger.info('[Dry run] Would have scaled up to %s', new_capacity)
for i in range(min(len(assigned_pods), units_requested)):
for pod in assigned_pods[i]:
accounted_pods[pod] = True
num_unaccounted -= 1
logger.debug("remining pending: %s", num_unaccounted)
if not num_unaccounted:
break
if num_unaccounted:
logger.warn('Failed to scale sufficiently.')
notify_failed_to_scale(selectors_hash, pods, hook=self.slack_hook)
def max_resource_capacity(self):
return self.max_size * capacity.get_unit_capacity(self)
def max_resource_capacity(self):
return self.max_size * capacity.get_unit_capacity(self)
def decide_num_instances(self, cluster, pending_pods, asgs,
async_operations):
# scale each node type to reach the new capacity
for selectors_hash in set(pending_pods.keys()):
pods = pending_pods.get(selectors_hash, [])
accounted_pods = dict((p, False) for p in pods)
num_unaccounted = len(pods)
groups = utils.get_groups_for_hash(asgs, selectors_hash)
groups = cluster._prioritize_groups(groups)
for group in groups:
if (cluster.autoscaling_timeouts.is_timed_out(
group) or group.is_timed_out() or group.max_size == group.desired_capacity) \
and not group.unschedulable_nodes:
continue
unit_capacity = capacity.get_unit_capacity(group)
new_instance_resources = []
assigned_pods = []
for pod, acc in accounted_pods.items():
if acc or not (
unit_capacity - pod.resources
).possible or not group.is_taints_tolerated(pod):
continue
found_fit = False
for i, instance in enumerate(new_instance_resources):
if (instance - pod.resources).possible:
new_instance_resources[
i] = instance - pod.resources
assigned_pods[i].append(pod)
found_fit = True
break
if not found_fit:
new_instance_resources.append(unit_capacity -
pod.resources)
assigned_pods.append([pod])
# new desired # machines = # running nodes + # machines required to fit jobs that don't
# fit on running nodes. This scaling is conservative but won't
# create starving
units_needed = len(new_instance_resources)
# The pods may not fit because of resource requests or taints. Don't scale in that case
if units_needed == 0:
continue
units_needed += cluster.over_provision
if cluster.autoscaling_timeouts.is_timed_out(
group) or group.is_timed_out():
# if a machine is timed out, it cannot be scaled further
# just account for its current capacity (it may have more
# being launched, but we're being conservative)
unavailable_units = max(
0, units_needed -
(group.desired_capacity - group.actual_capacity))
else:
unavailable_units = max(
0, units_needed -
(group.max_size - group.actual_capacity))
units_requested = units_needed - unavailable_units
new_capacity = group.actual_capacity + units_requested
async_operations.append(
self.create_async_operation(cluster, group, assigned_pods,
new_capacity, units_requested))
def decide_num_instances(self, cluster, pending_pods, asgs,
async_operations):
curr_time = time.time()
if (
self.start_time - curr_time
) / 3600 > self.num_hours: # If started a new hour, start new count
self.num_hours += 1
self.spent_this_hour = 0
# scale each node type to reach the new capacity
for selectors_hash in set(pending_pods.keys()):
pods = pending_pods.get(selectors_hash, [])
accounted_pods = dict((p, False) for p in pods)
num_unaccounted = len(pods)
groups = utils.get_groups_for_hash(asgs, selectors_hash)
groups = cluster._prioritize_groups(groups)
for group in groups:
if (cluster.autoscaling_timeouts.is_timed_out(
group) or group.is_timed_out() or group.max_size == group.desired_capacity) \
and not group.unschedulable_nodes:
continue
unit_capacity = capacity.get_unit_capacity(group)
new_instance_resources = []
assigned_pods = []
for pod, acc in accounted_pods.items():
if acc or not (
unit_capacity - pod.resources
).possible or not group.is_taints_tolerated(pod):
continue
found_fit = False
for i, instance in enumerate(new_instance_resources):
if (instance - pod.resources).possible:
new_instance_resources[
i] = instance - pod.resources
assigned_pods[i].append(pod)
found_fit = True
break
if not found_fit:
new_instance_resources.append(unit_capacity -
pod.resources)
assigned_pods.append([pod])
# new desired # machines = # running nodes + # machines required to fit jobs that don't
# fit on running nodes. This scaling is conservative but won't
# create starving
units_needed = len(new_instance_resources)
# The pods may not fit because of resource requests or taints. Don't scale in that case
if units_needed == 0:
continue
units_needed += cluster.over_provision
if cluster.autoscaling_timeouts.is_timed_out(
group) or group.is_timed_out():
# if a machine is timed out, it cannot be scaled further
# just account for its current capacity (it may have more
# being launched, but we're being conservative)
unavailable_units = max(
0, units_needed -
(group.desired_capacity - group.actual_capacity))
else:
unavailable_units = max(
0, units_needed -
(group.max_size - group.actual_capacity))
units_requested = units_needed - unavailable_units
avg_hours_used_per_instance = 0.25
if self.num_instances_tracked > 0:
avg_hours_used_per_instance = self.num_seconds_instances_used / self.num_instances_tracked / 3600
# If we've used 75% of budget stop provisioning. No guarantees we wont go over budget,
# but conservative enough for most uses.
for i in range(units_requested):
predicted_cost = self.spent_this_hour + (
i +
1) * avg_hours_used_per_instance * self.costs_per_hour[
group.instance_type]['cost-per-hour']
if predicted_cost > self.max_cost_per_hour * 0.75:
units_requested = i + 1
break
new_capacity = group.actual_capacity + units_requested
async_operations.append(
self.create_async_operation(cluster, group, assigned_pods,
new_capacity, units_requested))