def get_pods_to_schedule(self, pods): """ given a list of KubePod objects, return a map of (selectors hash -> pods) to be scheduled """ pending_unassigned_pods = [ p for p in pods if p.is_pending_unassigned_and_scaleworthy(self.scale_label) ] # we only consider a pod to be schedulable if it's pending and # unassigned and feasible pods_to_schedule = {} for pod in pending_unassigned_pods: if capacity.is_possible(pod): pods_to_schedule.setdefault( utils.selectors_to_hash(pod.selectors), []).append(pod) else: recommended_capacity = capacity.max_capacity_for_selectors( pod.selectors) logger.warn( "Pending pod %s cannot fit %s. " "Please check that requested resource amount is " "consistent with node selectors (recommended max: %s). " "Scheduling skipped." % (pod.name, pod.selectors, recommended_capacity)) self.notifier.notify_invalid_pod_capacity( pod, recommended_capacity) return pods_to_schedule
def get_pods_to_schedule(self, pods): """ given a list of KubePod objects, return a map of (selectors hash -> pods) to be scheduled """ pending_unassigned_pods = [ p for p in pods if p.status == KubePodStatus.PENDING and (not p.node_name) ] # we only consider a pod to be schedulable if it's pending and # unassigned and feasible pods_to_schedule = {} now = datetime.datetime.now(pytz.utc) for pod in pending_unassigned_pods: age = (now - pod.creation_time).total_seconds() self.stats.histogram('autoscaler.scaling_loop.pending_pod_age', age) if capacity.is_possible(pod): pods_to_schedule.setdefault( utils.selectors_to_hash(pod.selectors), []).append(pod) else: recommended_capacity = capacity.max_capacity_for_selectors( pod.selectors, pod.resources) logger.warn( "Pending pod %s cannot fit %s. " "Please check that requested resource amount is " "consistent with node selectors (recommended max: %s). " "Scheduling skipped." % (pod.name, pod.selectors, recommended_capacity)) self.notifier.notify_invalid_pod_capacity( pod, recommended_capacity) return pods_to_schedule
def test_scale_up(self): pod = KubePod(pykube.Pod(self.api, self.dummy_pod)) selectors_hash = utils.selectors_to_hash(pod.selectors) asgs = self.cluster.autoscaling_groups.get_all_groups([]) self.cluster.fulfill_pending(asgs, selectors_hash, [pod]) response = self.asg_client.describe_auto_scaling_groups() self.assertEqual(len(response['AutoScalingGroups']), 1) self.assertGreater(response['AutoScalingGroups'][0]['DesiredCapacity'], 0)
def test_scale_up(self): pod = KubePod(pykube.Pod(self.api, self.dummy_pod)) selectors_hash = utils.selectors_to_hash(pod.selectors) asgs = self.cluster.autoscaling_groups.get_all_groups([]) self.cluster.fulfill_pending(asgs, selectors_hash, [pod]) response = self.asg_client.describe_auto_scaling_groups() self.assertEqual(len(response['AutoScalingGroups']), 1) self.assertGreater(response['AutoScalingGroups'][0]['DesiredCapacity'], 0)
def test_scale_up_notification(self): big_pod_spec = copy.deepcopy(self.dummy_pod) for container in big_pod_spec['spec']['containers']: container['resources']['requests']['cpu'] = '100' pod = KubePod(pykube.Pod(self.api, self.dummy_pod)) big_pod = KubePod(pykube.Pod(self.api, big_pod_spec)) selectors_hash = utils.selectors_to_hash(pod.selectors) asgs = self.cluster.autoscaling_groups.get_all_groups([]) self.cluster.fulfill_pending(asgs, selectors_hash, [pod, big_pod]) self.cluster.notifier.notify_scale.assert_called_with(mock.ANY, mock.ANY, [pod])
def test_scale_up_selector(self): self.dummy_pod['spec']['nodeSelector'] = {'aws/type': 'm4.large'} pod = KubePod(pykube.Pod(self.api, self.dummy_pod)) selectors_hash = utils.selectors_to_hash(pod.selectors) asgs = self.cluster.autoscaling_groups.get_all_groups([]) self.cluster.fulfill_pending(asgs, selectors_hash, [pod]) response = self.asg_client.describe_auto_scaling_groups() self.assertEqual(len(response['AutoScalingGroups']), 1) self.assertEqual(response['AutoScalingGroups'][0]['DesiredCapacity'], 0)
def test_scale_up_selector(self): self.dummy_pod['spec']['nodeSelector'] = { 'aws/type': 'm4.large' } pod = KubePod(pykube.Pod(self.api, self.dummy_pod)) selectors_hash = utils.selectors_to_hash(pod.selectors) asgs = self.cluster.autoscaling_groups.get_all_groups([]) self.cluster.fulfill_pending(asgs, selectors_hash, [pod]) response = self.asg_client.describe_auto_scaling_groups() self.assertEqual(len(response['AutoScalingGroups']), 1) self.assertEqual(response['AutoScalingGroups'][0]['DesiredCapacity'], 0)
def scale(self, pods_to_schedule, all_nodes, asgs, running_insts_map): """ scale up logic """ # TODO: generalize to azure self.autoscaling_timeouts.refresh_timeouts( [asg for asg in asgs if asg.provider == 'aws'], dry_run=self.dry_run) cached_live_nodes = [] for node in all_nodes: # either we know the physical node behind it and know it's alive # or we don't know it and assume it's alive if (node.instance_id and node.instance_id in running_insts_map) \ or (not node.is_managed()): cached_live_nodes.append(node) # selectors -> pending KubePods pending_pods = {} # for each pending & unassigned job, try to fit them on current machines or count requested # resources towards future machines for selectors_hash, pods in pods_to_schedule.items(): for pod in pods: fitting = None for node in cached_live_nodes: if node.unschedulable: continue if node.is_match(pod) and node.can_fit(pod.resources): fitting = node break if fitting is None: # because a pod may be able to fit in multiple groups # pick a group now selectors = dict(pod.selectors) pending_pods.setdefault(utils.selectors_to_hash(selectors), []).append(pod) logger.info( "{pod} is pending ({selectors_hash})".format( pod=pod, selectors_hash=selectors_hash)) else: fitting.count_pod(pod) logger.info("{pod} fits on {node}".format(pod=pod, node=fitting)) # scale each node type to reach the new capacity for selectors_hash in set(pending_pods.keys()): self.fulfill_pending(asgs, selectors_hash, pending_pods.get(selectors_hash, []))
def test_timed_out_group(self): with mock.patch('autoscaler.autoscaling_groups.AutoScalingGroup.is_timed_out') as is_timed_out: with mock.patch('autoscaler.autoscaling_groups.AutoScalingGroup.scale') as scale: is_timed_out.return_value = True scale.return_value = utils.CompletedFuture(None) pod = KubePod(pykube.Pod(self.api, self.dummy_pod)) selectors_hash = utils.selectors_to_hash(pod.selectors) asgs = self.cluster.autoscaling_groups.get_all_groups([]) self.cluster.fulfill_pending(asgs, selectors_hash, [pod]) scale.assert_not_called() response = self.asg_client.describe_auto_scaling_groups() self.assertEqual(len(response['AutoScalingGroups']), 1) self.assertEqual(response['AutoScalingGroups'][0]['DesiredCapacity'], 0)
def test_scale_up(self): pod = KubePod(pykube.Pod(self.api, self.dummy_pod)) selectors_hash = utils.selectors_to_hash(pod.selectors) asgs = self.cluster.autoscaling_groups.get_all_groups([]) self.cluster.fulfill_pending(asgs, selectors_hash, [pod]) response = self.asg_client.describe_auto_scaling_groups() self.assertEqual(len(response['AutoScalingGroups']), 2) big_gpu_asg, small_gpu_asg = {}, {} if (response['AutoScalingGroups'][0]['AutoScalingGroupName'] == 'dummy-asg-small-gpu'): small_gpu_asg = response['AutoScalingGroups'][0] big_gpu_asg = response['AutoScalingGroups'][1] else: small_gpu_asg = response['AutoScalingGroups'][1] big_gpu_asg = response['AutoScalingGroups'][0] self.assertGreater(big_gpu_asg['DesiredCapacity'], 0) self.assertEqual(small_gpu_asg['DesiredCapacity'], 0)
def get_pods_to_schedule(self, pods): """ given a list of KubePod objects, return a map of (selectors hash -> pods) to be scheduled """ pending_unassigned_pods = [ p for p in pods if p.status == KubePodStatus.PENDING and (not p.node_name) ] # we only consider a pod to be schedulable if it's pending and unassigned and feasible pods_to_schedule = {} for pod in pending_unassigned_pods: if capacity.is_possible(pod): pods_to_schedule.setdefault(utils.selectors_to_hash(pod.selectors), []).append(pod) else: logger.warn( "Pending pod %s cannot fit %s. Ignored" % (pod.name, pod.selectors)) return pods_to_schedule
def get_pods_to_schedule(self, pods): """ given a list of KubePod objects, return a map of (selectors hash -> pods) to be scheduled """ pending_unassigned_pods = [ p for p in pods if p.status == KubePodStatus.PENDING and (not p.node_name) ] # we only consider a pod to be schedulable if it's pending and unassigned and feasible pods_to_schedule = {} for pod in pending_unassigned_pods: if capacity.is_possible(pod): pods_to_schedule.setdefault(utils.selectors_to_hash(pod.selectors), []).append(pod) else: logger.warn( "Pending pod %s cannot fit %s. " "Please check that requested resource amount is " "consistent with node selectors. Scheduling skipped." % (pod.name, pod.selectors)) return pods_to_schedule
def get_pods_to_schedule(self, pods): """ given a list of KubePod objects, return a map of (selectors hash -> pods) to be scheduled """ pending_unassigned_pods = [ p for p in pods if p.status == KubePodStatus.PENDING and (not p.node_name) ] # we only consider a pod to be schedulable if it's pending and # unassigned and feasible pods_to_schedule = {} for pod in pending_unassigned_pods: if capacity.is_possible(pod, self.cs_instance_type): pods_to_schedule.setdefault( utils.selectors_to_hash(pod.selectors), []).append(pod) else: logger.warn("Pending pod %s cannot fit %s. " "Please check that requested resource amount is " "consistent with node size." "Scheduling skipped." % (pod.name, pod.selectors)) return pods_to_schedule
def get_node_state(self, node, asg, node_pods, pods_to_schedule, running_insts_map, idle_selector_hash): """ returns the ClusterNodeState for the given node params: node - KubeNode object asg - AutoScalingGroup object that this node belongs in. can be None. node_pods - list of KubePods assigned to this node pods_to_schedule - list of all pending pods running_inst_map - map of all (instance_id -> ec2.Instance object) idle_selector_hash - current map of idle nodes by type. may be modified """ pending_list = [] for pods in pods_to_schedule.values(): for pod in pods: # a pod is considered schedulable onto this node if all the # node selectors match # AND it doesn't use pod affinity (which we don't support yet) if (node.is_match(pod) and 'scheduler.alpha.kubernetes.io/affinity' not in pod.annotations): pending_list.append(pod) # we consider a node to be busy if it's running any non-DaemonSet pods # TODO: we can be a bit more aggressive in killing pods that are # replicated busy_list = [p for p in node_pods if not p.is_mirrored()] undrainable_list = [p for p in node_pods if not p.is_drainable()] utilization = sum((p.resources for p in busy_list), KubeResource()) under_utilized = (self.UTIL_THRESHOLD * node.capacity - utilization).possible drainable = not undrainable_list maybe_inst = running_insts_map.get(node.instance_id) if maybe_inst: age = (datetime.datetime.now(maybe_inst.launch_time.tzinfo) - maybe_inst.launch_time).seconds logger.warn('AGE: %s', age) launch_hour_offset = age % 3600 else: age = None instance_type = utils.selectors_to_hash( asg.selectors) if asg else node.instance_type type_spare_capacity = (instance_type and self.type_idle_threshold and idle_selector_hash[instance_type] < self.TYPE_IDLE_COUNT) if maybe_inst is None: return ClusterNodeState.INSTANCE_TERMINATED if node.is_detached(): return ClusterNodeState.DETACHED if node.is_dead(): return ClusterNodeState.DEAD if asg and len(asg.nodes) <= asg.min_size: return ClusterNodeState.ASG_MIN_SIZE if busy_list and not under_utilized: if node.unschedulable: return ClusterNodeState.BUSY_UNSCHEDULABLE return ClusterNodeState.BUSY if pending_list and not node.unschedulable: # logger.warn('PENDING: %s', pending_list) return ClusterNodeState.POD_PENDING if launch_hour_offset < self.LAUNCH_HOUR_THRESHOLD[node.provider] and not node.unschedulable: return ClusterNodeState.LAUNCH_HR_GRACE_PERIOD # elif node.provider == 'azure': # disabling scale down in azure for now while we ramp up # TODO: remove once azure is bootstrapped # state = ClusterNodeState.GRACE_PERIOD if (not type_spare_capacity and age <= self.idle_threshold) and not node.unschedulable: # there is already an instance of this type sitting idle # so we use the regular idle threshold for the grace period return ClusterNodeState.GRACE_PERIOD if (type_spare_capacity and age <= self.type_idle_threshold) and not node.unschedulable: # we don't have an instance of this type yet! # use the type idle threshold for the grace period # and mark the type as seen idle_selector_hash[instance_type] += 1 return ClusterNodeState.TYPE_GRACE_PERIOD if under_utilized and (busy_list or not node.unschedulable): # nodes that are under utilized (but not completely idle) # have their own states to tell if we should drain them # for better binpacking or not if drainable: return ClusterNodeState.UNDER_UTILIZED_DRAINABLE return ClusterNodeState.UNDER_UTILIZED_UNDRAINABLE if node.unschedulable: return ClusterNodeState.IDLE_UNSCHEDULABLE return ClusterNodeState.IDLE_SCHEDULABLE
def __str__(self): return "{}: {} ({})".format(self.name, self.instance_id, utils.selectors_to_hash(self.selectors))
def __str__(self): return 'AzureVirtualScaleSet({name}, {selectors_hash})'.format(name=self.name, selectors_hash=utils.selectors_to_hash(self.selectors))
def __str__(self): return "{}: {} ({})".format(self.name, self.instance_id, utils.selectors_to_hash(self.selectors))
def get_node_state(self, node, asg, node_pods, pods_to_schedule, running_insts_map, idle_selector_hash): """ returns the ClusterNodeState for the given node params: node - KubeNode object asg - AutoScalingGroup object that this node belongs in. can be None. node_pods - list of KubePods assigned to this node pods_to_schedule - list of all pending pods running_inst_map - map of all (instance_id -> ec2.Instance object) idle_selector_hash - current map of idle nodes by type. may be modified. """ pending_list = [] for pods in pods_to_schedule.values(): for pod in pods: if node.is_match(pod): pending_list.append(pod) # we consider a node to be busy if it's running any non-DaemonSet pods # TODO: we can be a bit more aggressive in killing pods that are # replicated busy_list = [p for p in node_pods if not p.is_mirrored()] undrainable_list = [p for p in node_pods if not p.is_drainable()] utilization = sum((p.resources for p in busy_list), KubeResource()) under_utilized = (self.UTIL_THRESHOLD * node.capacity - utilization).possible drainable = not undrainable_list maybe_inst = running_insts_map.get(node.instance_id) instance_type = utils.selectors_to_hash(asg.selectors) if asg else None if maybe_inst: age = (datetime.datetime.now(maybe_inst.launch_time.tzinfo) - maybe_inst.launch_time).seconds else: age = None instance_type = utils.selectors_to_hash(asg.selectors) if asg else node.instance_type if maybe_inst is None: state = ClusterNodeState.INSTANCE_TERMINATED elif asg and len(asg.nodes) <= asg.min_size: state = ClusterNodeState.ASG_MIN_SIZE elif busy_list and not under_utilized: if node.unschedulable: state = ClusterNodeState.BUSY_UNSCHEDULABLE else: state = ClusterNodeState.BUSY elif pending_list and not node.unschedulable: state = ClusterNodeState.POD_PENDING elif ((not self.type_idle_threshold or idle_selector_hash[instance_type] >= self.TYPE_IDLE_COUNT) and age <= self.idle_threshold) and not node.unschedulable: # there is already an instance of this type sitting idle # so we use the regular idle threshold for the grace period state = ClusterNodeState.GRACE_PERIOD elif (instance_type and idle_selector_hash[instance_type] < self.TYPE_IDLE_COUNT and age <= self.type_idle_threshold) and not node.unschedulable: # we don't have an instance of this type yet! # use the type idle threshold for the grace period # and mark the type as seen idle_selector_hash[instance_type] += 1 state = ClusterNodeState.TYPE_GRACE_PERIOD elif under_utilized and not node.unschedulable: if drainable: state = ClusterNodeState.UNDER_UTILIZED_DRAINABLE else: state = ClusterNodeState.UNDER_UTILIZED_UNDRAINABLE else: if node.unschedulable: state = ClusterNodeState.IDLE_UNSCHEDULABLE else: state = ClusterNodeState.IDLE_SCHEDULABLE return state
def __str__(self): return 'AutoScalingGroup({name}, {selectors_hash})'.format( name=self.name, selectors_hash=utils.selectors_to_hash(self.selectors))
def __str__(self): return 'AutoScalingGroup({name}, {selectors_hash})'.format(name=self.name, selectors_hash=utils.selectors_to_hash(self.selectors))
def get_node_state(self, node, asg, node_pods, pods_to_schedule, running_insts_map, idle_selector_hash): """ returns the ClusterNodeState for the given node params: node - KubeNode object asg - AutoScalingGroup object that this node belongs in. can be None. node_pods - list of KubePods assigned to this node pods_to_schedule - list of all pending pods running_inst_map - map of all (instance_id -> ec2.Instance object) idle_selector_hash - current map of idle nodes by type. may be modified. """ pending_list = [] for pods in pods_to_schedule.itervalues(): for pod in pods: if node.is_match(pod): pending_list.append(pod) # we consider a node to be busy if it's running any non-DaemonSet pods # TODO: we can be a bit more aggressive in killing pods that are # replicated busy_list = [p for p in node_pods if not p.is_mirrored()] maybe_inst = running_insts_map.get(node.instance_id) instance_type = utils.selectors_to_hash(asg.selectors) if asg else None if maybe_inst: age = (datetime.datetime.now(maybe_inst.launch_time.tzinfo) - maybe_inst.launch_time).seconds else: age = None instance_type = utils.selectors_to_hash(asg.selectors) if asg else node.instance_type if maybe_inst is None: state = ClusterNodeState.INSTANCE_TERMINATED elif asg and len(asg.nodes) <= asg.min_size: state = ClusterNodeState.ASG_MIN_SIZE elif busy_list: if node.unschedulable: state = ClusterNodeState.BUSY_UNSCHEDULABLE else: state = ClusterNodeState.BUSY elif pending_list: state = ClusterNodeState.POD_PENDING elif (idle_selector_hash[instance_type] >= self.TYPE_IDLE_COUNT and age <= self.idle_threshold): # there is already an instance of this type sitting idle # so we use the regular idle threshold for the grace period state = ClusterNodeState.GRACE_PERIOD elif (instance_type and idle_selector_hash[instance_type] < self.TYPE_IDLE_COUNT and age <= self.type_idle_threshold): # we don't have an instance of this type yet! # use the type idle threshold for the grace period # and mark the type as seen idle_selector_hash[instance_type] += 1 state = ClusterNodeState.TYPE_GRACE_PERIOD else: if node.unschedulable: state = ClusterNodeState.IDLE_UNSCHEDULABLE else: state = ClusterNodeState.IDLE_SCHEDULABLE return state