def update_instance_info(self, nodes_to_check: List[NodeDescriptor], timeout: int = 600) -> List[NodeDescriptor]: # group nodes with the same provider checked_nodes = list() for _, nodes in sorted_groupby( nodes_to_check, lambda x: x.configuration.provider.provider_config_id).items(): nodes = list(nodes) provider_config = nodes[0].configuration.provider task_check_name = f"Checking nodes " \ f"`{', '.join(sorted([node.nickname for node in nodes]))}`" events = self.execute_check_template(nodes, provider_config, task_check_name, quiet=True) if not events: raise Exception('Update task returned no events to process') # remove instances... task_event = self._get_named_event(events, task_check_name) instances_status = { i['instance_id']: i for i in task_event['event_data']['res']['instances'] } for node in nodes: if node.cloud_instance_id not in instances_status: logger.error(f"Node `{node.node_id}` with invalid aws " f"instance id: {node.cloud_instance_id}") continue instance = instances_status[node.cloud_instance_id] if instance['state']['name'] == 'running': node.status = NodeStatus.STARTED elif instance['state']['name'] == 'stopped': node.status = NodeStatus.PAUSED elif instance['state']['name'] == 'terminated': node.status = NodeStatus.STOPPED else: node.status = NodeStatus.UNKNOWN if node.status != NodeStatus.STOPPED: node.ip = instance['public_ip_address'] \ if 'public_ip_address' in instance else None node.cloud_instance_id = instance['instance_id'] node.extra = { 'instance_id': instance['instance_id'], 'private_ip': instance['private_ip_address'], 'dns': instance['public_dns_name'], 'private_dns': instance['private_dns_name'], 'architecture': instance['architecture'], 'instance_tags': instance['tags'], 'vpc_id': instance['vpc_id'], 'subnet_id': instance['subnet_id'] } checked_nodes.append(node) return checked_nodes
def stop_instances(self, nodes_to_stop: List[NodeDescriptor], timeout: int = 180) -> List[NodeDescriptor]: state = 'absent' stopped_nodes: List[NodeDescriptor] = [] # group nodes with the same provider for _, nodes in sorted_groupby( nodes_to_stop, lambda x: x.configuration.provider.provider_config_id).items(): provider_config = nodes[0].configuration.provider task_check_name = f"Stopping nodes " \ f"{', '.join(sorted([node.nickname for node in nodes]))}" events = self.execute_common_template(nodes, provider_config, task_check_name, state, wait='no') if not events: raise Exception('Stop task returned no events to process') # remove instances... task_event = self._get_named_event(events, task_check_name) removed_instance_ids = task_event['event_data']['res'][ 'instance_ids'] successfully_stopped = [] for node in nodes: if node.cloud_instance_id in removed_instance_ids: node.status = NodeStatus.STOPPED node.ip = None successfully_stopped.append(node) stopped_nodes += successfully_stopped return stopped_nodes
def _tag_instances( self, node_list: List[NodeDescriptor]) -> List[NodeDescriptor]: task_check_name = 'Tagging instances' tagged_instances = [] for _, nodes in sorted_groupby( node_list, lambda x: x.configuration.provider.provider_config_id).items(): nodes = list(nodes) envvars = self.create_envvars(nodes[0].configuration.provider) names = [{ 'id': node.cloud_instance_id, 'name': f"{node.nickname.replace(' ', '')}-{node.node_id[:8]}" } for node in nodes] ec2_vals = { 'task_name': task_check_name, 'names': names, } rendered_template = self.jinjaenv.get_template( 'ec2_tag_instances.j2').render(ec2_vals) result = self._run_template('ec2_tag_instances.yml', rendered_template, envvars) if not result.ok: logger.error( f"Error tagging instances " f"`{', '.join(sorted([node.node_id for node in nodes]))}`. " f"Non-zero return code ({result.ret_code})") else: tagged_instances += nodes return tagged_instances
def resume_instances(self, nodes_to_resume: List[NodeDescriptor], timeout: int = 600) -> List[NodeDescriptor]: state = 'running' resumed_nodes = [] # group nodes with the same provider for _, nodes in sorted_groupby( nodes_to_resume, lambda x: x.configuration.provider.provider_config_id).items(): provider_config = nodes[0].configuration.provider task_check_name = f"Resuming nodes " \ f"`{', '.join(sorted([node.nickname for node in nodes]))}`" events = self.execute_common_template(nodes, provider_config, task_check_name, state, wait='yes') if not events: raise Exception('Resume task returned no events to process') # remove instances... task_event = self._get_named_event(events, task_check_name) already_running_instances = task_event['event_data']['res'][ 'instance_ids'] resumed_instances = { i['id']: i for i in task_event['event_data']['res']['instances'] } for node in nodes: if node.cloud_instance_id in resumed_instances: instance = resumed_instances[node.cloud_instance_id] node.ip = instance['public_ip'] node.extra = { 'instance_id': instance['id'], 'private_ip': instance['private_ip'], 'dns': instance['dns_name'], 'private_dns': instance['private_dns_name'], 'architecture': instance['architecture'], 'instance_tags': instance['tags'], 'vpc_id': None, 'subnet_id': None } node.status = NodeStatus.STARTED elif node.cloud_instance_id in already_running_instances: pass else: logger.error(f"Node `{node.node_id}` with invalid aws " f"instance id: {node.cloud_instance_id}") continue resumed_nodes.append(node) return resumed_nodes
def pause_instances(self, nodes_to_pause: List[NodeDescriptor], timeout: int = 600) -> List[NodeDescriptor]: state = 'stopped' paused_nodes = [] # group nodes with the same provider for _, nodes in sorted_groupby( nodes_to_pause, lambda x: x.configuration.provider.provider_config_id).items(): provider_config = nodes[0].configuration.provider task_check_name = f"Pausing nodes " \ f"`{', '.join(sorted([node.nickname for node in nodes]))}`" events = self.execute_common_template(nodes, provider_config, task_check_name, state) if not events: raise Exception('Pause task returned no events to process') # remove instances... task_event = self._get_named_event(events, task_check_name) already_paused_instance_ids = task_event['event_data']['res'][ 'instance_ids'] paused_instances = { i['id']: i for i in task_event['event_data']['res']['instances'] } for node in nodes: if node.cloud_instance_id in paused_instances: pass elif node.cloud_instance_id in already_paused_instance_ids: pass if node.cloud_instance_id not in paused_instances and \ node.cloud_instance_id not in already_paused_instance_ids: logger.error(f"Node '{node.node_id}' has an invalid aws " f"instance id: {node.cloud_instance_id}") continue node.ip = None node.status = NodeStatus.PAUSED paused_nodes.append(node) return paused_nodes
def group_nodes_by_provider(nodes: List[NodeDescriptor]) -> \ Dict[str, List[NodeDescriptor]]: return sorted_groupby(nodes, key=lambda x: x.configuration.provider.provider)
def _group_instances_by_provider(instances: List[Tuple[InstanceInfo, int]]) -> \ Dict[str, List[Tuple[InstanceInfo, int]]]: return sorted_groupby(instances, key=lambda x: x[0].provider.provider)