def update_instance_info(self,
nodes_to_check: List[NodeDescriptor],
timeout: int = 600) -> List[NodeDescriptor]:
# group nodes with the same provider
checked_nodes = list()
for _, nodes in sorted_groupby(
nodes_to_check,
lambda x: x.configuration.provider.provider_config_id).items():
nodes = list(nodes)
provider_config = nodes[0].configuration.provider
task_check_name = f"Checking nodes " \
f"`{', '.join(sorted([node.nickname for node in nodes]))}`"
events = self.execute_check_template(nodes,
provider_config,
task_check_name,
quiet=True)
if not events:
raise Exception('Update task returned no events to process')
# remove instances...
task_event = self._get_named_event(events, task_check_name)
instances_status = {
i['instance_id']: i
for i in task_event['event_data']['res']['instances']
}
for node in nodes:
if node.cloud_instance_id not in instances_status:
logger.error(f"Node `{node.node_id}` with invalid aws "
f"instance id: {node.cloud_instance_id}")
continue
instance = instances_status[node.cloud_instance_id]
if instance['state']['name'] == 'running':
node.status = NodeStatus.STARTED
elif instance['state']['name'] == 'stopped':
node.status = NodeStatus.PAUSED
elif instance['state']['name'] == 'terminated':
node.status = NodeStatus.STOPPED
else:
node.status = NodeStatus.UNKNOWN
if node.status != NodeStatus.STOPPED:
node.ip = instance['public_ip_address'] \
if 'public_ip_address' in instance else None
node.cloud_instance_id = instance['instance_id']
node.extra = {
'instance_id': instance['instance_id'],
'private_ip': instance['private_ip_address'],
'dns': instance['public_dns_name'],
'private_dns': instance['private_dns_name'],
'architecture': instance['architecture'],
'instance_tags': instance['tags'],
'vpc_id': instance['vpc_id'],
'subnet_id': instance['subnet_id']
}
checked_nodes.append(node)
return checked_nodes
def stop_instances(self,
nodes_to_stop: List[NodeDescriptor],
timeout: int = 180) -> List[NodeDescriptor]:
state = 'absent'
stopped_nodes: List[NodeDescriptor] = []
# group nodes with the same provider
for _, nodes in sorted_groupby(
nodes_to_stop,
lambda x: x.configuration.provider.provider_config_id).items():
provider_config = nodes[0].configuration.provider
task_check_name = f"Stopping nodes " \
f"{', '.join(sorted([node.nickname for node in nodes]))}"
events = self.execute_common_template(nodes,
provider_config,
task_check_name,
state,
wait='no')
if not events:
raise Exception('Stop task returned no events to process')
# remove instances...
task_event = self._get_named_event(events, task_check_name)
removed_instance_ids = task_event['event_data']['res'][
'instance_ids']
successfully_stopped = []
for node in nodes:
if node.cloud_instance_id in removed_instance_ids:
node.status = NodeStatus.STOPPED
node.ip = None
successfully_stopped.append(node)
stopped_nodes += successfully_stopped
return stopped_nodes
def _tag_instances(
self, node_list: List[NodeDescriptor]) -> List[NodeDescriptor]:
task_check_name = 'Tagging instances'
tagged_instances = []
for _, nodes in sorted_groupby(
node_list,
lambda x: x.configuration.provider.provider_config_id).items():
nodes = list(nodes)
envvars = self.create_envvars(nodes[0].configuration.provider)
names = [{
'id':
node.cloud_instance_id,
'name':
f"{node.nickname.replace(' ', '')}-{node.node_id[:8]}"
} for node in nodes]
ec2_vals = {
'task_name': task_check_name,
'names': names,
}
rendered_template = self.jinjaenv.get_template(
'ec2_tag_instances.j2').render(ec2_vals)
result = self._run_template('ec2_tag_instances.yml',
rendered_template, envvars)
if not result.ok:
logger.error(
f"Error tagging instances "
f"`{', '.join(sorted([node.node_id for node in nodes]))}`. "
f"Non-zero return code ({result.ret_code})")
else:
tagged_instances += nodes
return tagged_instances
def resume_instances(self,
nodes_to_resume: List[NodeDescriptor],
timeout: int = 600) -> List[NodeDescriptor]:
state = 'running'
resumed_nodes = []
# group nodes with the same provider
for _, nodes in sorted_groupby(
nodes_to_resume,
lambda x: x.configuration.provider.provider_config_id).items():
provider_config = nodes[0].configuration.provider
task_check_name = f"Resuming nodes " \
f"`{', '.join(sorted([node.nickname for node in nodes]))}`"
events = self.execute_common_template(nodes,
provider_config,
task_check_name,
state,
wait='yes')
if not events:
raise Exception('Resume task returned no events to process')
# remove instances...
task_event = self._get_named_event(events, task_check_name)
already_running_instances = task_event['event_data']['res'][
'instance_ids']
resumed_instances = {
i['id']: i
for i in task_event['event_data']['res']['instances']
}
for node in nodes:
if node.cloud_instance_id in resumed_instances:
instance = resumed_instances[node.cloud_instance_id]
node.ip = instance['public_ip']
node.extra = {
'instance_id': instance['id'],
'private_ip': instance['private_ip'],
'dns': instance['dns_name'],
'private_dns': instance['private_dns_name'],
'architecture': instance['architecture'],
'instance_tags': instance['tags'],
'vpc_id': None,
'subnet_id': None
}
node.status = NodeStatus.STARTED
elif node.cloud_instance_id in already_running_instances:
pass
else:
logger.error(f"Node `{node.node_id}` with invalid aws "
f"instance id: {node.cloud_instance_id}")
continue
resumed_nodes.append(node)
return resumed_nodes
def pause_instances(self,
nodes_to_pause: List[NodeDescriptor],
timeout: int = 600) -> List[NodeDescriptor]:
state = 'stopped'
paused_nodes = []
# group nodes with the same provider
for _, nodes in sorted_groupby(
nodes_to_pause,
lambda x: x.configuration.provider.provider_config_id).items():
provider_config = nodes[0].configuration.provider
task_check_name = f"Pausing nodes " \
f"`{', '.join(sorted([node.nickname for node in nodes]))}`"
events = self.execute_common_template(nodes, provider_config,
task_check_name, state)
if not events:
raise Exception('Pause task returned no events to process')
# remove instances...
task_event = self._get_named_event(events, task_check_name)
already_paused_instance_ids = task_event['event_data']['res'][
'instance_ids']
paused_instances = {
i['id']: i
for i in task_event['event_data']['res']['instances']
}
for node in nodes:
if node.cloud_instance_id in paused_instances:
pass
elif node.cloud_instance_id in already_paused_instance_ids:
pass
if node.cloud_instance_id not in paused_instances and \
node.cloud_instance_id not in already_paused_instance_ids:
logger.error(f"Node '{node.node_id}' has an invalid aws "
f"instance id: {node.cloud_instance_id}")
continue
node.ip = None
node.status = NodeStatus.PAUSED
paused_nodes.append(node)
return paused_nodes
def group_nodes_by_provider(nodes: List[NodeDescriptor]) -> \
Dict[str, List[NodeDescriptor]]:
return sorted_groupby(nodes,
key=lambda x: x.configuration.provider.provider)
def _group_instances_by_provider(instances: List[Tuple[InstanceInfo, int]]) -> \
Dict[str, List[Tuple[InstanceInfo, int]]]:
return sorted_groupby(instances, key=lambda x: x[0].provider.provider)