def create_or_update_stack(self, managed_policy_arns: list, output: AbstractOutputWriter, tags: list):
"""Creates or updates an instance profile.
It was moved to a separate stack because creating of an instance profile resource takes 2 minutes.
"""
# check that policies exist
iam = boto3.client('iam', region_name=self._region)
for policy_arn in managed_policy_arns:
# if the policy doesn't exist, an error will be raised
iam.get_policy(PolicyArn=policy_arn)
template = prepare_instance_profile_template(managed_policy_arns)
stack = Stack.get_by_name(self._cf, self._stack_name)
try:
if stack:
# update the stack and wait until it will be updated
self._update_stack(template, output, tags)
else:
# create the stack and wait until it will be created
self._create_stack(template, output, tags)
stack = Stack.get_by_name(self._cf, self._stack_name)
except WaiterError:
stack = None
if not stack or stack.status not in ['CREATE_COMPLETE', 'UPDATE_COMPLETE']:
raise ValueError('Stack "%s" was not created.\n'
'Please, see CloudFormation logs for the details.' % self._stack_name)
profile_arn = [row['OutputValue'] for row in stack.outputs if row['OutputKey'] == 'ProfileArn'][0]
return profile_arn
def delete_stack(self, output: AbstractOutputWriter, stack_id=None):
"""Deletes an AMI stack.
Args:
output: output writer
stack_id: ID of the stack to delete (for older versions of Spotty)
"""
# delete the image
stack = Stack.get_by_name(self._cf,
stack_id) if stack_id else self.get_stack()
stack.delete()
output.write('Waiting for the AMI to be deleted...')
# wait for the deletion to be completed
with output.prefix(' '):
stack = stack.wait_status_changed(
waiting_status='DELETE_IN_PROGRESS',
resource_messages=[],
resource_success_status='DELETE_COMPLETE',
output=output)
if stack.status == 'DELETE_COMPLETE':
output.write('\n'
'-----------------------------\n'
'AMI was successfully deleted.\n'
'-----------------------------')
else:
raise ValueError(
'Stack "%s" not deleted.\n'
'See CloudFormation and CloudWatch logs for details.' %
stack_id)
def _create_stack(self, template: str, output: AbstractOutputWriter):
"""Creates the stack and waits until it will be created."""
output.write('Creating IAM role for the instance...')
stack = Stack.create_stack(
cf=self._cf,
StackName=self._stack_name,
TemplateBody=template,
Capabilities=['CAPABILITY_IAM'],
OnFailure='DELETE',
)
# wait for the stack to be created
stack.wait_stack_created(delay=15)
def _update_stack(self, template: str, output: AbstractOutputWriter):
"""Updates the stack and waits until it will be updated."""
try:
updated_stack = Stack.update_stack(
cf=self._cf,
StackName=self._stack_name,
TemplateBody=template,
Capabilities=['CAPABILITY_IAM'],
)
except ClientError as e:
# the stack was not updated because there are no changes
updated_stack = None
error_code = e.response.get('Error', {}).get('Code', 'Unknown')
if error_code != 'ValidationError':
raise e
if updated_stack:
# wait for the stack to be updated
output.write('Updating IAM role for the instance...')
updated_stack.wait_stack_updated(delay=15)
def create_stack(self, template: str, parameters: dict, debug_mode: bool,
output: AbstractOutputWriter):
"""Creates an AMI stack and waits for the AMI to be created.
Args:
template: CloudFormation template
parameters: parameters for the template
debug_mode: if "True", NVIDIA Docker will be installed, but an AMI will not be created and the instance
will not be terminated, so the user can connect to the instance for debugging.
output: output writer
"""
stack = Stack.create_stack(
cf=self._cf,
StackName=self._stack_name,
TemplateBody=template,
Parameters=[{
'ParameterKey': key,
'ParameterValue': value
} for key, value in parameters.items()],
Capabilities=['CAPABILITY_IAM'],
OnFailure='DO_NOTHING' if debug_mode else 'DELETE',
)
output.write('Waiting for the AMI to be created...')
resource_messages = [
('InstanceProfile', 'creating IAM role for the instance'),
('Instance', 'launching the instance'),
('InstanceReadyWaitCondition', 'installing NVIDIA Docker'),
('AMICreatedWaitCondition',
'creating AMI and terminating the instance'),
]
# wait for the stack to be created
with output.prefix(' '):
stack = stack.wait_status_changed(
waiting_status='CREATE_IN_PROGRESS',
resource_messages=resource_messages,
resource_success_status='CREATE_COMPLETE',
output=output)
if stack.status != 'CREATE_COMPLETE':
raise ValueError(
'Stack "%s" was not created.\n'
'Please, see CloudFormation logs for the details.' %
self._stack_name)
if debug_mode:
output.write('Stack "%s" was created in debug mode.' %
self._stack_name)
else:
ami_id = [
row['OutputValue'] for row in stack.outputs
if row['OutputKey'] == 'NewAMI'
][0]
output.write('\n'
'--------------------------------------------------\n'
'AMI "%s" (ID=%s) was successfully created.\n'
'Use the "spotty start" command to run an instance.\n'
'--------------------------------------------------' %
(parameters['ImageName'], ami_id))
def get_stack(self) -> Stack:
return Stack.get_by_name(self._cf, self._stack_name)