def run(self, output: AbstractOutputWriter):
region = self._config['instance']['region']
cf = boto3.client('cloudformation', region_name=region)
ec2 = boto3.client('ec2', region_name=region)
# get image info
ami_name = self._config['instance']['amiName']
ami_info = get_ami(ec2, ami_name)
# check that the image exists
if not ami_info:
raise ValueError('AMI with name "%s" not found.' % ami_name)
# get stack ID for the image
tag_values = [
tag['Value'] for tag in ami_info['Tags']
if tag['Key'] == 'spotty:stack-id'
]
if not len(tag_values):
raise ValueError('AMI wasn\'t created by Spotty')
# ask user to confirm the deletion
ami_id = ami_info['ImageId']
confirm = input('AMI "%s" (ID=%s) will be deleted.\n'
'Type "y" to confirm: ' % (ami_name, ami_id))
if confirm != 'y':
output.write('You didn\'t confirm the operation.')
return
# delete the image
stack_id = tag_values[0]
cf.delete_stack(StackName=stack_id)
output.write('Waiting for the AMI to be deleted...')
# wait for the deletion to be completed
status, stack = wait_stack_status_changed(
cf,
stack_id=stack_id,
waiting_status='DELETE_IN_PROGRESS',
resource_messages=[],
resource_success_status='DELETE_COMPLETE',
output=output)
if status == 'DELETE_COMPLETE':
output.write('\n'
'--------------------\n'
'AMI was successfully deleted.\n'
'--------------------')
else:
raise ValueError(
'Stack "%s" not deleted.\n'
'See CloudFormation and CloudWatch logs for details.' %
stack_id)
def create_stack(self, ec2, template: str, instance_profile_arn: str,
instance_type: str, ami_name: str, root_volume_size: int,
mount_dirs: list, bucket_name: str,
remote_project_dir: str, project_name: str,
project_dir: str, docker_config: dict):
"""Runs CloudFormation template."""
# get default VPC ID
res = ec2.describe_vpcs(Filters=[{
'Name': 'isDefault',
'Values': ['true']
}])
if not len(res['Vpcs']):
raise ValueError('Default VPC not found')
vpc_id = res['Vpcs'][0]['VpcId']
# get image info
ami_info = get_ami(ec2, ami_name)
if not ami_info:
raise ValueError(
'AMI with name "%s" not found.\n'
'Use "spotty create-ami" command to create an AMI with NVIDIA Docker.'
% ami_name)
ami_id = ami_info['ImageId']
# check root volume size
image_volume_size = ami_info['BlockDeviceMappings'][0]['Ebs'][
'VolumeSize']
if root_volume_size and root_volume_size < image_volume_size:
raise ValueError(
'Root volume size cannot be less than the size of AMI (%dGB).'
% image_volume_size)
elif not root_volume_size:
root_volume_size = image_volume_size + 5
# create key pair
project_key = KeyPairResource(ec2, self._project_name, self._region)
key_name = project_key.create_key()
# working directory for the Docker container
working_dir = docker_config['workingDir']
if not working_dir:
working_dir = remote_project_dir
# get the Dockerfile path and the build's context path
dockerfile_path = docker_config.get('file', '')
docker_context_path = ''
if dockerfile_path:
if not os.path.isfile(os.path.join(project_dir, dockerfile_path)):
raise ValueError('File "%s" doesn\'t exist.' % dockerfile_path)
dockerfile_path = remote_project_dir + '/' + dockerfile_path
docker_context_path = os.path.dirname(dockerfile_path)
# create stack
params = {
'VpcId':
vpc_id,
'InstanceProfileArn':
instance_profile_arn,
'InstanceType':
instance_type,
'KeyName':
key_name,
'ImageId':
ami_id,
'RootVolumeSize':
str(root_volume_size),
'VolumeMountDirectories':
('"%s"' % '" "'.join(mount_dirs)) if mount_dirs else '',
'DockerDataRootDirectory':
docker_config['dataRoot'],
'DockerImage':
docker_config.get('image', ''),
'DockerfilePath':
dockerfile_path,
'DockerBuildContextPath':
docker_context_path,
'DockerNvidiaRuntime':
'true' if is_gpu_instance(instance_type) else 'false',
'DockerWorkingDirectory':
working_dir,
'InstanceNameTag':
project_name,
'ProjectS3Bucket':
bucket_name,
'ProjectDirectory':
remote_project_dir,
}
res = self._cf.create_stack(
StackName=self._stack_name,
TemplateBody=template,
Parameters=[{
'ParameterKey': key,
'ParameterValue': value
} for key, value in params.items()],
Capabilities=['CAPABILITY_IAM'],
OnFailure='DO_NOTHING',
)
return res
def run(self, output: AbstractOutputWriter):
instance_config = self._config['instance']
# check that it's a GPU instance type
instance_type = instance_config['instanceType']
if not is_gpu_instance(instance_type):
raise ValueError('"%s" is not a GPU instance' % instance_type)
region = instance_config['region']
availability_zone = instance_config['availabilityZone']
subnet_id = instance_config['subnetId']
cf = boto3.client('cloudformation', region_name=region)
ec2 = boto3.client('ec2', region_name=region)
# check that an image with this name doesn't exist yet
ami_name = self._config['instance']['amiName']
ami_info = get_ami(ec2, ami_name)
if ami_info:
raise ValueError('AMI with name "%s" already exists.' % ami_name)
# check availability zone and subnet
check_az_and_subnet(ec2, availability_zone, subnet_id, region)
ami_stack = AmiStackResource(cf)
# prepare CF template
key_name = instance_config.get('keyName', '')
template = ami_stack.prepare_template(availability_zone, subnet_id,
key_name)
# create stack
res, stack_name = ami_stack.create_stack(template, instance_type,
ami_name, key_name)
output.write('Waiting for the AMI to be created...')
resource_messages = [
('InstanceProfile', 'creating IAM role for the instance'),
('SpotInstance', 'launching the instance'),
('InstanceReadyWaitCondition', 'installing NVIDIA Docker'),
('AMICreatedWaitCondition',
'creating AMI and terminating the instance'),
]
# wait for the stack to be created
status, stack = wait_stack_status_changed(
cf,
stack_id=res['StackId'],
waiting_status='CREATE_IN_PROGRESS',
resource_messages=resource_messages,
resource_success_status='CREATE_COMPLETE',
output=output)
if status == 'CREATE_COMPLETE':
ami_id = [
row['OutputValue'] for row in stack['Outputs']
if row['OutputKey'] == 'NewAMI'
][0]
output.write('\n'
'--------------------\n'
'AMI "%s" (ID=%s) was successfully created.\n'
'Use "spotty start" command to run a Spot Instance.\n'
'--------------------' % (ami_name, ami_id))
else:
raise ValueError(
'Stack "%s" was not created.\n'
'See CloudFormation and CloudWatch logs for details.' %
stack_name)