def run(self, output: AbstractOutputWriter):
region = self._config['instance']['region']
cf = boto3.client('cloudformation', region_name=region)
ec2 = boto3.client('ec2', region_name=region)
# get image info
ami_name = self._config['instance']['amiName']
ami_info = get_ami(ec2, ami_name)
# check that the image exists
if not ami_info:
raise ValueError('AMI with name "%s" not found.' % ami_name)
# get stack ID for the image
tag_values = [
tag['Value'] for tag in ami_info['Tags']
if tag['Key'] == 'spotty:stack-id'
]
if not len(tag_values):
raise ValueError('AMI wasn\'t created by Spotty')
# ask user to confirm the deletion
ami_id = ami_info['ImageId']
confirm = input('AMI "%s" (ID=%s) will be deleted.\n'
'Type "y" to confirm: ' % (ami_name, ami_id))
if confirm != 'y':
output.write('You didn\'t confirm the operation.')
return
# delete the image
stack_id = tag_values[0]
cf.delete_stack(StackName=stack_id)
output.write('Waiting for the AMI to be deleted...')
# wait for the deletion to be completed
status, stack = wait_stack_status_changed(
cf,
stack_id=stack_id,
waiting_status='DELETE_IN_PROGRESS',
resource_messages=[],
resource_success_status='DELETE_COMPLETE',
output=output)
if status == 'DELETE_COMPLETE':
output.write('\n'
'--------------------\n'
'AMI was successfully deleted.\n'
'--------------------')
else:
raise ValueError(
'Stack "%s" not deleted.\n'
'See CloudFormation and CloudWatch logs for details.' %
stack_id)
def run(self, output: AbstractOutputWriter):
project_config = self._config['project']
instance_config = self._config['instance']
project_name = project_config['name']
region = instance_config['region']
cf = boto3.client('cloudformation', region_name=region)
stack = StackResource(cf, project_name, region)
# check that the stack exists
if not stack.stack_exists():
raise ValueError('Stack "%s" doesn\'t exists.' % stack.name)
# get stack ID
stack_info = stack.get_stack_info()
stack_id = stack_info['StackId']
# delete the stack
stack.delete_stack()
output.write('Waiting for the stack to be deleted...')
resource_messages = [
('TerminateInstance', 'terminating the instance'),
('_', 'creating snapshots and deleting the volumes'),
]
# wait for the deletion to be completed
status, stack_info = wait_stack_status_changed(cf, stack_id=stack_id, waiting_status='DELETE_IN_PROGRESS',
resource_messages=resource_messages,
resource_success_status='DELETE_COMPLETE', output=output)
if status == 'DELETE_COMPLETE':
output.write('\n'
'--------------------\n'
'Stack was successfully deleted.\n'
'--------------------')
else:
raise ValueError('Stack "%s" was not deleted.\n'
'See CloudFormation and CloudWatch logs for details.' % stack.name)
def run(self, output: AbstractOutputWriter):
project_config = self._config['project']
instance_config = self._config['instance']
region = instance_config['region']
cf = boto3.client('cloudformation', region_name=region)
ec2 = boto3.client('ec2', region_name=region)
s3 = boto3.client('s3', region_name=region)
project_name = project_config['name']
stack = StackResource(cf, project_name, region)
# check if the stack already exists
if stack.stack_exists():
raise ValueError('Stack "%s" already exists.\n'
'Use "spotty stop" command to delete the stack.' %
stack.name)
# create bucket for the project
project_bucket = BucketResource(s3, project_name, region)
bucket_name = project_bucket.create_bucket(output)
# sync the project with S3
output.write('Syncing the project with S3...')
project_filters = project_config['syncFilters']
AwsCli(region=region).s3_sync(self._project_dir,
's3://%s/project' % bucket_name,
delete=True,
filters=project_filters,
capture_output=False)
# create or update instance profile
instance_profile_arn = create_or_update_instance_profile(cf, output)
# prepare CloudFormation template
output.write('Preparing CloudFormation template...')
# check availability zone
availability_zone = instance_config['availabilityZone']
if availability_zone:
zones = ec2.describe_availability_zones()
zone_names = [
zone['ZoneName'] for zone in zones['AvailabilityZones']
]
if availability_zone not in zone_names:
raise ValueError(
'Availability zone "%s" doesn\'t exist in the "%s" region.'
% (availability_zone, region))
instance_type = instance_config['instanceType']
volumes = instance_config['volumes']
ports = instance_config['ports']
max_price = instance_config['maxPrice']
docker_commands = instance_config['docker']['commands']
template = stack.prepare_template(ec2, availability_zone,
instance_type, volumes, ports,
max_price, docker_commands)
# create stack
ami_name = instance_config['amiName']
root_volume_size = instance_config['rootVolumeSize']
mount_dirs = [volume['directory'] for volume in volumes]
docker_config = instance_config['docker']
remote_project_dir = project_config['remoteDir']
res = stack.create_stack(ec2, template, instance_profile_arn,
instance_type, ami_name, root_volume_size,
mount_dirs, bucket_name, remote_project_dir,
docker_config)
output.write('Waiting for the stack to be created...')
resource_messages = [
('SpotInstance', 'launching the instance'),
('DockerReadyWaitCondition',
'waiting for the Docker container to be ready'),
]
# wait for the stack to be created
status, info = wait_stack_status_changed(
cf,
stack_id=res['StackId'],
waiting_status='CREATE_IN_PROGRESS',
resource_messages=resource_messages,
resource_success_status='CREATE_COMPLETE',
output=output)
if status == 'CREATE_COMPLETE':
ip_address = [
row['OutputValue'] for row in info['Outputs']
if row['OutputKey'] == 'InstanceIpAddress'
][0]
availability_zone = [
row['OutputValue'] for row in info['Outputs']
if row['OutputKey'] == 'AvailabilityZone'
][0]
# get the current spot price
current_price = get_current_spot_price(ec2, instance_type,
availability_zone)
output.write(
'\n'
'--------------------\n'
'Instance is running.\n'
'\n'
'IP address: %s\n'
'Current Spot price: $%.04f\n'
'\n'
'Use "spotty ssh" command to connect to the Docker container.\n'
'--------------------' % (ip_address, current_price))
else:
raise ValueError(
'Stack "%s" was not created.\n'
'Please, see CloudFormation and CloudWatch logs for the details.'
% stack.name)
def run(self, output: AbstractOutputWriter):
# check that it's a GPU instance type
instance_type = self._config['instance']['instanceType']
if not is_gpu_instance(instance_type):
raise ValueError('"%s" is not a GPU instance' % instance_type)
region = self._config['instance']['region']
cf = boto3.client('cloudformation', region_name=region)
ec2 = boto3.client('ec2', region_name=region)
# check that an image with this name doesn't exist yet
ami_name = self._config['instance']['amiName']
res = ec2.describe_images(Filters=[
{'Name': 'name', 'Values': [ami_name]},
])
if len(res['Images']):
raise ValueError('AMI with name "%s" already exists.' % ami_name)
# read and update CF template
with open(data_dir('create_ami.yaml')) as f:
template = yaml.load(f, Loader=CfnYamlLoader)
# remove key parameter if key is not provided
key_name = self._config['instance'].get('keyName', '')
if not key_name:
del template['Parameters']['KeyName']
del template['Resources']['SpotInstanceLaunchTemplate']['Properties']['LaunchTemplateData']['KeyName']
# create stack
params = [
{'ParameterKey': 'InstanceType', 'ParameterValue': instance_type},
{'ParameterKey': 'ImageName', 'ParameterValue': ami_name},
]
if key_name:
params.append({'ParameterKey': 'KeyName', 'ParameterValue': key_name})
stack_name = 'spotty-nvidia-docker-ami-%s' % random_string(8)
res = cf.create_stack(
StackName=stack_name,
TemplateBody=yaml.dump(template, Dumper=CfnYamlDumper),
Parameters=params,
Capabilities=['CAPABILITY_IAM'],
OnFailure='DELETE',
)
output.write('Waiting for the AMI to be created...')
resource_messages = [
('InstanceProfile', 'creating IAM role for the instance'),
('SpotInstance', 'launching the instance'),
('InstanceReadyWaitCondition', 'installing NVIDIA Docker'),
('AMICreatedWaitCondition', 'creating AMI and terminating the instance'),
]
# wait for the stack to be created
status, stack = wait_stack_status_changed(cf, stack_id=res['StackId'], waiting_status='CREATE_IN_PROGRESS',
resource_messages=resource_messages,
resource_success_status='CREATE_COMPLETE', output=output)
if status == 'CREATE_COMPLETE':
ami_id = [row['OutputValue'] for row in stack['Outputs'] if row['OutputKey'] == 'NewAMI'][0]
output.write('\n'
'--------------------\n'
'AMI "%s" (ID=%s) was successfully created.\n'
'Use "spotty start" command to run a Spot Instance.\n'
'--------------------' % (ami_name, ami_id))
else:
raise ValueError('Stack "%s" was not created.\n'
'See CloudFormation and CloudWatch logs for details.' % stack_name)
def run(self, output: AbstractOutputWriter):
instance_config = self._config['instance']
# check that it's a GPU instance type
instance_type = instance_config['instanceType']
if not is_gpu_instance(instance_type):
raise ValueError('"%s" is not a GPU instance' % instance_type)
region = instance_config['region']
availability_zone = instance_config['availabilityZone']
subnet_id = instance_config['subnetId']
cf = boto3.client('cloudformation', region_name=region)
ec2 = boto3.client('ec2', region_name=region)
# check that an image with this name doesn't exist yet
ami_name = self._config['instance']['amiName']
ami_info = get_ami(ec2, ami_name)
if ami_info:
raise ValueError('AMI with name "%s" already exists.' % ami_name)
# check availability zone and subnet
check_az_and_subnet(ec2, availability_zone, subnet_id, region)
ami_stack = AmiStackResource(cf)
# prepare CF template
key_name = instance_config.get('keyName', '')
template = ami_stack.prepare_template(availability_zone, subnet_id,
key_name)
# create stack
res, stack_name = ami_stack.create_stack(template, instance_type,
ami_name, key_name)
output.write('Waiting for the AMI to be created...')
resource_messages = [
('InstanceProfile', 'creating IAM role for the instance'),
('SpotInstance', 'launching the instance'),
('InstanceReadyWaitCondition', 'installing NVIDIA Docker'),
('AMICreatedWaitCondition',
'creating AMI and terminating the instance'),
]
# wait for the stack to be created
status, stack = wait_stack_status_changed(
cf,
stack_id=res['StackId'],
waiting_status='CREATE_IN_PROGRESS',
resource_messages=resource_messages,
resource_success_status='CREATE_COMPLETE',
output=output)
if status == 'CREATE_COMPLETE':
ami_id = [
row['OutputValue'] for row in stack['Outputs']
if row['OutputKey'] == 'NewAMI'
][0]
output.write('\n'
'--------------------\n'
'AMI "%s" (ID=%s) was successfully created.\n'
'Use "spotty start" command to run a Spot Instance.\n'
'--------------------' % (ami_name, ami_id))
else:
raise ValueError(
'Stack "%s" was not created.\n'
'See CloudFormation and CloudWatch logs for details.' %
stack_name)
def run(self, output: AbstractOutputWriter):
project_config = self._config['project']
instance_config = self._config['instance']
region = instance_config['region']
availability_zone = instance_config['availabilityZone']
subnet_id = instance_config['subnetId']
cf = boto3.client('cloudformation', region_name=region)
ec2 = boto3.client('ec2', region_name=region)
project_name = project_config['name']
stack = StackResource(cf, project_name, region)
# check if the stack already exists
if stack.stack_exists():
raise ValueError('Stack "%s" already exists.\n'
'Use "spotty stop" command to delete the stack.' %
stack.name)
# check availability zone and subnet
check_az_and_subnet(ec2, availability_zone, subnet_id, region)
# sync the project with S3
output.write('Syncing the project with S3...')
sync_filters = project_config['syncFilters']
bucket_name = sync_project_with_s3(self._project_dir, project_name,
region, sync_filters, output)
# create or update instance profile
instance_profile_arn = create_or_update_instance_profile(cf, output)
# prepare CloudFormation template
output.write('Preparing CloudFormation template...')
instance_type = instance_config['instanceType']
volumes = instance_config['volumes']
ports = instance_config['ports']
max_price = instance_config['maxPrice']
on_demand = instance_config['onDemandInstance']
docker_commands = instance_config['docker']['commands']
template = stack.prepare_template(ec2, availability_zone, subnet_id,
instance_type, volumes, ports,
max_price, on_demand,
docker_commands)
# create stack
ami_name = instance_config['amiName']
root_volume_size = instance_config['rootVolumeSize']
mount_dirs = [volume['directory'] for volume in volumes]
docker_config = instance_config['docker']
remote_project_dir = project_config['remoteDir']
res = stack.create_stack(ec2, template, instance_profile_arn,
instance_type, ami_name, root_volume_size,
mount_dirs, bucket_name, remote_project_dir,
project_name, self._project_dir,
docker_config)
output.write('Waiting for the stack to be created...')
resource_messages = [
('SpotInstance', 'launching the instance'),
('DockerReadyWaitCondition',
'waiting for the Docker container to be ready'),
]
# wait for the stack to be created
status, info = wait_stack_status_changed(
cf,
stack_id=res['StackId'],
waiting_status='CREATE_IN_PROGRESS',
resource_messages=resource_messages,
resource_success_status='CREATE_COMPLETE',
output=output)
if status == 'CREATE_COMPLETE':
ip_address = get_instance_ip_address(ec2, stack.name)
output.write('\n'
'--------------------\n'
'Instance is running.\n'
'\n'
'IP address: %s' % ip_address)
if not on_demand:
# get the current spot price
availability_zone = [
row['OutputValue'] for row in info['Outputs']
if row['OutputKey'] == 'AvailabilityZone'
][0]
current_price = get_current_spot_price(ec2, instance_type,
availability_zone)
output.write('Current Spot price: $%.04f' % current_price)
output.write(
'\n'
'Use "spotty ssh" command to connect to the Docker container.\n'
'--------------------')
else:
raise ValueError(
'Stack "%s" was not created.\n'
'Please, see CloudFormation and CloudWatch logs for the details.'
% stack.name)