def create_bucket(self) -> Bucket:
bucket_name = '-'.join(
[self._bucket_prefix,
random_string(12), self._region])
bucket = self._gs.create_bucket(bucket_name, self._region)
return Bucket(bucket)
def get_or_create_bucket(self, output: AbstractOutputWriter, dry_run=False):
bucket_name = self._find_bucket()
if not bucket_name:
bucket_name = '-'.join([self._bucket_prefix, random_string(12), self._region])
if not dry_run:
self._s3.create_bucket(ACL='private', Bucket=bucket_name,
CreateBucketConfiguration={'LocationConstraint': self._region})
output.write('Bucket "%s" was created.' % bucket_name)
return bucket_name
def get_or_create_bucket(self, output: AbstractOutputWriter, tags: list, dry_run=False):
bucket_name = self._find_bucket()
if not bucket_name:
bucket_name = '-'.join([self._bucket_prefix, random_string(12), self._region])
if not dry_run:
# a fix for the boto3 issue: https://github.com/boto/boto3/issues/125
if self._region == 'us-east-1':
self._s3.create_bucket(ACL='private', Bucket=bucket_name)
else:
self._s3.create_bucket(ACL='private', Bucket=bucket_name,
CreateBucketConfiguration={'LocationConstraint': self._region})
self._s3.put_bucket_tagging(Bucket=bucket_name, Tagging={'TagSet': tags})
output.write('Bucket "%s" was created.' % bucket_name)
return bucket_name
def get_or_create_bucket(self,
output: AbstractOutputWriter,
dry_run=False):
bucket_name = self._find_bucket()
if not bucket_name:
bucket_name = '-'.join(
[self._bucket_prefix,
random_string(12), self._region])
if not dry_run:
self._gs.create_bucket(bucket_name, self._region)
self._gs.create_dir(bucket_name, BUCKET_SYNC_DIR)
output.write('Bucket "%s" was created.' % bucket_name)
return bucket_name
def create_bucket(self) -> Bucket:
bucket_name = '-'.join(
[self._bucket_prefix,
random_string(12), self._region])
# a fix for the boto3 issue: https://github.com/boto/boto3/issues/125
if self._region == 'us-east-1':
self._s3.create_bucket(ACL='private', Bucket=bucket_name)
else:
self._s3.create_bucket(
ACL='private',
Bucket=bucket_name,
CreateBucketConfiguration={'LocationConstraint': self._region})
return Bucket({'Name': bucket_name})
def get_or_create_bucket(self,
output: AbstractOutputWriter,
dry_run=False):
bucket_name = self._find_bucket()
if not bucket_name:
bucket_name = '-'.join(
[self._bucket_prefix,
random_string(12), self._region])
if not dry_run:
# GL: this is a workaround to fix the issue https://github.com/boto/boto3/issues/125
if self._region == 'us-east-1':
self._s3.create_bucket(ACL='private', Bucket=bucket_name)
else:
self._s3.create_bucket(ACL='private',
Bucket=bucket_name,
CreateBucketConfiguration={
'LocationConstraint':
self._region
})
output.write('Bucket "%s" was created.' % bucket_name)
return bucket_name
def create_stack(self, template: str, instance_type: str, ami_name: str,
key_name: str):
"""Runs CloudFormation template."""
params = {
'InstanceType': instance_type,
'ImageName': ami_name,
}
if key_name:
params['KeyName'] = key_name
stack_name = 'spotty-nvidia-docker-ami-%s' % random_string(8)
res = self._cf.create_stack(
StackName=stack_name,
TemplateBody=template,
Parameters=[{
'ParameterKey': key,
'ParameterValue': value
} for key, value in params.items()],
Capabilities=['CAPABILITY_IAM'],
OnFailure='DELETE',
)
return res, stack_name
def run(self, output: AbstractOutputWriter):
# check that it's a GPU instance type
instance_type = self._config['instance']['instanceType']
if not is_gpu_instance(instance_type):
raise ValueError('"%s" is not a GPU instance' % instance_type)
region = self._config['instance']['region']
cf = boto3.client('cloudformation', region_name=region)
ec2 = boto3.client('ec2', region_name=region)
# check that an image with this name doesn't exist yet
ami_name = self._config['instance']['amiName']
res = ec2.describe_images(Filters=[
{'Name': 'name', 'Values': [ami_name]},
])
if len(res['Images']):
raise ValueError('AMI with name "%s" already exists.' % ami_name)
# read and update CF template
with open(data_dir('create_ami.yaml')) as f:
template = yaml.load(f, Loader=CfnYamlLoader)
# remove key parameter if key is not provided
key_name = self._config['instance'].get('keyName', '')
if not key_name:
del template['Parameters']['KeyName']
del template['Resources']['SpotInstanceLaunchTemplate']['Properties']['LaunchTemplateData']['KeyName']
# create stack
params = [
{'ParameterKey': 'InstanceType', 'ParameterValue': instance_type},
{'ParameterKey': 'ImageName', 'ParameterValue': ami_name},
]
if key_name:
params.append({'ParameterKey': 'KeyName', 'ParameterValue': key_name})
stack_name = 'spotty-nvidia-docker-ami-%s' % random_string(8)
res = cf.create_stack(
StackName=stack_name,
TemplateBody=yaml.dump(template, Dumper=CfnYamlDumper),
Parameters=params,
Capabilities=['CAPABILITY_IAM'],
OnFailure='DELETE',
)
output.write('Waiting for the AMI to be created...')
resource_messages = [
('InstanceProfile', 'creating IAM role for the instance'),
('SpotInstance', 'launching the instance'),
('InstanceReadyWaitCondition', 'installing NVIDIA Docker'),
('AMICreatedWaitCondition', 'creating AMI and terminating the instance'),
]
# wait for the stack to be created
status, stack = wait_stack_status_changed(cf, stack_id=res['StackId'], waiting_status='CREATE_IN_PROGRESS',
resource_messages=resource_messages,
resource_success_status='CREATE_COMPLETE', output=output)
if status == 'CREATE_COMPLETE':
ami_id = [row['OutputValue'] for row in stack['Outputs'] if row['OutputKey'] == 'NewAMI'][0]
output.write('\n'
'--------------------\n'
'AMI "%s" (ID=%s) was successfully created.\n'
'Use "spotty start" command to run a Spot Instance.\n'
'--------------------' % (ami_name, ami_id))
else:
raise ValueError('Stack "%s" was not created.\n'
'See CloudFormation and CloudWatch logs for details.' % stack_name)
