def create_bucket(self) -> Bucket: bucket_name = '-'.join( [self._bucket_prefix, random_string(12), self._region]) bucket = self._gs.create_bucket(bucket_name, self._region) return Bucket(bucket)
def get_or_create_bucket(self, output: AbstractOutputWriter, dry_run=False): bucket_name = self._find_bucket() if not bucket_name: bucket_name = '-'.join([self._bucket_prefix, random_string(12), self._region]) if not dry_run: self._s3.create_bucket(ACL='private', Bucket=bucket_name, CreateBucketConfiguration={'LocationConstraint': self._region}) output.write('Bucket "%s" was created.' % bucket_name) return bucket_name
def get_or_create_bucket(self, output: AbstractOutputWriter, tags: list, dry_run=False): bucket_name = self._find_bucket() if not bucket_name: bucket_name = '-'.join([self._bucket_prefix, random_string(12), self._region]) if not dry_run: # a fix for the boto3 issue: https://github.com/boto/boto3/issues/125 if self._region == 'us-east-1': self._s3.create_bucket(ACL='private', Bucket=bucket_name) else: self._s3.create_bucket(ACL='private', Bucket=bucket_name, CreateBucketConfiguration={'LocationConstraint': self._region}) self._s3.put_bucket_tagging(Bucket=bucket_name, Tagging={'TagSet': tags}) output.write('Bucket "%s" was created.' % bucket_name) return bucket_name
def get_or_create_bucket(self, output: AbstractOutputWriter, dry_run=False): bucket_name = self._find_bucket() if not bucket_name: bucket_name = '-'.join( [self._bucket_prefix, random_string(12), self._region]) if not dry_run: self._gs.create_bucket(bucket_name, self._region) self._gs.create_dir(bucket_name, BUCKET_SYNC_DIR) output.write('Bucket "%s" was created.' % bucket_name) return bucket_name
def create_bucket(self) -> Bucket: bucket_name = '-'.join( [self._bucket_prefix, random_string(12), self._region]) # a fix for the boto3 issue: https://github.com/boto/boto3/issues/125 if self._region == 'us-east-1': self._s3.create_bucket(ACL='private', Bucket=bucket_name) else: self._s3.create_bucket( ACL='private', Bucket=bucket_name, CreateBucketConfiguration={'LocationConstraint': self._region}) return Bucket({'Name': bucket_name})
def get_or_create_bucket(self, output: AbstractOutputWriter, dry_run=False): bucket_name = self._find_bucket() if not bucket_name: bucket_name = '-'.join( [self._bucket_prefix, random_string(12), self._region]) if not dry_run: # GL: this is a workaround to fix the issue https://github.com/boto/boto3/issues/125 if self._region == 'us-east-1': self._s3.create_bucket(ACL='private', Bucket=bucket_name) else: self._s3.create_bucket(ACL='private', Bucket=bucket_name, CreateBucketConfiguration={ 'LocationConstraint': self._region }) output.write('Bucket "%s" was created.' % bucket_name) return bucket_name
def create_stack(self, template: str, instance_type: str, ami_name: str, key_name: str): """Runs CloudFormation template.""" params = { 'InstanceType': instance_type, 'ImageName': ami_name, } if key_name: params['KeyName'] = key_name stack_name = 'spotty-nvidia-docker-ami-%s' % random_string(8) res = self._cf.create_stack( StackName=stack_name, TemplateBody=template, Parameters=[{ 'ParameterKey': key, 'ParameterValue': value } for key, value in params.items()], Capabilities=['CAPABILITY_IAM'], OnFailure='DELETE', ) return res, stack_name
def run(self, output: AbstractOutputWriter): # check that it's a GPU instance type instance_type = self._config['instance']['instanceType'] if not is_gpu_instance(instance_type): raise ValueError('"%s" is not a GPU instance' % instance_type) region = self._config['instance']['region'] cf = boto3.client('cloudformation', region_name=region) ec2 = boto3.client('ec2', region_name=region) # check that an image with this name doesn't exist yet ami_name = self._config['instance']['amiName'] res = ec2.describe_images(Filters=[ {'Name': 'name', 'Values': [ami_name]}, ]) if len(res['Images']): raise ValueError('AMI with name "%s" already exists.' % ami_name) # read and update CF template with open(data_dir('create_ami.yaml')) as f: template = yaml.load(f, Loader=CfnYamlLoader) # remove key parameter if key is not provided key_name = self._config['instance'].get('keyName', '') if not key_name: del template['Parameters']['KeyName'] del template['Resources']['SpotInstanceLaunchTemplate']['Properties']['LaunchTemplateData']['KeyName'] # create stack params = [ {'ParameterKey': 'InstanceType', 'ParameterValue': instance_type}, {'ParameterKey': 'ImageName', 'ParameterValue': ami_name}, ] if key_name: params.append({'ParameterKey': 'KeyName', 'ParameterValue': key_name}) stack_name = 'spotty-nvidia-docker-ami-%s' % random_string(8) res = cf.create_stack( StackName=stack_name, TemplateBody=yaml.dump(template, Dumper=CfnYamlDumper), Parameters=params, Capabilities=['CAPABILITY_IAM'], OnFailure='DELETE', ) output.write('Waiting for the AMI to be created...') resource_messages = [ ('InstanceProfile', 'creating IAM role for the instance'), ('SpotInstance', 'launching the instance'), ('InstanceReadyWaitCondition', 'installing NVIDIA Docker'), ('AMICreatedWaitCondition', 'creating AMI and terminating the instance'), ] # wait for the stack to be created status, stack = wait_stack_status_changed(cf, stack_id=res['StackId'], waiting_status='CREATE_IN_PROGRESS', resource_messages=resource_messages, resource_success_status='CREATE_COMPLETE', output=output) if status == 'CREATE_COMPLETE': ami_id = [row['OutputValue'] for row in stack['Outputs'] if row['OutputKey'] == 'NewAMI'][0] output.write('\n' '--------------------\n' 'AMI "%s" (ID=%s) was successfully created.\n' 'Use "spotty start" command to run a Spot Instance.\n' '--------------------' % (ami_name, ami_id)) else: raise ValueError('Stack "%s" was not created.\n' 'See CloudFormation and CloudWatch logs for details.' % stack_name)