def package_template(session, args): print('\nPackaging...') client = session.create_client('s3') config = utils.read_json(args.config) s3_prefix = args.s3_prefix or conventions.generate_stack_name( config['Parameters']) try: s3_uploader = S3Uploader(client, args.s3_bucket, aws.get_region(session), s3_prefix, args.kms_key_id, False) template = Template(args.template, os.getcwd(), s3_uploader) exported_template = template.export() exported_template_yaml = yaml_dump(exported_template) except exceptions.ExportFailedError as ex: if template_has_resources_to_upload_to_s3( template) and not args.s3_bucket: logging.error( 'The template contains resources to upload, please provide an S3 Bucket (--s3-bucket).' ) else: logging.error(ex) sys.exit(1) logging.info(exported_template_yaml) print('Done.') return exported_template_yaml
def _run_main(self, parsed_args, parsed_globals): s3_client = self._session.create_client( "s3", config=Config(signature_version='s3v4'), region_name=parsed_globals.region, verify=parsed_globals.verify_ssl) template_path = parsed_args.template_file if not os.path.isfile(template_path): raise exceptions.InvalidTemplatePathError( template_path=template_path) bucket = parsed_args.s3_bucket self.s3_uploader = S3Uploader(s3_client, bucket, parsed_args.s3_prefix, parsed_args.kms_key_id, parsed_args.force_upload) # attach the given metadata to the artifacts to be uploaded self.s3_uploader.artifact_metadata = parsed_args.metadata output_file = parsed_args.output_template_file use_json = parsed_args.use_json exported_str = self._export(template_path, use_json) sys.stdout.write("\n") self.write_output(output_file, exported_str) if output_file: msg = self.MSG_PACKAGED_TEMPLATE_WRITTEN.format( output_file_name=output_file, output_file_path=os.path.abspath(output_file)) sys.stdout.write(msg) sys.stdout.flush() return 0
def test_artifact_metadata_invalid_type(self): prefix = "SomePrefix" s3uploader = S3Uploader(self.s3client, self.bucket_name, prefix, None, False, self.transfer_manager_mock) invalid_metadata = ["key", "val"] with self.assertRaises(TypeError): s3uploader.artifact_metadata = invalid_metadata
def test_upload_successful_odict(self, progress_percentage_mock, get_size_patch): file_name = "filename" remote_path = "remotepath" prefix = "SomePrefix" remote_path_with_prefix = "{0}/{1}".format(prefix, remote_path) s3uploader = S3Uploader(self.s3client, self.bucket_name, prefix, None, False, self.transfer_manager_mock) expected_upload_url = "s3://{0}/{1}/{2}".format( self.bucket_name, prefix, remote_path) # Setup mock to fake that file does not exist s3uploader.file_exists = Mock() s3uploader.file_exists.return_value = False # set the metadata used by the uploader when uploading artifact_metadata = OrderedDict({"key": "val"}) s3uploader.artifact_metadata = artifact_metadata upload_url = s3uploader.upload(file_name, remote_path) self.assertEquals(expected_upload_url, upload_url) expected_extra_args = { # expected encryption args "ServerSideEncryption": "AES256", # expected metadata "Metadata": artifact_metadata } self.transfer_manager_mock.upload.assert_called_once_with( file_name, self.bucket_name, remote_path_with_prefix, expected_extra_args, mock.ANY) s3uploader.file_exists.assert_called_once_with(remote_path_with_prefix)
def test_upload_successful_custom_kms_key(self, progress_percentage_mock, get_size_patch): file_name = "filename" remote_path = "remotepath" kms_key_id = "kms_id" expected_upload_url = "s3://{0}/{1}".format(self.bucket_name, remote_path) # Set KMS Key Id self.s3uploader = S3Uploader(self.s3client, self.bucket_name, self.prefix, kms_key_id, False, self.transfer_manager_mock) # Setup mock to fake that file does not exist self.s3uploader.file_exists = Mock() self.s3uploader.file_exists.return_value = False upload_url = self.s3uploader.upload(file_name, remote_path) self.assertEquals(expected_upload_url, upload_url) expected_encryption_args = { "ServerSideEncryption": "aws:kms", "SSEKMSKeyId": kms_key_id } self.transfer_manager_mock.upload.assert_called_once_with( file_name, self.bucket_name, remote_path, expected_encryption_args, mock.ANY) self.s3uploader.file_exists.assert_called_once_with(remote_path)
def test_file_exists(self): key = "some/path" expected_params = {"Bucket": self.bucket_name, "Key": key} response = { "AcceptRanges": "bytes", "ContentType": "text/html", "LastModified": "Thu, 16 Apr 2015 18:19:14 GMT", "ContentLength": 77, "VersionId": "null", "ETag": "\"30a6ec7e1a9ad79c203d05a589c8b400\"", "Metadata": {} } # Let's pretend file exists self.s3client_stub.add_response("head_object", response, expected_params) with self.s3client_stub: self.assertTrue(self.s3uploader.file_exists(key)) # Let's pretend file does not exist self.s3client_stub.add_client_error('head_object', "ClientError", "some error") with self.s3client_stub: self.assertFalse(self.s3uploader.file_exists(key)) # Let's pretend some other unknown exception happened s3mock = Mock() uploader = S3Uploader(s3mock, self.bucket_name) s3mock.head_object = Mock() s3mock.head_object.side_effect = RuntimeError() with self.assertRaises(RuntimeError): uploader.file_exists(key)
def test_upload_force_upload(self, progress_percentage_mock, get_size_patch): file_name = "filename" remote_path = "remotepath" expected_upload_url = "s3://{0}/{1}".format(self.bucket_name, remote_path) # Set ForceUpload = True self.s3uploader = S3Uploader(self.s3client, self.bucket_name, self.prefix, None, True, self.transfer_manager_mock) # Pretend file already exists self.s3uploader.file_exists = Mock() self.s3uploader.file_exists.return_value = True # Because we forced an update, this should reupload even if file exists upload_url = self.s3uploader.upload(file_name, remote_path) self.assertEquals(expected_upload_url, upload_url) expected_encryption_args = {"ServerSideEncryption": "AES256"} self.transfer_manager_mock.upload.assert_called_once_with( file_name, self.bucket_name, remote_path, expected_encryption_args, mock.ANY) # Since ForceUpload=True, we should NEVER do the file-exists check self.s3uploader.file_exists.assert_not_called()
def test_to_path_style_s3_url_other_regions(self): key = "path/to/file" version = "someversion" region = "us-west-2" s3uploader = S3Uploader(self.s3client, self.bucket_name, region) result = s3uploader.to_path_style_s3_url(key, version) self.assertEqual( result, "https://s3-{0}.amazonaws.com/{1}/{2}?versionId={3}".format( region, self.bucket_name, key, version)) # Without versionId, that query parameter should be omitted s3uploader = S3Uploader(self.s3client, self.bucket_name, region) result = s3uploader.to_path_style_s3_url(key) self.assertEqual( result, "https://s3-{0}.amazonaws.com/{1}/{2}".format( region, self.bucket_name, key))
def _run_main(self, parsed_args, parsed_globals): cloudformation_client = \ self._session.create_client( 'cloudformation', region_name=parsed_globals.region, endpoint_url=parsed_globals.endpoint_url, verify=parsed_globals.verify_ssl) template_path = parsed_args.template_file if not os.path.isfile(template_path): raise exceptions.InvalidTemplatePathError( template_path=template_path) # Parse parameters with open(template_path, "r") as handle: template_str = handle.read() stack_name = parsed_args.stack_name parameter_overrides = self.parse_key_value_arg( parsed_args.parameter_overrides, self.PARAMETER_OVERRIDE_CMD) tags_dict = self.parse_key_value_arg(parsed_args.tags, self.TAGS_CMD) tags = [{"Key": key, "Value": value} for key, value in tags_dict.items()] template_dict = yaml_parse(template_str) parameters = self.merge_parameters(template_dict, parameter_overrides) template_size = os.path.getsize(parsed_args.template_file) if template_size > 51200 and not parsed_args.s3_bucket: raise exceptions.DeployBucketRequiredError() bucket = parsed_args.s3_bucket if bucket: s3_client = self._session.create_client( "s3", config=Config(signature_version='s3v4'), region_name=parsed_globals.region, verify=parsed_globals.verify_ssl) s3_uploader = S3Uploader(s3_client, bucket, parsed_args.s3_prefix, parsed_args.kms_key_id, parsed_args.force_upload) else: s3_uploader = None deployer = Deployer(cloudformation_client) return self.deploy(deployer, stack_name, template_str, parameters, parsed_args.capabilities, parsed_args.execute_changeset, parsed_args.role_arn, parsed_args.notification_arns, s3_uploader, tags, parsed_args.fail_on_empty_changeset)
def test_to_path_style_s3_url_us_east_1(self): key = "path/to/file" version = "someversion" region = "us-east-1" self._construct_uploader(region) s3uploader = S3Uploader(self.s3client, self.bucket_name) result = s3uploader.to_path_style_s3_url(key, version) self.assertEqual( result, "https://s3.amazonaws.com/{0}/{1}?versionId={2}".format( self.bucket_name, key, version)) # Without versionId, that query parameter should be omitted s3uploader = S3Uploader(self.s3client, self.bucket_name, region) result = s3uploader.to_path_style_s3_url(key) self.assertEqual( result, "https://s3.amazonaws.com/{0}/{1}".format(self.bucket_name, key, version))
def _construct_uploader(self, region): self.s3client = botocore.session.get_session().create_client( 's3', region_name=region) self.s3client_stub = Stubber(self.s3client) self.transfer_manager_mock = Mock(spec=S3Transfer) self.transfer_manager_mock.upload = Mock() self.bucket_name = "bucketname" self.prefix = None self.s3uploader = S3Uploader(self.s3client, self.bucket_name, self.prefix, None, False, self.transfer_manager_mock)
def setUp(self): self.s3client = botocore.session.get_session().create_client( 's3', region_name="us-east-1") self.s3client_stub = Stubber(self.s3client) self.transfer_manager_mock = Mock(spec=S3Transfer) self.transfer_manager_mock.upload = Mock() self.bucket_name = "bucketname" self.prefix = None self.region = "us-east-1" self.s3uploader = S3Uploader(self.s3client, self.bucket_name, self.region, self.prefix, None, False, self.transfer_manager_mock)
def _run_main(self, parsed_args, parsed_globals): self.region = self.get_and_validate_region(parsed_globals) self.s3_client = self._session.create_client( 's3', region_name=self.region, endpoint_url=parsed_globals.endpoint_url, verify=parsed_globals.verify_ssl ) self.s3_uploader = S3Uploader(self.s3_client, parsed_args.bucket_name, self.region, force_upload=True) self.s3_uploader.upload(parsed_args.file_path, get_s3_path(parsed_args.file_path))
def _run_main(self, parsed_args, parsed_globals): self.region = self.get_and_validate_region(parsed_globals) self.s3_client = self._session.create_client( 's3', region_name=self.region, endpoint_url=parsed_globals.endpoint_url, verify=parsed_globals.verify_ssl) self.s3_uploader = S3Uploader(self.s3_client, parsed_args.bucket_name, force_upload=True) try: self.s3_uploader.upload(parsed_args.file_path, get_s3_path(parsed_args.file_path)) except OSError as ex: raise RuntimeError("%s cannot be found" % parsed_args.file_path)
def test_upload_successful(self, progress_percentage_mock, get_size_patch): file_name = "filename" remote_path = "remotepath" prefix = "SomePrefix" remote_path_with_prefix = "{0}/{1}".format(prefix, remote_path) s3uploader = S3Uploader(self.s3client, self.bucket_name, self.region, prefix, None, False, self.transfer_manager_mock) expected_upload_url = "s3://{0}/{1}/{2}".format( self.bucket_name, prefix, remote_path) # Setup mock to fake that file does not exist s3uploader.file_exists = Mock() s3uploader.file_exists.return_value = False upload_url = s3uploader.upload(file_name, remote_path) self.assertEquals(expected_upload_url, upload_url) expected_encryption_args = {"ServerSideEncryption": "AES256"} self.transfer_manager_mock.upload.assert_called_once_with( file_name, self.bucket_name, remote_path_with_prefix, expected_encryption_args, mock.ANY) s3uploader.file_exists.assert_called_once_with(remote_path_with_prefix)
def package_template(ppt, session, template_path, bucket_region, bucket_name=None, prefix=None, kms_key_id=None): # validate template path if not os.path.isfile(template_path): raise ConfigError('Invalid Template Path "%s"' % template_path) # if bucket name is not provided, create a default bucket with name # awscfncli-{AWS::AccountId}-{AWS::Region} if bucket_name is None: sts = session.client('sts') account_id = sts.get_caller_identity()["Account"] bucket_name = 'awscfncli-%s-%s' % (account_id, bucket_region) ppt.secho('Using default artifact bucket s3://{}'.format(bucket_name)) else: ppt.secho( 'Using specified artifact bucket s3://{}'.format(bucket_name)) s3_client = session.client('s3') # create bucket if not exists try: s3_client.head_bucket(Bucket=bucket_name) except ClientError as e: if e.response['Error']['Code'] == '404': if bucket_region != 'us-east-1': s3_client.create_bucket(Bucket=bucket_name, CreateBucketConfiguration={ 'LocationConstraint': bucket_region }) else: s3_client.create_bucket(Bucket=bucket_name) ppt.secho('Created artifact bucket {}'.format(bucket_name)) else: raise e try: s3_uploader = S3Uploader(s3_client, bucket_name, bucket_region, prefix, kms_key_id, force_upload=False) except TypeError: # HACK: since awscli 1.16.145+ the bucket region parameter is removed s3_uploader = S3Uploader(s3_client, bucket_name, prefix, kms_key_id, force_upload=False) template = Template(template_path, os.getcwd(), s3_uploader, resources_to_export=EXPORTS) exported_template = template.export() ppt.secho( 'Successfully packaged artifacts and ' 'uploaded to s3://{bucket_name}.'.format(bucket_name=bucket_name), fg='green') template_body = yaml_dump(exported_template) template_data = template_body.encode('ascii') if len(template_data) <= TEMPLATE_BODY_SIZE_LIMIT: template_url = None else: ppt.secho('Template body is too large, uploading as artifact.', fg='red') with tempfile.NamedTemporaryFile(mode='wb') as fp: # write template body to local temp file fp.write(template_data) fp.flush() # upload to s3 template_location = s3_uploader.upload_with_dedup( fp.name, extension='template.json') ppt.secho('Template uploaded to %s' % template_location) # get s3 object key ...upload_with_dedup() returns s3://bucket/key template_key = template_location.replace('s3://%s/' % bucket_name, '') # generate a pre-signed url for CloudFormation as the object in S3 # is private by default template_url = s3_client.generate_presigned_url( ClientMethod='get_object', Params={ 'Bucket': bucket_name, 'Key': template_key }, ExpiresIn=3600) return template_body, template_url