def ApplyAclChanges(self, url):
"""Applies the changes in self.changes to the provided URL."""
bucket = self.gsutil_api.GetBucket(
url.bucket_name, provider=url.scheme,
fields=['defaultObjectAcl', 'metageneration'])
current_acl = bucket.defaultObjectAcl
if not current_acl:
self._WarnServiceAccounts()
self.logger.warning('Failed to set acl for %s. Please ensure you have '
'OWNER-role access to this resource.', url)
return
modification_count = 0
for change in self.changes:
modification_count += change.Execute(
url, current_acl, 'defacl', self.logger)
if modification_count == 0:
self.logger.info('No changes to %s', url)
return
try:
preconditions = Preconditions(meta_gen_match=bucket.metageneration)
bucket_metadata = apitools_messages.Bucket(defaultObjectAcl=current_acl)
self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata,
preconditions=preconditions,
provider=url.scheme, fields=['id'])
except BadRequestException as e:
# Don't retry on bad requests, e.g. invalid email address.
raise CommandException('Received bad request from server: %s' % str(e))
self.logger.info('Updated default ACL on %s', url)
def _SetWeb(self):
"""Sets website configuration for a bucket."""
main_page_suffix = None
error_page = None
if self.sub_opts:
for o, a in self.sub_opts:
if o == '-m':
main_page_suffix = a
elif o == '-e':
error_page = a
url_args = self.args
website = apitools_messages.Bucket.WebsiteValue(
mainPageSuffix=main_page_suffix, notFoundPage=error_page)
# Iterate over URLs, expanding wildcards and setting the website
# configuration on each.
some_matched = False
for url_str in url_args:
bucket_iter = self.GetBucketUrlIterFromArg(url_str,
bucket_fields=['id'])
for blr in bucket_iter:
url = blr.storage_url
some_matched = True
self.logger.info('Setting website configuration on %s...', blr)
bucket_metadata = apitools_messages.Bucket(website=website)
self.gsutil_api.PatchBucket(url.bucket_name,
bucket_metadata,
provider=url.scheme,
fields=['id'])
if not some_matched:
raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args))
return 0
def _SetLifecycleConfig(self):
"""Sets lifecycle configuration for a Google Cloud Storage bucket."""
lifecycle_arg = self.args[0]
url_args = self.args[1:]
# Disallow multi-provider 'lifecycle set' requests.
if not UrlsAreForSingleProvider(url_args):
raise CommandException('"%s" command spanning providers not allowed.' %
self.command_name)
# Open, read and parse file containing JSON document.
lifecycle_file = open(lifecycle_arg, 'r')
lifecycle_txt = lifecycle_file.read()
lifecycle_file.close()
# Iterate over URLs, expanding wildcards and setting the lifecycle on each.
some_matched = False
for url_str in url_args:
bucket_iter = self.GetBucketUrlIterFromArg(url_str,
bucket_fields=['lifecycle'])
for blr in bucket_iter:
url = blr.storage_url
some_matched = True
self.logger.info('Setting lifecycle configuration on %s...', blr)
if url.scheme == 's3':
self.gsutil_api.XmlPassThroughSetLifecycle(
lifecycle_txt, url, provider=url.scheme)
else:
lifecycle = LifecycleTranslation.JsonLifecycleToMessage(lifecycle_txt)
bucket_metadata = apitools_messages.Bucket(lifecycle=lifecycle)
self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata,
provider=url.scheme, fields=['id'])
if not some_matched:
raise CommandException('No URLs matched')
return 0
def __init__(self, bucket_name, versioned=False):
self.root_object = apitools_messages.Bucket(
name=bucket_name,
versioning=apitools_messages.Bucket.VersioningValue(enabled=versioned))
# Dict of object_name: (dict of 'live': MockObject
# 'versioned': ordered list of MockObject).
self.objects = {}
def _SetVersioning(self):
"""Gets versioning configuration for a bucket."""
versioning_arg = self.args[0].lower()
if versioning_arg not in ('on', 'off'):
raise CommandException('Argument to "%s set" must be either <on|off>'
% (self.command_name))
url_args = self.args[1:]
if not url_args:
self.RaiseWrongNumberOfArgumentsException()
# Iterate over URLs, expanding wildcards and set the versioning
# configuration on each.
some_matched = False
for url_str in url_args:
bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id'])
for blr in bucket_iter:
url = blr.storage_url
some_matched = True
bucket_metadata = apitools_messages.Bucket(
versioning=apitools_messages.Bucket.VersioningValue())
if versioning_arg == 'on':
self.logger.info('Enabling versioning for %s...', url)
bucket_metadata.versioning.enabled = True
else:
self.logger.info('Suspending versioning for %s...', url)
bucket_metadata.versioning.enabled = False
self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata,
provider=url.scheme, fields=['id'])
if not some_matched:
raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args))
def ApplyAclChanges(self, url):
"""Applies the changes in self.changes to the provided URL."""
bucket = self.gsutil_api.GetBucket(
url.bucket_name, provider=url.scheme,
fields=['defaultObjectAcl', 'metageneration'])
# Default object ACLs can be blank if the ACL was set to private, or
# if the user doesn't have permission. We warn about this with defacl get,
# so just try the modification here and if the user doesn't have
# permission they'll get an AccessDeniedException.
current_acl = bucket.defaultObjectAcl
modification_count = 0
for change in self.changes:
modification_count += change.Execute(
url, current_acl, 'defacl', self.logger)
if modification_count == 0:
self.logger.info('No changes to %s', url)
return
try:
preconditions = Preconditions(meta_gen_match=bucket.metageneration)
bucket_metadata = apitools_messages.Bucket(defaultObjectAcl=current_acl)
self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata,
preconditions=preconditions,
provider=url.scheme, fields=['id'])
except BadRequestException as e:
# Don't retry on bad requests, e.g. invalid email address.
raise CommandException('Received bad request from server: %s' % str(e))
except AccessDeniedException:
self._WarnServiceAccounts()
raise CommandException('Failed to set acl for %s. Please ensure you have '
'OWNER-role access to this resource.' % url)
self.logger.info('Updated default ACL on %s', url)
def _DefaultEventHold(self):
"""Sets default value for Event-Based Hold on one or more buckets."""
hold = None
if self.args:
if self.args[0].lower() == 'set':
hold = True
elif self.args[0].lower() == 'release':
hold = False
else:
raise CommandException(
('Invalid subcommand "{}" for the "retention event-default"'
' command.\nSee "gsutil help retention event".'
).format(self.sub_opts))
verb = 'Setting' if hold else 'Releasing'
log_msg_template = '{} default Event-Based Hold on %s...'.format(verb)
bucket_metadata_update = apitools_messages.Bucket(
defaultEventBasedHold=hold)
url_args = self.args[1:]
self.BucketUpdateFunc(
url_args,
bucket_metadata_update,
fields=['id', 'defaultEventBasedHold'],
log_msg_template=log_msg_template)
return 0
def _SetDefStorageClass(self):
"""Sets the default storage class for a bucket."""
# At this point, "set" has been popped off the front of self.args.
normalized_storage_class = NormalizeStorageClass(self.args[0])
url_args = self.args[1:]
if not url_args:
self.RaiseWrongNumberOfArgumentsException()
some_matched = False
for url_str in url_args:
self._CheckIsGsUrl(url_str)
# Throws a CommandException if the argument is not a bucket.
bucket_iter = self.GetBucketUrlIterFromArg(url_str,
bucket_fields=['id'])
for blr in bucket_iter:
some_matched = True
bucket_metadata = apitools_messages.Bucket()
self.logger.info(
'Setting default storage class to "%s" for bucket %s' %
(normalized_storage_class, blr.url_string.rstrip('/')))
bucket_metadata.storageClass = normalized_storage_class
self.gsutil_api.PatchBucket(blr.storage_url.bucket_name,
bucket_metadata,
provider=blr.storage_url.scheme,
fields=['id'])
if not some_matched:
raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args))
def CreateBucketJson(self, bucket_name=None, test_objects=0,
storage_class=None, location=None,
versioning_enabled=False,
retention_policy=None,
bucket_policy_only=False):
"""Creates a test bucket using the JSON API.
The bucket and all of its contents will be deleted after the test.
Args:
bucket_name: Create the bucket with this name. If not provided, a
temporary test bucket name is constructed.
test_objects: The number of objects that should be placed in the bucket.
Defaults to 0.
storage_class: Storage class to use. If not provided we use standard.
location: Location to use.
versioning_enabled: If True, set the bucket's versioning attribute to
True.
retention_policy: Retention policy to be used on the bucket.
bucket_policy_only: If True, set the bucket's iamConfiguration's
bucketPolicyOnly attribute to True.
Returns:
Apitools Bucket for the created bucket.
"""
bucket_name = util.MakeBucketNameValid(
bucket_name or self.MakeTempName('bucket'))
bucket_metadata = apitools_messages.Bucket(name=bucket_name.lower())
if storage_class:
bucket_metadata.storageClass = storage_class
if location:
bucket_metadata.location = location
if versioning_enabled:
bucket_metadata.versioning = (
apitools_messages.Bucket.VersioningValue(enabled=True))
if retention_policy:
bucket_metadata.retentionPolicy = retention_policy
if bucket_policy_only:
iam_config = apitools_messages.Bucket.IamConfigurationValue()
iam_config.bucketPolicyOnly = iam_config.BucketPolicyOnlyValue()
iam_config.bucketPolicyOnly.enabled = True
bucket_metadata.iamConfiguration = iam_config
# TODO: Add retry and exponential backoff.
bucket = self.json_api.CreateBucket(bucket_name,
metadata=bucket_metadata)
# Add bucket to list of buckets to be cleaned up.
# TODO: Clean up JSON buckets using JSON API.
self.bucket_uris.append(
boto.storage_uri('gs://%s' % bucket_name,
suppress_consec_slashes=False))
for i in range(test_objects):
self.CreateObjectJson(bucket_name=bucket_name,
object_name=self.MakeTempName('obj'),
contents='test %d' % i)
return bucket
def ApplyAclChanges(self, name_expansion_result, thread_state=None):
"""Applies the changes in self.changes to the provided URL.
Args:
name_expansion_result: NameExpansionResult describing the target object.
thread_state: If present, gsutil Cloud API instance to apply the changes.
"""
if thread_state:
gsutil_api = thread_state
else:
gsutil_api = self.gsutil_api
url = name_expansion_result.expanded_storage_url
if url.IsBucket():
bucket = gsutil_api.GetBucket(url.bucket_name, provider=url.scheme,
fields=['acl', 'metageneration'])
current_acl = bucket.acl
elif url.IsObject():
gcs_object = encoding.JsonToMessage(apitools_messages.Object,
name_expansion_result.expanded_result)
current_acl = gcs_object.acl
if not current_acl:
self._RaiseForAccessDenied(url)
modification_count = 0
for change in self.changes:
modification_count += change.Execute(url, current_acl, 'acl', self.logger)
if modification_count == 0:
self.logger.info('No changes to %s', url)
return
try:
if url.IsBucket():
preconditions = Preconditions(meta_gen_match=bucket.metageneration)
bucket_metadata = apitools_messages.Bucket(acl=current_acl)
gsutil_api.PatchBucket(url.bucket_name, bucket_metadata,
preconditions=preconditions,
provider=url.scheme, fields=['id'])
else: # Object
preconditions = Preconditions(gen_match=gcs_object.generation,
meta_gen_match=gcs_object.metageneration)
object_metadata = apitools_messages.Object(acl=current_acl)
gsutil_api.PatchObjectMetadata(
url.bucket_name, url.object_name, object_metadata,
preconditions=preconditions, provider=url.scheme,
generation=url.generation, fields=['id'])
except BadRequestException as e:
# Don't retry on bad requests, e.g. invalid email address.
raise CommandException('Received bad request from server: %s' % str(e))
except AccessDeniedException:
self._RaiseForAccessDenied(url)
self.logger.info('Updated ACL on %s', url)
def RunCommand(self):
"""Command entry point for the mb command."""
location = None
storage_class = None
if self.sub_opts:
for o, a in self.sub_opts:
if o == '-l':
location = a
elif o == '-p':
# Project IDs are sent as header values when using gs and s3 XML APIs.
InsistAscii(
a, 'Invalid non-ASCII character found in project ID')
self.project_id = a
elif o == '-c' or o == '-s':
storage_class = NormalizeStorageClass(a)
bucket_metadata = apitools_messages.Bucket(location=location,
storageClass=storage_class)
for bucket_url_str in self.args:
bucket_url = StorageUrlFromString(bucket_url_str)
if not bucket_url.IsBucket():
raise CommandException(
'The mb command requires a URL that specifies a '
'bucket.\n"%s" is not valid.' % bucket_url)
if (not BUCKET_NAME_RE.match(bucket_url.bucket_name)
or TOO_LONG_DNS_NAME_COMP.search(bucket_url.bucket_name)):
raise InvalidUrlError('Invalid bucket name in URL "%s"' %
bucket_url.bucket_name)
self.logger.info('Creating %s...', bucket_url)
# Pass storage_class param only if this is a GCS bucket. (In S3 the
# storage class is specified on the key object.)
try:
self.gsutil_api.CreateBucket(bucket_url.bucket_name,
project_id=self.project_id,
metadata=bucket_metadata,
provider=bucket_url.scheme)
except BadRequestException as e:
if (e.status == 400
and e.reason == 'DotfulBucketNameNotUnderTld'
and bucket_url.scheme == 'gs'):
bucket_name = bucket_url.bucket_name
final_comp = bucket_name[bucket_name.rfind('.') + 1:]
raise CommandException('\n'.join(
textwrap.wrap(
'Buckets with "." in the name must be valid DNS names. The bucket'
' you are attempting to create (%s) is not a valid DNS name,'
' because the final component (%s) is not currently a valid part'
' of the top-level DNS tree.' %
(bucket_name, final_comp))))
else:
raise
return 0
def _ChLabelForBucket(blr):
url = blr.storage_url
self.logger.info('Setting label configuration on %s...', blr)
labels_message = None
# When performing a read-modify-write cycle, include metageneration to
# avoid race conditions (supported for GS buckets only).
metageneration = None
if (self.gsutil_api.GetApiSelector(
url.scheme) == ApiSelector.JSON):
# The JSON API's PATCH semantics allow us to skip read-modify-write,
# with the exception of one edge case - attempting to delete a
# nonexistent label returns an error iff no labels previously existed
corrected_changes = self.label_changes
if self.num_deletions:
(_, bucket_metadata) = self.GetSingleBucketUrlFromArg(
url.url_string,
bucket_fields=['labels', 'metageneration'])
if not bucket_metadata.labels:
metageneration = bucket_metadata.metageneration
# Remove each change that would try to delete a nonexistent key.
corrected_changes = dict(
(k, v)
for k, v in six.iteritems(self.label_changes) if v)
labels_message = LabelTranslation.DictToMessage(
corrected_changes)
else: # ApiSelector.XML
# Perform a read-modify-write cycle so that we can specify which
# existing labels need to be deleted.
(_, bucket_metadata) = self.GetSingleBucketUrlFromArg(
url.url_string, bucket_fields=['labels', 'metageneration'])
metageneration = bucket_metadata.metageneration
label_json = {}
if bucket_metadata.labels:
label_json = json.loads(
LabelTranslation.JsonFromMessage(
bucket_metadata.labels))
# Modify label_json such that all specified labels are added
# (overwriting old labels if necessary) and all specified deletions
# are removed from label_json if already present.
for key, value in six.iteritems(self.label_changes):
if not value and key in label_json:
del label_json[key]
else:
label_json[key] = value
labels_message = LabelTranslation.DictToMessage(label_json)
preconditions = Preconditions(meta_gen_match=metageneration)
bucket_metadata = apitools_messages.Bucket(labels=labels_message)
self.gsutil_api.PatchBucket(url.bucket_name,
bucket_metadata,
preconditions=preconditions,
provider=url.scheme,
fields=['id'])
def _ClearRetention(self):
"""Clear retention retention_period on one or more buckets."""
retention_policy = (apitools_messages.Bucket.RetentionPolicyValue(
retentionPeriod=None))
log_msg_template = 'Clearing Retention Policy on %s...'
bucket_metadata_update = apitools_messages.Bucket(
retentionPolicy=retention_policy)
url_args = self.args
self.BucketUpdateFunc(url_args,
bucket_metadata_update,
fields=['id', 'retentionPolicy'],
log_msg_template=log_msg_template)
return 0
def test_satisfies_pzs_is_displayed_if_present(self, mock_wildcard):
bucket_uri = self.CreateBucket(bucket_name='foo')
bucket_metadata = apitools_messages.Bucket(name='foo', satisfiesPZS=True)
bucket_uri.root_object = bucket_metadata
bucket_uri.url_string = 'foo'
bucket_uri.storage_url = mock.Mock()
mock_wildcard.return_value.IterBuckets.return_value = [bucket_uri]
# MockKey doesn't support hash_algs, so the MD5 will not match.
with SetBotoConfigForTest([('GSUtil', 'check_hashes', 'never')]):
stdout = self.RunCommand('ls', ['-Lb', suri(bucket_uri)],
return_stdout=True)
self.assertRegex(stdout, 'Satisfies PZS:\t\t\tTrue')
def RunCommand(self):
"""Command entry point for the mb command."""
location = None
storage_class = None
if self.sub_opts:
for o, a in self.sub_opts:
if o == '-l':
location = a
elif o == '-p':
self.project_id = a
elif o == '-c':
storage_class = self._Normalize_Storage_Class(a)
bucket_metadata = apitools_messages.Bucket(location=location,
storageClass=storage_class)
for bucket_uri_str in self.args:
bucket_uri = StorageUrlFromString(bucket_uri_str)
if not bucket_uri.IsBucket():
raise CommandException(
'The mb command requires a URI that specifies a '
'bucket.\n"%s" is not valid.' % bucket_uri)
self.logger.info('Creating %s...', bucket_uri)
# Pass storage_class param only if this is a GCS bucket. (In S3 the
# storage class is specified on the key object.)
try:
self.gsutil_api.CreateBucket(bucket_uri.bucket_name,
project_id=self.project_id,
metadata=bucket_metadata,
provider=bucket_uri.scheme)
except BadRequestException as e:
if (e.status == 400
and e.reason == 'DotfulBucketNameNotUnderTld'
and bucket_uri.scheme == 'gs'):
bucket_name = bucket_uri.bucket_name
final_comp = bucket_name[bucket_name.rfind('.') + 1:]
raise CommandException('\n'.join(
textwrap.wrap(
'Buckets with "." in the name must be valid DNS names. The bucket'
' you are attempting to create (%s) is not a valid DNS name,'
' because the final component (%s) is not currently a valid part'
' of the top-level DNS tree.' %
(bucket_name, final_comp))))
else:
raise
return 0
def _SetRetention(self):
"""Set retention retention_period on one or more buckets."""
seconds = RetentionInSeconds(self.args[0])
retention_policy = (apitools_messages.Bucket.RetentionPolicyValue(
retentionPeriod=seconds))
log_msg_template = 'Setting Retention Policy on %s...'
bucket_metadata_update = apitools_messages.Bucket(
retentionPolicy=retention_policy)
url_args = self.args[1:]
self.BucketUpdateFunc(url_args,
bucket_metadata_update,
fields=['id', 'retentionPolicy'],
log_msg_template=log_msg_template)
return 0
def _SetLabelForBucket(blr):
url = blr.storage_url
self.logger.info('Setting label configuration on %s...', blr)
if url.scheme == 's3': # Uses only XML.
self.gsutil_api.XmlPassThroughSetTagging(label_text,
url,
provider=url.scheme)
else: # Must be a 'gs://' bucket.
labels_message = None
# When performing a read-modify-write cycle, include metageneration to
# avoid race conditions (supported for GS buckets only).
metageneration = None
new_label_json = json.loads(label_text)
if (self.gsutil_api.GetApiSelector(
url.scheme) == ApiSelector.JSON):
# Perform a read-modify-write so that we can specify which
# existing labels need to be deleted.
_, bucket_metadata = self.GetSingleBucketUrlFromArg(
url.url_string,
bucket_fields=['labels', 'metageneration'])
metageneration = bucket_metadata.metageneration
label_json = {}
if bucket_metadata.labels:
label_json = json.loads(
LabelTranslation.JsonFromMessage(
bucket_metadata.labels))
# Set all old keys' values to None; this will delete each key that
# is not included in the new set of labels.
merged_labels = dict(
(key, None) for key, _ in six.iteritems(label_json))
merged_labels.update(new_label_json)
labels_message = LabelTranslation.DictToMessage(
merged_labels)
else: # ApiSelector.XML
# No need to read-modify-write with the XML API.
labels_message = LabelTranslation.DictToMessage(
new_label_json)
preconditions = Preconditions(meta_gen_match=metageneration)
bucket_metadata = apitools_messages.Bucket(
labels=labels_message)
self.gsutil_api.PatchBucket(url.bucket_name,
bucket_metadata,
preconditions=preconditions,
provider=url.scheme,
fields=['id'])
def _SetPublicAccessPrevention(self, blr, setting_arg):
"""Sets the Public Access Prevention setting for a bucket enforced or inherited."""
bucket_url = blr.storage_url
iam_config = IamConfigurationValue()
iam_config.publicAccessPrevention = setting_arg
bucket_metadata = apitools_messages.Bucket(iamConfiguration=iam_config)
print('Setting Public Access Prevention %s for %s' %
(setting_arg, str(bucket_url).rstrip('/')))
self.gsutil_api.PatchBucket(bucket_url.bucket_name,
bucket_metadata,
fields=['iamConfiguration'],
provider=bucket_url.scheme)
return 0
def _Disable(self):
"""Disables logging configuration for a bucket."""
# Iterate over URLs, expanding wildcards, and disabling logging on each.
some_matched = False
for url_str in self.args:
bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id'])
for blr in bucket_iter:
url = blr.storage_url
some_matched = True
self.logger.info('Disabling logging on %s...', blr)
logging = apitools_messages.Bucket.LoggingValue()
bucket_metadata = apitools_messages.Bucket(logging=logging)
self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata,
provider=url.scheme, fields=['id'])
if not some_matched:
raise CommandException(NO_URLS_MATCHED_TARGET % list(self.args))
return 0
def _Enable(self):
"""Enables logging configuration for a bucket."""
# Disallow multi-provider 'logging set on' calls, because the schemas
# differ.
if not UrlsAreForSingleProvider(self.args):
raise CommandException(
'"logging set on" command spanning providers not '
'allowed.')
target_bucket_url = None
target_prefix = None
for opt, opt_arg in self.sub_opts:
if opt == '-b':
target_bucket_url = StorageUrlFromString(opt_arg)
if opt == '-o':
target_prefix = opt_arg
if not target_bucket_url:
raise CommandException(
'"logging set on" requires \'-b <log_bucket>\' '
'option')
if not target_bucket_url.IsBucket():
raise CommandException('-b option must specify a bucket URL.')
# Iterate over URLs, expanding wildcards and setting logging on each.
some_matched = False
for url_str in self.args:
bucket_iter = self.GetBucketUrlIterFromArg(url_str,
bucket_fields=['id'])
for blr in bucket_iter:
url = blr.storage_url
some_matched = True
self.logger.info('Enabling logging on %s...', blr)
logging = apitools_messages.Bucket.LoggingValue(
logBucket=target_bucket_url.bucket_name,
logObjectPrefix=target_prefix or url.bucket_name)
bucket_metadata = apitools_messages.Bucket(logging=logging)
self.gsutil_api.PatchBucket(url.bucket_name,
bucket_metadata,
provider=url.scheme,
fields=['id'])
if not some_matched:
raise CommandException(NO_URLS_MATCHED_TARGET % list(self.args))
return 0
def _SetCors(self):
"""Sets CORS configuration on a Google Cloud Storage bucket."""
cors_arg = self.args[0]
url_args = self.args[1:]
# Disallow multi-provider 'cors set' requests.
if not UrlsAreForSingleProvider(url_args):
raise CommandException(
'"%s" command spanning providers not allowed.' %
self.command_name)
# Open, read and parse file containing JSON document.
cors_file = open(cors_arg, 'r')
cors_txt = cors_file.read()
cors_file.close()
self.api = self.gsutil_api.GetApiSelector(
StorageUrlFromString(url_args[0]).scheme)
cors = CorsTranslation.JsonCorsToMessageEntries(cors_txt)
if not cors:
cors = REMOVE_CORS_CONFIG
# Iterate over URLs, expanding wildcards and setting the CORS on each.
some_matched = False
for url_str in url_args:
bucket_iter = self.GetBucketUrlIterFromArg(url_str,
bucket_fields=['id'])
for blr in bucket_iter:
url = blr.storage_url
some_matched = True
self.logger.info('Setting CORS on %s...', blr)
if url.scheme == 's3':
self.gsutil_api.XmlPassThroughSetCors(cors_txt,
url,
provider=url.scheme)
else:
bucket_metadata = apitools_messages.Bucket(cors=cors)
self.gsutil_api.PatchBucket(url.bucket_name,
bucket_metadata,
provider=url.scheme,
fields=['id'])
if not some_matched:
raise CommandException('No URLs matched')
return 0
def _SetBucketPolicyOnly(self, blr, setting_arg):
"""Sets the Bucket Policy Only setting for a bucket on or off."""
self._ValidateBucketListingRefAndReturnBucketName(blr)
bucket_url = blr.storage_url
iam_config = IamConfigurationValue()
iam_config.bucketPolicyOnly = BucketPolicyOnlyValue()
iam_config.bucketPolicyOnly.enabled = (setting_arg == 'on')
bucket_metadata = apitools_messages.Bucket(iamConfiguration=iam_config)
setting_verb = 'Enabling' if setting_arg == 'on' else 'Disabling'
print('%s Bucket Policy Only for %s...' %
(setting_verb, str(bucket_url).rstrip('/')))
self.gsutil_api.PatchBucket(bucket_url.bucket_name,
bucket_metadata,
fields=['iamConfiguration'],
provider=bucket_url.scheme)
return 0
def CreateBucketJson(self,
bucket_name=None,
test_objects=0,
storage_class=None,
location=None):
"""Creates a test bucket using the JSON API.
The bucket and all of its contents will be deleted after the test.
Args:
bucket_name: Create the bucket with this name. If not provided, a
temporary test bucket name is constructed.
test_objects: The number of objects that should be placed in the bucket.
Defaults to 0.
storage_class: Storage class to use. If not provided we use standard.
location: Location to use.
Returns:
Apitools Bucket for the created bucket.
"""
bucket_name = bucket_name or self.MakeTempName('bucket')
bucket_metadata = apitools_messages.Bucket(name=bucket_name.lower())
if storage_class:
bucket_metadata.storageClass = storage_class
if location:
bucket_metadata.location = location
# TODO: Add retry and exponential backoff.
bucket = self.json_api.CreateBucket(bucket_name.lower(),
metadata=bucket_metadata)
# Add bucket to list of buckets to be cleaned up.
# TODO: Clean up JSON buckets using JSON API.
self.bucket_uris.append(
boto.storage_uri('gs://%s' % (bucket_name.lower()),
suppress_consec_slashes=False))
for i in range(test_objects):
self.CreateObjectJson(bucket_name=bucket_name,
object_name=self.MakeTempName('obj'),
contents='test %d' % i)
return bucket
def _SetUbla(self, blr, setting_arg):
"""Sets the Uniform bucket-level access setting for a bucket on or off."""
self._ValidateBucketListingRefAndReturnBucketName(blr)
bucket_url = blr.storage_url
iam_config = IamConfigurationValue()
# TODO(mynameisrafe): Replace bucketPolicyOnly with uniformBucketLevelAccess
# when the property is live.
iam_config.bucketPolicyOnly = uniformBucketLevelAccessValue()
iam_config.bucketPolicyOnly.enabled = (setting_arg == 'on')
bucket_metadata = apitools_messages.Bucket(iamConfiguration=iam_config)
setting_verb = 'Enabling' if setting_arg == 'on' else 'Disabling'
print('%s Uniform bucket-level access for %s...' %
(setting_verb, str(bucket_url).rstrip('/')))
self.gsutil_api.PatchBucket(bucket_url.bucket_name,
bucket_metadata,
fields=['iamConfiguration'],
provider=bucket_url.scheme)
return 0
def RunCommand(self):
"""Command entry point for the mb command."""
bucket_policy_only = None
location = None
storage_class = None
seconds = None
if self.sub_opts:
for o, a in self.sub_opts:
if o == '-l':
location = a
elif o == '-p':
# Project IDs are sent as header values when using gs and s3 XML APIs.
InsistAscii(
a, 'Invalid non-ASCII character found in project ID')
self.project_id = a
elif o == '-c' or o == '-s':
storage_class = NormalizeStorageClass(a)
elif o == '--retention':
seconds = RetentionInSeconds(a)
elif o == '-b':
if self.gsutil_api.GetApiSelector(
'gs') != ApiSelector.JSON:
raise CommandException(
'The -b <on|off> option '
'can only be used with the JSON API')
InsistOnOrOff(
a, 'Only on and off values allowed for -b option')
bucket_policy_only = (a == 'on')
bucket_metadata = apitools_messages.Bucket(location=location,
storageClass=storage_class)
if bucket_policy_only:
bucket_metadata.iamConfiguration = IamConfigurationValue()
iam_config = bucket_metadata.iamConfiguration
iam_config.bucketPolicyOnly = BucketPolicyOnlyValue()
iam_config.bucketPolicyOnly.enabled = bucket_policy_only
for bucket_url_str in self.args:
bucket_url = StorageUrlFromString(bucket_url_str)
if seconds is not None:
if bucket_url.scheme != 'gs':
raise CommandException(
'Retention policy can only be specified for '
'GCS buckets.')
retention_policy = (
apitools_messages.Bucket.RetentionPolicyValue(
retentionPeriod=seconds))
bucket_metadata.retentionPolicy = retention_policy
if not bucket_url.IsBucket():
raise CommandException(
'The mb command requires a URL that specifies a '
'bucket.\n"%s" is not valid.' % bucket_url)
if (not BUCKET_NAME_RE.match(bucket_url.bucket_name)
or TOO_LONG_DNS_NAME_COMP.search(bucket_url.bucket_name)):
raise InvalidUrlError('Invalid bucket name in URL "%s"' %
bucket_url.bucket_name)
self.logger.info('Creating %s...', bucket_url)
# Pass storage_class param only if this is a GCS bucket. (In S3 the
# storage class is specified on the key object.)
try:
self.gsutil_api.CreateBucket(bucket_url.bucket_name,
project_id=self.project_id,
metadata=bucket_metadata,
provider=bucket_url.scheme)
except BadRequestException as e:
if (e.status == 400
and e.reason == 'DotfulBucketNameNotUnderTld'
and bucket_url.scheme == 'gs'):
bucket_name = bucket_url.bucket_name
final_comp = bucket_name[bucket_name.rfind('.') + 1:]
raise CommandException('\n'.join(
textwrap.wrap(
'Buckets with "." in the name must be valid DNS names. The bucket'
' you are attempting to create (%s) is not a valid DNS name,'
' because the final component (%s) is not currently a valid part'
' of the top-level DNS tree.' %
(bucket_name, final_comp))))
else:
raise
return 0
def ApplyAclChanges(self, name_expansion_result, thread_state=None):
"""Applies the changes in self.changes to the provided URL.
Args:
name_expansion_result: NameExpansionResult describing the target object.
thread_state: If present, gsutil Cloud API instance to apply the changes.
"""
if thread_state:
gsutil_api = thread_state
else:
gsutil_api = self.gsutil_api
url = name_expansion_result.expanded_storage_url
if url.IsBucket():
bucket = gsutil_api.GetBucket(url.bucket_name, provider=url.scheme,
fields=['acl', 'metageneration'])
current_acl = bucket.acl
elif url.IsObject():
gcs_object = encoding.JsonToMessage(apitools_messages.Object,
name_expansion_result.expanded_result)
current_acl = gcs_object.acl
if not current_acl:
self._RaiseForAccessDenied(url)
if self._ApplyAclChangesAndReturnChangeCount(url, current_acl) == 0:
self.logger.info('No changes to %s', url)
return
try:
if url.IsBucket():
preconditions = Preconditions(meta_gen_match=bucket.metageneration)
bucket_metadata = apitools_messages.Bucket(acl=current_acl)
gsutil_api.PatchBucket(url.bucket_name, bucket_metadata,
preconditions=preconditions,
provider=url.scheme, fields=['id'])
else: # Object
preconditions = Preconditions(gen_match=gcs_object.generation,
meta_gen_match=gcs_object.metageneration)
object_metadata = apitools_messages.Object(acl=current_acl)
try:
gsutil_api.PatchObjectMetadata(
url.bucket_name, url.object_name, object_metadata,
preconditions=preconditions, provider=url.scheme,
generation=url.generation, fields=['id'])
except PreconditionException as e:
# Special retry case where we want to do an additional step, the read
# of the read-modify-write cycle, to fetch the correct object
# metadata before reattempting ACL changes.
self._RefetchObjectMetadataAndApplyAclChanges(url, gsutil_api)
self.logger.info('Updated ACL on %s', url)
except BadRequestException as e:
# Don't retry on bad requests, e.g. invalid email address.
raise CommandException('Received bad request from server: %s' % str(e))
except AccessDeniedException:
self._RaiseForAccessDenied(url)
except PreconditionException as e:
# For objects, retry attempts should have already been handled.
if url.IsObject():
raise CommandException(str(e))
# For buckets, raise PreconditionException and continue to next retry.
raise e
def RunCommand(self):
"""Command entry point for the mb command."""
autoclass = False
bucket_policy_only = None
kms_key = None
location = None
storage_class = None
seconds = None
public_access_prevention = None
rpo = None
json_only_flags_in_command = []
if self.sub_opts:
for o, a in self.sub_opts:
if o == '--autoclass':
autoclass = True
json_only_flags_in_command.append(o)
elif o == '-k':
kms_key = a
ValidateCMEK(kms_key)
json_only_flags_in_command.append(o)
elif o == '-l':
location = a
elif o == '-p':
# Project IDs are sent as header values when using gs and s3 XML APIs.
InsistAscii(
a, 'Invalid non-ASCII character found in project ID')
self.project_id = a
elif o == '-c' or o == '-s':
storage_class = NormalizeStorageClass(a)
elif o == '--retention':
seconds = RetentionInSeconds(a)
elif o == '--rpo':
rpo = a.strip()
if rpo not in VALID_RPO_VALUES:
raise CommandException(
'Invalid value for --rpo. Must be one of: {},'
' provided: {}'.format(VALID_RPO_VALUES_STRING, a))
json_only_flags_in_command.append(o)
elif o == '-b':
InsistOnOrOff(
a, 'Only on and off values allowed for -b option')
bucket_policy_only = (a == 'on')
json_only_flags_in_command.append(o)
elif o == '--pap':
public_access_prevention = a
json_only_flags_in_command.append(o)
bucket_metadata = apitools_messages.Bucket(location=location,
rpo=rpo,
storageClass=storage_class)
if autoclass:
bucket_metadata.autoclass = apitools_messages.Bucket.AutoclassValue(
enabled=autoclass)
if bucket_policy_only or public_access_prevention:
bucket_metadata.iamConfiguration = IamConfigurationValue()
iam_config = bucket_metadata.iamConfiguration
if bucket_policy_only:
iam_config.bucketPolicyOnly = BucketPolicyOnlyValue()
iam_config.bucketPolicyOnly.enabled = bucket_policy_only
if public_access_prevention:
iam_config.publicAccessPrevention = public_access_prevention
if kms_key:
encryption = apitools_messages.Bucket.EncryptionValue()
encryption.defaultKmsKeyName = kms_key
bucket_metadata.encryption = encryption
for bucket_url_str in self.args:
bucket_url = StorageUrlFromString(bucket_url_str)
if seconds is not None:
if bucket_url.scheme != 'gs':
raise CommandException(
'Retention policy can only be specified for '
'GCS buckets.')
retention_policy = (
apitools_messages.Bucket.RetentionPolicyValue(
retentionPeriod=seconds))
bucket_metadata.retentionPolicy = retention_policy
if json_only_flags_in_command and self.gsutil_api.GetApiSelector(
bucket_url.scheme) != ApiSelector.JSON:
raise CommandException(
'The {} option(s) can only be used for GCS'
' Buckets with the JSON API'.format(
', '.join(json_only_flags_in_command)))
if not bucket_url.IsBucket():
raise CommandException(
'The mb command requires a URL that specifies a '
'bucket.\n"%s" is not valid.' % bucket_url)
if (not BUCKET_NAME_RE.match(bucket_url.bucket_name)
or TOO_LONG_DNS_NAME_COMP.search(bucket_url.bucket_name)):
raise InvalidUrlError('Invalid bucket name in URL "%s"' %
bucket_url.bucket_name)
self.logger.info('Creating %s...', bucket_url)
# Pass storage_class param only if this is a GCS bucket. (In S3 the
# storage class is specified on the key object.)
try:
self.gsutil_api.CreateBucket(bucket_url.bucket_name,
project_id=self.project_id,
metadata=bucket_metadata,
provider=bucket_url.scheme)
except AccessDeniedException as e:
message = e.reason
if 'key' in message:
# This will print the error reason and append the following as a
# suggested next step:
#
# To authorize, run:
# gsutil kms authorize \
# -k <kms_key> \
# -p <project_id>
message += ' To authorize, run:\n gsutil kms authorize'
message += ' \\\n -k %s' % kms_key
if (self.project_id):
message += ' \\\n -p %s' % self.project_id
raise CommandException(message)
else:
raise
except BadRequestException as e:
if (e.status == 400
and e.reason == 'DotfulBucketNameNotUnderTld'
and bucket_url.scheme == 'gs'):
bucket_name = bucket_url.bucket_name
final_comp = bucket_name[bucket_name.rfind('.') + 1:]
raise CommandException('\n'.join(
textwrap.wrap(
'Buckets with "." in the name must be valid DNS names. The bucket'
' you are attempting to create (%s) is not a valid DNS name,'
' because the final component (%s) is not currently a valid part'
' of the top-level DNS tree.' %
(bucket_name, final_comp))))
else:
raise
return 0
