class AclCommand(Command):
"""Implementation of gsutil acl command."""
# Command specification. See base class for documentation.
command_spec = Command.CreateCommandSpec(
'acl',
command_name_aliases=['getacl', 'setacl', 'chacl'],
usage_synopsis=_SYNOPSIS,
min_args=2,
max_args=NO_MAX,
supported_sub_args='afRrg:u:d:p:',
file_url_ok=False,
provider_url_ok=False,
urls_start_arg=1,
gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments={
'set': [
CommandArgument.MakeFileURLOrCannedACLArgument(),
CommandArgument.MakeZeroOrMoreCloudURLsArgument()
],
'get': [
CommandArgument.MakeNCloudURLsArgument(1)
],
'ch': [
CommandArgument.MakeZeroOrMoreCloudURLsArgument()
],
}
)
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='acl',
help_name_aliases=['getacl', 'setacl', 'chmod', 'chacl'],
help_type='command_help',
help_one_line_summary='Get, set, or change bucket and/or object ACLs',
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={
'get': _get_help_text, 'set': _set_help_text, 'ch': _ch_help_text},
)
def _CalculateUrlsStartArg(self):
if not self.args:
self.RaiseWrongNumberOfArgumentsException()
if (self.args[0].lower() == 'set') or (self.command_alias_used == 'setacl'):
return 1
else:
return 0
def _SetAcl(self):
"""Parses options and sets ACLs on the specified buckets/objects."""
self.continue_on_error = False
if self.sub_opts:
for o, unused_a in self.sub_opts:
if o == '-a':
self.all_versions = True
elif o == '-f':
self.continue_on_error = True
elif o == '-r' or o == '-R':
self.recursion_requested = True
else:
self.RaiseInvalidArgumentException()
try:
self.SetAclCommandHelper(SetAclFuncWrapper, SetAclExceptionHandler)
except AccessDeniedException, unused_e:
self._WarnServiceAccounts()
raise
if not self.everything_set_okay:
raise CommandException('ACLs for some objects could not be set.')
class BucketPolicyOnlyCommand(Command):
"""Implements the gsutil bucketpolicyonly command."""
command_spec = Command.CreateCommandSpec(
'bucketpolicyonly',
usage_synopsis=_SYNOPSIS,
min_args=2,
max_args=NO_MAX,
supported_sub_args='',
file_url_ok=False,
provider_url_ok=False,
urls_start_arg=2,
gs_api_support=[ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments={
'get': [CommandArgument.MakeNCloudURLsArgument(1),],
'set': [
CommandArgument('mode', choices=['on', 'off']),
CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()
],
})
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='bucketpolicyonly',
help_name_aliases=[],
help_type='command_help',
help_one_line_summary='Configure uniform bucket-level access',
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={
'get': _get_help_text,
'set': _set_help_text,
},
)
def _ValidateBucketListingRefAndReturnBucketName(self, blr):
if blr.storage_url.scheme != 'gs':
raise CommandException(
'The %s command can only be used with gs:// bucket URLs.' %
self.command_name)
def _GetBucketPolicyOnly(self, blr):
"""Gets the Bucket Policy Only setting for a bucket."""
self._ValidateBucketListingRefAndReturnBucketName(blr)
bucket_url = blr.storage_url
bucket_metadata = self.gsutil_api.GetBucket(bucket_url.bucket_name,
fields=['iamConfiguration'],
provider=bucket_url.scheme)
iam_config = bucket_metadata.iamConfiguration
bucket_policy_only = iam_config.bucketPolicyOnly
fields = {
'bucket': str(bucket_url).rstrip('/'),
'enabled': bucket_policy_only.enabled
}
locked_time_line = ''
if bucket_policy_only.lockedTime:
fields['locked_time'] = bucket_policy_only.lockedTime
locked_time_line = ' LockedTime: {locked_time}\n'
if bucket_policy_only:
print(('Bucket Policy Only setting for {bucket}:\n'
' Enabled: {enabled}\n' + locked_time_line).format(**fields))
def _SetBucketPolicyOnly(self, blr, setting_arg):
"""Sets the Bucket Policy Only setting for a bucket on or off."""
self._ValidateBucketListingRefAndReturnBucketName(blr)
bucket_url = blr.storage_url
iam_config = IamConfigurationValue()
iam_config.bucketPolicyOnly = BucketPolicyOnlyValue()
iam_config.bucketPolicyOnly.enabled = (setting_arg == 'on')
bucket_metadata = apitools_messages.Bucket(iamConfiguration=iam_config)
setting_verb = 'Enabling' if setting_arg == 'on' else 'Disabling'
print('%s Bucket Policy Only for %s...' %
(setting_verb, str(bucket_url).rstrip('/')))
self.gsutil_api.PatchBucket(bucket_url.bucket_name,
bucket_metadata,
fields=['iamConfiguration'],
provider=bucket_url.scheme)
return 0
def _BucketPolicyOnly(self):
"""Handles bucketpolicyonly command on a Cloud Storage bucket."""
subcommand = self.args.pop(0)
if subcommand not in ('get', 'set'):
raise CommandException('bucketpolicyonly only supports get|set')
subcommand_func = None
subcommand_args = []
setting_arg = None
if subcommand == 'get':
subcommand_func = self._GetBucketPolicyOnly
elif subcommand == 'set':
subcommand_func = self._SetBucketPolicyOnly
setting_arg = self.args.pop(0)
InsistOnOrOff(setting_arg,
'Only on and off values allowed for set option')
subcommand_args.append(setting_arg)
# Iterate over bucket args, performing the specified subsubcommand.
some_matched = False
url_args = self.args
if not url_args:
self.RaiseWrongNumberOfArgumentsException()
for url_str in url_args:
# Throws a CommandException if the argument is not a bucket.
bucket_iter = self.GetBucketUrlIterFromArg(url_str)
for bucket_listing_ref in bucket_iter:
some_matched = True
subcommand_func(bucket_listing_ref, *subcommand_args)
if not some_matched:
raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args))
return 0
def RunCommand(self):
"""Command entry point for the bucketpolicyonly command."""
if self.gsutil_api.GetApiSelector(provider='gs') != ApiSelector.JSON:
raise CommandException('\n'.join(
textwrap.wrap(
'The "%s" command can only be used with the Cloud Storage JSON API.'
% self.command_name)))
action_subcommand = self.args[0]
self.ParseSubOpts(check_args=True)
if action_subcommand == 'get' or action_subcommand == 'set':
metrics.LogCommandParams(sub_opts=self.sub_opts)
metrics.LogCommandParams(subcommands=[action_subcommand])
self._BucketPolicyOnly()
else:
raise CommandException('Invalid subcommand "%s", use get|set instead.' %
action_subcommand)
class PapCommand(Command):
"""Implements the gsutil pap command."""
command_spec = Command.CreateCommandSpec(
'pap',
command_name_aliases=['publicaccessprevention'],
usage_synopsis=_SYNOPSIS,
min_args=2,
max_args=NO_MAX,
supported_sub_args='',
file_url_ok=False,
provider_url_ok=False,
urls_start_arg=2,
gs_api_support=[ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments={
'get': [
CommandArgument.MakeNCloudURLsArgument(1),
],
'set': [
CommandArgument('mode', choices=['enforced', 'inherited']),
CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()
],
})
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='pap',
help_name_aliases=['publicaccessprevention'],
help_type='command_help',
help_one_line_summary='Configure public access prevention',
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={
'get': _get_help_text,
'set': _set_help_text,
},
)
def _ValidateBucketListingRefAndReturnBucketName(self, blr):
if blr.storage_url.scheme != 'gs':
raise CommandException(
'The %s command can only be used with gs:// bucket URLs.' %
self.command_name)
def _GetPublicAccessPrevention(self, blr):
"""Gets the public access prevention setting for a bucket."""
bucket_url = blr.storage_url
bucket_metadata = self.gsutil_api.GetBucket(
bucket_url.bucket_name,
fields=['iamConfiguration'],
provider=bucket_url.scheme)
iam_config = bucket_metadata.iamConfiguration
public_access_prevention = iam_config.publicAccessPrevention or 'inherited'
bucket = str(bucket_url).rstrip('/')
print('%s: %s' % (bucket, public_access_prevention))
def _SetPublicAccessPrevention(self, blr, setting_arg):
"""Sets the Public Access Prevention setting for a bucket enforced or inherited."""
bucket_url = blr.storage_url
iam_config = IamConfigurationValue()
iam_config.publicAccessPrevention = setting_arg
bucket_metadata = apitools_messages.Bucket(iamConfiguration=iam_config)
print('Setting Public Access Prevention %s for %s' %
(setting_arg, str(bucket_url).rstrip('/')))
self.gsutil_api.PatchBucket(bucket_url.bucket_name,
bucket_metadata,
fields=['iamConfiguration'],
provider=bucket_url.scheme)
return 0
def _Pap(self):
"""Handles pap command on Cloud Storage buckets."""
subcommand = self.args.pop(0)
if subcommand not in ('get', 'set'):
raise CommandException('pap only supports get|set')
subcommand_func = None
subcommand_args = []
setting_arg = None
if subcommand == 'get':
subcommand_func = self._GetPublicAccessPrevention
elif subcommand == 'set':
subcommand_func = self._SetPublicAccessPrevention
setting_arg = self.args.pop(0)
subcommand_args.append(setting_arg)
if self.gsutil_api.GetApiSelector('gs') != ApiSelector.JSON:
raise CommandException('\n'.join(
textwrap.wrap(
('The "%s" command can only be with the Cloud Storage '
'JSON API.') % self.command_name)))
# Iterate over bucket args, performing the specified subsubcommand.
some_matched = False
url_args = self.args
if not url_args:
self.RaiseWrongNumberOfArgumentsException()
for url_str in url_args:
# Throws a CommandException if the argument is not a bucket.
bucket_iter = self.GetBucketUrlIterFromArg(url_str)
for bucket_listing_ref in bucket_iter:
if self.gsutil_api.GetApiSelector(
bucket_listing_ref.storage_url.scheme
) != ApiSelector.JSON:
raise CommandException('\n'.join(
textwrap.wrap(
('The "%s" command can only be used for GCS '
'Buckets.') % self.command_name)))
some_matched = True
subcommand_func(bucket_listing_ref, *subcommand_args)
if not some_matched:
raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args))
return 0
def RunCommand(self):
"""Command entry point for the pap command."""
action_subcommand = self.args[0]
self.ParseSubOpts(check_args=True)
if action_subcommand == 'get' or action_subcommand == 'set':
metrics.LogCommandParams(sub_opts=self.sub_opts)
metrics.LogCommandParams(subcommands=[action_subcommand])
self._Pap()
else:
raise CommandException(
'Invalid subcommand "%s", use get|set instead.' %
action_subcommand)
class IamCommand(Command):
"""Implementation of gsutil iam command."""
command_spec = Command.CreateCommandSpec(
'iam',
min_args=2,
max_args=NO_MAX,
supported_sub_args='afRrd:e:',
file_url_ok=True,
provider_url_ok=False,
urls_start_arg=1,
gs_api_support=[ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments={
'get': [CommandArgument.MakeNCloudURLsArgument(1)],
'set': [
CommandArgument.MakeNFileURLsArgument(1),
CommandArgument.MakeZeroOrMoreCloudURLsArgument()
],
'ch': [
CommandArgument.MakeOneOrMoreBindingsArgument(),
CommandArgument.MakeZeroOrMoreCloudURLsArgument()
],
},
)
help_spec = Command.HelpSpec(
help_name='iam',
help_name_aliases=[],
help_type='command_help',
help_one_line_summary=('Get, set, or change'
' bucket and/or object IAM permissions.'),
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={
'get': _get_help_text,
'set': _set_help_text,
'ch': _ch_help_text,
})
def GetIamHelper(self, storage_url, thread_state=None):
"""Gets an IAM policy for a single, resolved bucket / object URL.
Args:
storage_url: A CloudUrl instance with no wildcards, pointing to a
specific bucket or object.
thread_state: CloudApiDelegator instance which is passed from
command.WorkerThread.__init__() if the global -m flag is
specified. Will use self.gsutil_api if thread_state is set
to None.
Returns:
Policy instance.
"""
gsutil_api = GetCloudApiInstance(self, thread_state=thread_state)
if storage_url.IsBucket():
policy = gsutil_api.GetBucketIamPolicy(
storage_url.bucket_name,
provider=storage_url.scheme,
fields=['bindings', 'etag'],
)
else:
policy = gsutil_api.GetObjectIamPolicy(
storage_url.bucket_name,
storage_url.object_name,
generation=storage_url.generation,
provider=storage_url.scheme,
fields=['bindings', 'etag'],
)
return policy
def _GetIam(self, thread_state=None):
"""Gets IAM policy for single bucket or object."""
pattern = self.args[0]
matches = PluralityCheckableIterator(
self.WildcardIterator(pattern).IterAll(
bucket_listing_fields=['name']))
if matches.IsEmpty():
raise CommandException('%s matched no URLs' % pattern)
if matches.HasPlurality():
raise CommandException(
'%s matched more than one URL, which is not allowed by the %s '
'command' % (pattern, self.command_name))
storage_url = StorageUrlFromString(list(matches)[0].url_string)
policy = self.GetIamHelper(storage_url, thread_state=thread_state)
print json.dumps(json.loads(protojson.encode_message(policy)),
sort_keys=True,
indent=2)
def _SetIamHelperInternal(self, storage_url, policy, thread_state=None):
"""Sets IAM policy for a single, resolved bucket / object URL.
Args:
storage_url: A CloudUrl instance with no wildcards, pointing to a
specific bucket or object.
policy: A Policy object to set on the bucket / object.
thread_state: CloudApiDelegator instance which is passed from
command.WorkerThread.__init__() if the -m flag is
specified. Will use self.gsutil_api if thread_state is set
to None.
Raises:
ServiceException passed from the API call if an HTTP error was returned.
"""
# SetIamHelper may be called by a command.WorkerThread. In the
# single-threaded case, WorkerThread will not pass the CloudApiDelegator
# instance to thread_state. GetCloudInstance is called to resolve the
# edge case.
gsutil_api = GetCloudApiInstance(self, thread_state=thread_state)
if storage_url.IsBucket():
gsutil_api.SetBucketIamPolicy(storage_url.bucket_name,
policy,
provider=storage_url.scheme)
else:
gsutil_api.SetObjectIamPolicy(storage_url.bucket_name,
storage_url.object_name,
policy,
generation=storage_url.generation,
provider=storage_url.scheme)
def SetIamHelper(self, storage_url, policy, thread_state=None):
"""Handles the potential exception raised by the internal set function."""
try:
self._SetIamHelperInternal(storage_url,
policy,
thread_state=thread_state)
except ServiceException:
if self.continue_on_error:
self.everything_set_okay = False
else:
raise
def PatchIamHelper(self, storage_url, bindings_tuples, thread_state=None):
"""Patches an IAM policy for a single, resolved bucket / object URL.
The patch is applied by altering the policy from an IAM get request, and
setting the new IAM with the specified etag. Because concurrent IAM set
requests may alter the etag, we may need to retry this operation several
times before success.
Args:
storage_url: A CloudUrl instance with no wildcards, pointing to a
specific bucket or object.
bindings_tuples: A list of BindingsTuple instances.
thread_state: CloudApiDelegator instance which is passed from
command.WorkerThread.__init__() if the -m flag is
specified. Will use self.gsutil_api if thread_state is set
to None.
"""
try:
self._PatchIamHelperInternal(storage_url,
bindings_tuples,
thread_state=thread_state)
except ServiceException:
if self.continue_on_error:
self.everything_set_okay = False
else:
raise
except IamChOnResourceWithConditionsException as e:
if self.continue_on_error:
self.everything_set_okay = False
self.tried_ch_on_resource_with_conditions = True
self.logger.debug(e.message)
else:
raise CommandException(e.message)
@Retry(PreconditionException, tries=3, timeout_secs=1.0)
def _PatchIamHelperInternal(self,
storage_url,
bindings_tuples,
thread_state=None):
policy = self.GetIamHelper(storage_url, thread_state=thread_state)
(etag, bindings) = (policy.etag, policy.bindings)
# If any of the bindings have conditions present, raise an exception.
# See the docstring for the IamChOnResourceWithConditionsException class
# for more details on why we raise this exception.
for binding in bindings:
if binding.condition:
message = 'Could not patch IAM policy for %s.' % storage_url
message += '\n'
message += '\n'.join(
textwrap.wrap(
'The resource had conditions present in its IAM policy bindings, '
'which is not supported by "iam ch". %s' %
IAM_CH_CONDITIONS_WORKAROUND_MSG))
raise IamChOnResourceWithConditionsException(message)
# Create a backup which is untainted by any references to the original
# bindings.
orig_bindings = list(bindings)
for (is_grant, diff) in bindings_tuples:
bindings = PatchBindings(bindings, BindingsTuple(is_grant, diff))
if IsEqualBindings(bindings, orig_bindings):
self.logger.info('No changes made to %s', storage_url)
return
policy = apitools_messages.Policy(bindings=bindings, etag=etag)
# We explicitly wish for etag mismatches to raise an error and allow this
# function to error out, so we are bypassing the exception handling offered
# by IamCommand.SetIamHelper in lieu of our own handling (@Retry).
self._SetIamHelperInternal(storage_url,
policy,
thread_state=thread_state)
def _PatchIam(self):
self.continue_on_error = False
self.recursion_requested = False
patch_bindings_tuples = []
if self.sub_opts:
for o, a in self.sub_opts:
if o in ['-r', '-R']:
self.recursion_requested = True
elif o == '-f':
self.continue_on_error = True
elif o == '-d':
patch_bindings_tuples.append(BindingStringToTuple(
False, a))
patterns = []
# N.B.: self.sub_opts stops taking in options at the first non-flagged
# token. The rest of the tokens are sent to self.args. Thus, in order to
# handle input of the form "-d <binding> <binding> <url>", we will have to
# parse self.args for a mix of both bindings and CloudUrls. We are not
# expecting to come across the -r, -f flags here.
it = iter(self.args)
for token in it:
if STORAGE_URI_REGEX.match(token):
patterns.append(token)
break
if token == '-d':
patch_bindings_tuples.append(
BindingStringToTuple(False, it.next()))
else:
patch_bindings_tuples.append(BindingStringToTuple(True, token))
if not patch_bindings_tuples:
raise CommandException('Must specify at least one binding.')
# All following arguments are urls.
for token in it:
patterns.append(token)
self.everything_set_okay = True
self.tried_ch_on_resource_with_conditions = False
threaded_wildcards = []
for pattern in patterns:
surl = StorageUrlFromString(pattern)
try:
if surl.IsBucket():
if self.recursion_requested:
surl.object = '*'
threaded_wildcards.append(surl.url_string)
else:
self.PatchIamHelper(surl, patch_bindings_tuples)
else:
threaded_wildcards.append(surl.url_string)
except AttributeError:
error_msg = 'Invalid Cloud URL "%s".' % surl.object_name
if set(surl.object_name).issubset(set('-Rrf')):
error_msg += (
' This resource handle looks like a flag, which must appear '
'before all bindings. See "gsutil help iam ch" for more details.'
)
raise CommandException(error_msg)
if threaded_wildcards:
name_expansion_iterator = NameExpansionIterator(
self.command_name,
self.debug,
self.logger,
self.gsutil_api,
threaded_wildcards,
self.recursion_requested,
all_versions=self.all_versions,
continue_on_error=self.continue_on_error
or self.parallel_operations,
bucket_listing_fields=['name'])
seek_ahead_iterator = SeekAheadNameExpansionIterator(
self.command_name,
self.debug,
self.GetSeekAheadGsutilApi(),
threaded_wildcards,
self.recursion_requested,
all_versions=self.all_versions)
serialized_bindings_tuples_it = itertools.repeat(
[SerializeBindingsTuple(t) for t in patch_bindings_tuples])
self.Apply(_PatchIamWrapper,
itertools.izip(serialized_bindings_tuples_it,
name_expansion_iterator),
_PatchIamExceptionHandler,
fail_on_error=not self.continue_on_error,
seek_ahead_iterator=seek_ahead_iterator)
self.everything_set_okay &= not GetFailureCount() > 0
# TODO: Add an error counter for files and objects.
if not self.everything_set_okay:
msg = 'Some IAM policies could not be patched.'
if self.tried_ch_on_resource_with_conditions:
msg += '\n'
msg += '\n'.join(
textwrap.wrap(
'Some resources had conditions present in their IAM policy '
'bindings, which is not supported by "iam ch". %s' %
(IAM_CH_CONDITIONS_WORKAROUND_MSG)))
raise CommandException(msg)
# TODO(iam-beta): Add an optional flag to specify etag and edit the policy
# accordingly to be passed into the helper functions.
def _SetIam(self):
"""Set IAM policy for given wildcards on the command line."""
self.continue_on_error = False
self.recursion_requested = False
self.all_versions = False
force_etag = False
etag = ''
if self.sub_opts:
for o, arg in self.sub_opts:
if o in ['-r', '-R']:
self.recursion_requested = True
elif o == '-f':
self.continue_on_error = True
elif o == '-a':
self.all_versions = True
elif o == '-e':
etag = str(arg)
force_etag = True
else:
self.RaiseInvalidArgumentException()
file_url = self.args[0]
patterns = self.args[1:]
# Load the IAM policy file and raise error if the file is invalid JSON or
# does not exist.
try:
with open(file_url, 'r') as fp:
policy = json.loads(fp.read())
except IOError:
raise ArgumentException(
'Specified IAM policy file "%s" does not exist.' % file_url)
except ValueError as e:
self.logger.debug('Invalid IAM policy file, ValueError:\n', e)
raise ArgumentException('Invalid IAM policy file "%s".' % file_url)
bindings = policy.get('bindings', [])
if not force_etag:
etag = policy.get('etag', '')
policy_json = json.dumps({'bindings': bindings, 'etag': etag})
try:
policy = protojson.decode_message(apitools_messages.Policy,
policy_json)
except DecodeError:
raise ArgumentException(
'Invalid IAM policy file "%s" or etag "%s".' %
(file_url, etag))
self.everything_set_okay = True
# This list of wildcard strings will be handled by NameExpansionIterator.
threaded_wildcards = []
for pattern in patterns:
surl = StorageUrlFromString(pattern)
if surl.IsBucket():
if self.recursion_requested:
surl.object_name = '*'
threaded_wildcards.append(surl.url_string)
else:
self.SetIamHelper(surl, policy)
else:
threaded_wildcards.append(surl.url_string)
# N.B.: If threaded_wildcards contains a non-existent bucket
# (e.g. ["gs://non-existent", "gs://existent"]), NameExpansionIterator
# will raise an exception in iter.next. This halts all iteration, even
# when -f is set. This behavior is also evident in acl set. This behavior
# also appears for any exception that will be raised when iterating over
# wildcard expansions (access denied if bucket cannot be listed, etc.).
if threaded_wildcards:
name_expansion_iterator = NameExpansionIterator(
self.command_name,
self.debug,
self.logger,
self.gsutil_api,
threaded_wildcards,
self.recursion_requested,
all_versions=self.all_versions,
continue_on_error=self.continue_on_error
or self.parallel_operations,
bucket_listing_fields=['name'])
seek_ahead_iterator = SeekAheadNameExpansionIterator(
self.command_name,
self.debug,
self.GetSeekAheadGsutilApi(),
threaded_wildcards,
self.recursion_requested,
all_versions=self.all_versions)
policy_it = itertools.repeat(protojson.encode_message(policy))
self.Apply(_SetIamWrapper,
itertools.izip(policy_it, name_expansion_iterator),
_SetIamExceptionHandler,
fail_on_error=not self.continue_on_error,
seek_ahead_iterator=seek_ahead_iterator)
self.everything_set_okay &= not GetFailureCount() > 0
# TODO: Add an error counter for files and objects.
if not self.everything_set_okay:
raise CommandException('Some IAM policies could not be set.')
def RunCommand(self):
"""Command entry point for the acl command."""
action_subcommand = self.args.pop(0)
self.ParseSubOpts(check_args=True)
# Commands with both suboptions and subcommands need to reparse for
# suboptions, so we log again.
LogCommandParams(sub_opts=self.sub_opts)
self.def_acl = False
if action_subcommand == 'get':
LogCommandParams(subcommands=[action_subcommand])
self._GetIam()
elif action_subcommand == 'set':
LogCommandParams(subcommands=[action_subcommand])
self._SetIam()
elif action_subcommand == 'ch':
LogCommandParams(subcommands=[action_subcommand])
self._PatchIam()
else:
raise CommandException(
'Invalid subcommand "%s" for the %s command.\n'
'See "gsutil help iam".' %
(action_subcommand, self.command_name))
return 0
class LabelCommand(Command):
"""Implementation of gsutil label command."""
# Command specification. See base class for documentation.
command_spec = Command.CreateCommandSpec(
'label',
usage_synopsis=_SYNOPSIS,
min_args=2,
max_args=NO_MAX,
supported_sub_args='l:d:',
file_url_ok=False,
provider_url_ok=False,
urls_start_arg=1,
gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments={
'set': [
CommandArgument.MakeNFileURLsArgument(1),
CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(),
],
'get': [
CommandArgument.MakeNCloudURLsArgument(1),
],
'ch': [
CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(),
],
},
)
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='label',
help_name_aliases=[],
help_type='command_help',
help_one_line_summary=(
'Get, set, or change the label configuration of a bucket.'),
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={
'get': _get_help_text,
'set': _set_help_text,
'ch': _ch_help_text,
},
)
def _CalculateUrlsStartArg(self):
if not self.args:
self.RaiseWrongNumberOfArgumentsException()
if self.args[0].lower() == 'set':
return 2 # Filename comes before bucket arg(s).
return 1
def _SetLabel(self):
"""Parses options and sets labels on the specified buckets."""
# At this point, "set" has been popped off the front of self.args.
if len(self.args) < 2:
self.RaiseWrongNumberOfArgumentsException()
label_filename = self.args[0]
if not os.path.isfile(label_filename):
raise CommandException('Could not find the file "%s".' %
label_filename)
with codecs.open(label_filename, 'r', UTF8) as label_file:
label_text = label_file.read()
@Retry(PreconditionException, tries=3, timeout_secs=1)
def _SetLabelForBucket(blr):
url = blr.storage_url
self.logger.info('Setting label configuration on %s...', blr)
if url.scheme == 's3': # Uses only XML.
self.gsutil_api.XmlPassThroughSetTagging(label_text,
url,
provider=url.scheme)
else: # Must be a 'gs://' bucket.
labels_message = None
# When performing a read-modify-write cycle, include metageneration to
# avoid race conditions (supported for GS buckets only).
metageneration = None
new_label_json = json.loads(label_text)
if (self.gsutil_api.GetApiSelector(
url.scheme) == ApiSelector.JSON):
# Perform a read-modify-write so that we can specify which
# existing labels need to be deleted.
_, bucket_metadata = self.GetSingleBucketUrlFromArg(
url.url_string,
bucket_fields=['labels', 'metageneration'])
metageneration = bucket_metadata.metageneration
label_json = {}
if bucket_metadata.labels:
label_json = json.loads(
LabelTranslation.JsonFromMessage(
bucket_metadata.labels))
# Set all old keys' values to None; this will delete each key that
# is not included in the new set of labels.
merged_labels = dict(
(key, None) for key, _ in six.iteritems(label_json))
merged_labels.update(new_label_json)
labels_message = LabelTranslation.DictToMessage(
merged_labels)
else: # ApiSelector.XML
# No need to read-modify-write with the XML API.
labels_message = LabelTranslation.DictToMessage(
new_label_json)
preconditions = Preconditions(meta_gen_match=metageneration)
bucket_metadata = apitools_messages.Bucket(
labels=labels_message)
self.gsutil_api.PatchBucket(url.bucket_name,
bucket_metadata,
preconditions=preconditions,
provider=url.scheme,
fields=['id'])
some_matched = False
url_args = self.args[1:]
for url_str in url_args:
# Throws a CommandException if the argument is not a bucket.
bucket_iter = self.GetBucketUrlIterFromArg(url_str,
bucket_fields=['id'])
for bucket_listing_ref in bucket_iter:
some_matched = True
_SetLabelForBucket(bucket_listing_ref)
if not some_matched:
raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args))
def _ChLabel(self):
"""Parses options and changes labels on the specified buckets."""
self.label_changes = {}
self.num_deletions = 0
if self.sub_opts:
for o, a in self.sub_opts:
if o == '-l':
label_split = a.split(':')
if len(label_split) != 2:
raise CommandException(
'Found incorrectly formatted option for "gsutil label ch": '
'"%s". To add a label, please use the form <key>:<value>.'
% a)
self.label_changes[label_split[0]] = label_split[1]
elif o == '-d':
# Ensure only the key is supplied; stop if key:value was given.
val_split = a.split(':')
if len(val_split) != 1:
raise CommandException(
'Found incorrectly formatted option for "gsutil label ch": '
'"%s". To delete a label, provide only its key.' %
a)
self.label_changes[a] = None
self.num_deletions += 1
else:
self.RaiseInvalidArgumentException()
if not self.label_changes:
raise CommandException(
'Please specify at least one label change with the -l or -d flags.'
)
@Retry(PreconditionException, tries=3, timeout_secs=1)
def _ChLabelForBucket(blr):
url = blr.storage_url
self.logger.info('Setting label configuration on %s...', blr)
labels_message = None
# When performing a read-modify-write cycle, include metageneration to
# avoid race conditions (supported for GS buckets only).
metageneration = None
if (self.gsutil_api.GetApiSelector(
url.scheme) == ApiSelector.JSON):
# The JSON API's PATCH semantics allow us to skip read-modify-write,
# with the exception of one edge case - attempting to delete a
# nonexistent label returns an error iff no labels previously existed
corrected_changes = self.label_changes
if self.num_deletions:
(_, bucket_metadata) = self.GetSingleBucketUrlFromArg(
url.url_string,
bucket_fields=['labels', 'metageneration'])
if not bucket_metadata.labels:
metageneration = bucket_metadata.metageneration
# Remove each change that would try to delete a nonexistent key.
corrected_changes = dict(
(k, v)
for k, v in six.iteritems(self.label_changes) if v)
labels_message = LabelTranslation.DictToMessage(
corrected_changes)
else: # ApiSelector.XML
# Perform a read-modify-write cycle so that we can specify which
# existing labels need to be deleted.
(_, bucket_metadata) = self.GetSingleBucketUrlFromArg(
url.url_string, bucket_fields=['labels', 'metageneration'])
metageneration = bucket_metadata.metageneration
label_json = {}
if bucket_metadata.labels:
label_json = json.loads(
LabelTranslation.JsonFromMessage(
bucket_metadata.labels))
# Modify label_json such that all specified labels are added
# (overwriting old labels if necessary) and all specified deletions
# are removed from label_json if already present.
for key, value in six.iteritems(self.label_changes):
if not value and key in label_json:
del label_json[key]
else:
label_json[key] = value
labels_message = LabelTranslation.DictToMessage(label_json)
preconditions = Preconditions(meta_gen_match=metageneration)
bucket_metadata = apitools_messages.Bucket(labels=labels_message)
self.gsutil_api.PatchBucket(url.bucket_name,
bucket_metadata,
preconditions=preconditions,
provider=url.scheme,
fields=['id'])
some_matched = False
url_args = self.args
if not url_args:
self.RaiseWrongNumberOfArgumentsException()
for url_str in url_args:
# Throws a CommandException if the argument is not a bucket.
bucket_iter = self.GetBucketUrlIterFromArg(url_str)
for bucket_listing_ref in bucket_iter:
some_matched = True
_ChLabelForBucket(bucket_listing_ref)
if not some_matched:
raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args))
def _GetAndPrintLabel(self, bucket_arg):
"""Gets and prints the labels for a cloud bucket."""
bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg(
bucket_arg, bucket_fields=['labels'])
if bucket_url.scheme == 's3':
print((self.gsutil_api.XmlPassThroughGetTagging(
bucket_url, provider=bucket_url.scheme)))
else:
if bucket_metadata.labels:
print((LabelTranslation.JsonFromMessage(bucket_metadata.labels,
pretty_print=True)))
else:
print(('%s has no label configuration.' % bucket_url))
def RunCommand(self):
"""Command entry point for the label command."""
action_subcommand = self.args.pop(0)
self.ParseSubOpts(check_args=True)
# Commands with both suboptions and subcommands need to reparse for
# suboptions, so we log again.
metrics.LogCommandParams(sub_opts=self.sub_opts)
if action_subcommand == 'get':
metrics.LogCommandParams(subcommands=[action_subcommand])
self._GetAndPrintLabel(self.args[0])
elif action_subcommand == 'set':
metrics.LogCommandParams(subcommands=[action_subcommand])
self._SetLabel()
elif action_subcommand == 'ch':
metrics.LogCommandParams(subcommands=[action_subcommand])
self._ChLabel()
else:
raise CommandException(
'Invalid subcommand "%s" for the %s command.\nSee "gsutil help %s".'
% (action_subcommand, self.command_name, self.command_name))
return 0
class AclCommand(Command):
"""Implementation of gsutil acl command."""
# Command specification. See base class for documentation.
command_spec = Command.CreateCommandSpec(
'acl',
command_name_aliases=['getacl', 'setacl', 'chacl'],
usage_synopsis=_SYNOPSIS,
min_args=2,
max_args=NO_MAX,
supported_sub_args='afRrg:u:d:p:',
file_url_ok=False,
provider_url_ok=False,
urls_start_arg=1,
gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments={
'set': [
CommandArgument.MakeFileURLOrCannedACLArgument(),
CommandArgument.MakeZeroOrMoreCloudURLsArgument()
],
'get': [CommandArgument.MakeNCloudURLsArgument(1)],
'ch': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()],
})
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='acl',
help_name_aliases=['getacl', 'setacl', 'chmod', 'chacl'],
help_type='command_help',
help_one_line_summary='Get, set, or change bucket and/or object ACLs',
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={
'get': _get_help_text,
'set': _set_help_text,
'ch': _ch_help_text
},
)
def _CalculateUrlsStartArg(self):
if not self.args:
self.RaiseWrongNumberOfArgumentsException()
if (self.args[0].lower() == 'set') or (self.command_alias_used
== 'setacl'):
return 1
else:
return 0
def _SetAcl(self):
"""Parses options and sets ACLs on the specified buckets/objects."""
self.continue_on_error = False
if self.sub_opts:
for o, unused_a in self.sub_opts:
if o == '-a':
self.all_versions = True
elif o == '-f':
self.continue_on_error = True
elif o == '-r' or o == '-R':
self.recursion_requested = True
else:
self.RaiseInvalidArgumentException()
try:
self.SetAclCommandHelper(SetAclFuncWrapper, SetAclExceptionHandler)
except AccessDeniedException as unused_e:
self._WarnServiceAccounts()
raise
if not self.everything_set_okay:
raise CommandException('ACLs for some objects could not be set.')
def _ChAcl(self):
"""Parses options and changes ACLs on the specified buckets/objects."""
self.parse_versions = True
self.changes = []
self.continue_on_error = False
if self.sub_opts:
for o, a in self.sub_opts:
if o == '-f':
self.continue_on_error = True
elif o == '-g':
if 'gserviceaccount.com' in a:
raise CommandException(
'Service accounts are considered users, not groups; please use '
'"gsutil acl ch -u" instead of "gsutil acl ch -g"')
self.changes.append(
acl_helper.AclChange(
a, scope_type=acl_helper.ChangeType.GROUP))
elif o == '-p':
self.changes.append(
acl_helper.AclChange(
a, scope_type=acl_helper.ChangeType.PROJECT))
elif o == '-u':
self.changes.append(
acl_helper.AclChange(
a, scope_type=acl_helper.ChangeType.USER))
elif o == '-d':
self.changes.append(acl_helper.AclDel(a))
elif o == '-r' or o == '-R':
self.recursion_requested = True
else:
self.RaiseInvalidArgumentException()
if not self.changes:
raise CommandException('Please specify at least one access change '
'with the -g, -u, or -d flags')
if (not UrlsAreForSingleProvider(self.args)
or StorageUrlFromString(self.args[0]).scheme != 'gs'):
raise CommandException(
'The "{0}" command can only be used with gs:// URLs'.format(
self.command_name))
self.everything_set_okay = True
self.ApplyAclFunc(
_ApplyAclChangesWrapper,
_ApplyExceptionHandler,
self.args,
object_fields=['acl', 'generation', 'metageneration'])
if not self.everything_set_okay:
raise CommandException('ACLs for some objects could not be set.')
def _RaiseForAccessDenied(self, url):
self._WarnServiceAccounts()
raise CommandException(
'Failed to set acl for %s. Please ensure you have '
'OWNER-role access to this resource.' % url)
@Retry(ServiceException, tries=3, timeout_secs=1)
def ApplyAclChanges(self, name_expansion_result, thread_state=None):
"""Applies the changes in self.changes to the provided URL.
Args:
name_expansion_result: NameExpansionResult describing the target object.
thread_state: If present, gsutil Cloud API instance to apply the changes.
"""
if thread_state:
gsutil_api = thread_state
else:
gsutil_api = self.gsutil_api
url = name_expansion_result.expanded_storage_url
if url.IsBucket():
bucket = gsutil_api.GetBucket(url.bucket_name,
provider=url.scheme,
fields=['acl', 'metageneration'])
current_acl = bucket.acl
elif url.IsObject():
gcs_object = encoding.JsonToMessage(
apitools_messages.Object,
name_expansion_result.expanded_result)
current_acl = gcs_object.acl
if not current_acl:
self._RaiseForAccessDenied(url)
if self._ApplyAclChangesAndReturnChangeCount(url, current_acl) == 0:
self.logger.info('No changes to %s', url)
return
try:
if url.IsBucket():
preconditions = Preconditions(
meta_gen_match=bucket.metageneration)
bucket_metadata = apitools_messages.Bucket(acl=current_acl)
gsutil_api.PatchBucket(url.bucket_name,
bucket_metadata,
preconditions=preconditions,
provider=url.scheme,
fields=['id'])
else: # Object
preconditions = Preconditions(
gen_match=gcs_object.generation,
meta_gen_match=gcs_object.metageneration)
object_metadata = apitools_messages.Object(acl=current_acl)
try:
gsutil_api.PatchObjectMetadata(url.bucket_name,
url.object_name,
object_metadata,
preconditions=preconditions,
provider=url.scheme,
generation=url.generation,
fields=['id'])
except PreconditionException as e:
# Special retry case where we want to do an additional step, the read
# of the read-modify-write cycle, to fetch the correct object
# metadata before reattempting ACL changes.
self._RefetchObjectMetadataAndApplyAclChanges(
url, gsutil_api)
self.logger.info('Updated ACL on %s', url)
except BadRequestException as e:
# Don't retry on bad requests, e.g. invalid email address.
raise CommandException('Received bad request from server: %s' %
str(e))
except AccessDeniedException:
self._RaiseForAccessDenied(url)
except PreconditionException as e:
# For objects, retry attempts should have already been handled.
if url.IsObject():
raise CommandException(str(e))
# For buckets, raise PreconditionException and continue to next retry.
raise e
@Retry(PreconditionException, tries=3, timeout_secs=1)
def _RefetchObjectMetadataAndApplyAclChanges(self, url, gsutil_api):
"""Reattempts object ACL changes after a PreconditionException."""
gcs_object = gsutil_api.GetObjectMetadata(
url.bucket_name,
url.object_name,
provider=url.scheme,
fields=['acl', 'generation', 'metageneration'])
current_acl = gcs_object.acl
if self._ApplyAclChangesAndReturnChangeCount(url, current_acl) == 0:
self.logger.info('No changes to %s', url)
return
object_metadata = apitools_messages.Object(acl=current_acl)
preconditions = Preconditions(gen_match=gcs_object.generation,
meta_gen_match=gcs_object.metageneration)
gsutil_api.PatchObjectMetadata(url.bucket_name,
url.object_name,
object_metadata,
preconditions=preconditions,
provider=url.scheme,
generation=gcs_object.generation,
fields=['id'])
def _ApplyAclChangesAndReturnChangeCount(self, storage_url, acl_message):
modification_count = 0
for change in self.changes:
modification_count += change.Execute(storage_url, acl_message,
'acl', self.logger)
return modification_count
def RunCommand(self):
"""Command entry point for the acl command."""
action_subcommand = self.args.pop(0)
self.ParseSubOpts(check_args=True)
# Commands with both suboptions and subcommands need to reparse for
# suboptions, so we log again.
metrics.LogCommandParams(sub_opts=self.sub_opts)
self.def_acl = False
if action_subcommand == 'get':
metrics.LogCommandParams(subcommands=[action_subcommand])
self.GetAndPrintAcl(self.args[0])
elif action_subcommand == 'set':
metrics.LogCommandParams(subcommands=[action_subcommand])
self._SetAcl()
elif action_subcommand in ('ch', 'change'):
metrics.LogCommandParams(subcommands=[action_subcommand])
self._ChAcl()
else:
raise CommandException(
('Invalid subcommand "%s" for the %s command.\n'
'See "gsutil help acl".') %
(action_subcommand, self.command_name))
return 0