class CorsCommand(Command):
"""Implementation of gsutil cors command."""
# Command specification. See base class for documentation.
command_spec = Command.CreateCommandSpec(
'cors',
command_name_aliases=['getcors', 'setcors'],
usage_synopsis=_SYNOPSIS,
min_args=2,
max_args=NO_MAX,
supported_sub_args='',
file_url_ok=False,
provider_url_ok=False,
urls_start_arg=1,
gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments={
'set': [
CommandArgument.MakeNFileURLsArgument(1),
CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()
],
'get': [CommandArgument.MakeNCloudBucketURLsArgument(1)]
})
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='cors',
help_name_aliases=['getcors', 'setcors', 'cross-origin'],
help_type='command_help',
help_one_line_summary=(
'Set a CORS JSON document for one or more buckets'),
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={
'get': _get_help_text,
'set': _set_help_text
},
)
def _CalculateUrlsStartArg(self):
if not self.args:
self.RaiseWrongNumberOfArgumentsException()
if self.args[0].lower() == 'set':
return 2
else:
return 1
def _SetCors(self):
"""Sets CORS configuration on a Google Cloud Storage bucket."""
cors_arg = self.args[0]
url_args = self.args[1:]
# Disallow multi-provider 'cors set' requests.
if not UrlsAreForSingleProvider(url_args):
raise CommandException(
'"%s" command spanning providers not allowed.' %
self.command_name)
# Open, read and parse file containing JSON document.
cors_file = open(cors_arg, 'r')
cors_txt = cors_file.read()
cors_file.close()
self.api = self.gsutil_api.GetApiSelector(
StorageUrlFromString(url_args[0]).scheme)
# Iterate over URLs, expanding wildcards and setting the CORS on each.
some_matched = False
for url_str in url_args:
bucket_iter = self.GetBucketUrlIterFromArg(url_str,
bucket_fields=['id'])
for blr in bucket_iter:
url = blr.storage_url
some_matched = True
self.logger.info('Setting CORS on %s...', blr)
if url.scheme == 's3':
self.gsutil_api.XmlPassThroughSetCors(cors_txt,
url,
provider=url.scheme)
else:
cors = CorsTranslation.JsonCorsToMessageEntries(cors_txt)
if not cors:
cors = REMOVE_CORS_CONFIG
bucket_metadata = apitools_messages.Bucket(cors=cors)
self.gsutil_api.PatchBucket(url.bucket_name,
bucket_metadata,
provider=url.scheme,
fields=['id'])
if not some_matched:
raise CommandException('No URLs matched')
return 0
def _GetCors(self):
"""Gets CORS configuration for a Google Cloud Storage bucket."""
bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg(
self.args[0], bucket_fields=['cors'])
if bucket_url.scheme == 's3':
sys.stdout.write(
self.gsutil_api.XmlPassThroughGetCors(
bucket_url, provider=bucket_url.scheme))
else:
if bucket_metadata.cors:
sys.stdout.write(
CorsTranslation.MessageEntriesToJson(bucket_metadata.cors))
else:
sys.stdout.write('%s has no CORS configuration.\n' %
bucket_url)
return 0
def RunCommand(self):
"""Command entry point for the cors command."""
action_subcommand = self.args.pop(0)
if action_subcommand == 'get':
func = self._GetCors
elif action_subcommand == 'set':
func = self._SetCors
else:
raise CommandException(
('Invalid subcommand "%s" for the %s command.\n'
'See "gsutil help cors".') %
(action_subcommand, self.command_name))
return func()
class IamCommand(Command):
"""Implementation of gsutil iam command."""
command_spec = Command.CreateCommandSpec(
'iam',
min_args=2,
max_args=NO_MAX,
supported_sub_args='afRrd:e:',
file_url_ok=True,
provider_url_ok=False,
urls_start_arg=1,
gs_api_support=[ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments={
'get': [CommandArgument.MakeNCloudURLsArgument(1)],
'set': [
CommandArgument.MakeNFileURLsArgument(1),
CommandArgument.MakeZeroOrMoreCloudURLsArgument()
],
'ch': [
CommandArgument.MakeOneOrMoreBindingsArgument(),
CommandArgument.MakeZeroOrMoreCloudURLsArgument()
],
},
)
help_spec = Command.HelpSpec(
help_name='iam',
help_name_aliases=[],
help_type='command_help',
help_one_line_summary=('Get, set, or change'
' bucket and/or object IAM permissions.'),
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={
'get': _get_help_text,
'set': _set_help_text,
'ch': _ch_help_text,
})
def GetIamHelper(self, storage_url, thread_state=None):
"""Gets an IAM policy for a single, resolved bucket / object URL.
Args:
storage_url: A CloudUrl instance with no wildcards, pointing to a
specific bucket or object.
thread_state: CloudApiDelegator instance which is passed from
command.WorkerThread.__init__() if the global -m flag is
specified. Will use self.gsutil_api if thread_state is set
to None.
Returns:
Policy instance.
"""
gsutil_api = GetCloudApiInstance(self, thread_state=thread_state)
if storage_url.IsBucket():
policy = gsutil_api.GetBucketIamPolicy(
storage_url.bucket_name,
provider=storage_url.scheme,
fields=['bindings', 'etag'],
)
else:
policy = gsutil_api.GetObjectIamPolicy(
storage_url.bucket_name,
storage_url.object_name,
generation=storage_url.generation,
provider=storage_url.scheme,
fields=['bindings', 'etag'],
)
return policy
def _GetIam(self, thread_state=None):
"""Gets IAM policy for single bucket or object."""
pattern = self.args[0]
matches = PluralityCheckableIterator(
self.WildcardIterator(pattern).IterAll(
bucket_listing_fields=['name']))
if matches.IsEmpty():
raise CommandException('%s matched no URLs' % pattern)
if matches.HasPlurality():
raise CommandException(
'%s matched more than one URL, which is not allowed by the %s '
'command' % (pattern, self.command_name))
storage_url = StorageUrlFromString(list(matches)[0].url_string)
policy = self.GetIamHelper(storage_url, thread_state=thread_state)
print json.dumps(json.loads(protojson.encode_message(policy)),
sort_keys=True,
indent=2)
def _SetIamHelperInternal(self, storage_url, policy, thread_state=None):
"""Sets IAM policy for a single, resolved bucket / object URL.
Args:
storage_url: A CloudUrl instance with no wildcards, pointing to a
specific bucket or object.
policy: A Policy object to set on the bucket / object.
thread_state: CloudApiDelegator instance which is passed from
command.WorkerThread.__init__() if the -m flag is
specified. Will use self.gsutil_api if thread_state is set
to None.
Raises:
ServiceException passed from the API call if an HTTP error was returned.
"""
# SetIamHelper may be called by a command.WorkerThread. In the
# single-threaded case, WorkerThread will not pass the CloudApiDelegator
# instance to thread_state. GetCloudInstance is called to resolve the
# edge case.
gsutil_api = GetCloudApiInstance(self, thread_state=thread_state)
if storage_url.IsBucket():
gsutil_api.SetBucketIamPolicy(storage_url.bucket_name,
policy,
provider=storage_url.scheme)
else:
gsutil_api.SetObjectIamPolicy(storage_url.bucket_name,
storage_url.object_name,
policy,
generation=storage_url.generation,
provider=storage_url.scheme)
def SetIamHelper(self, storage_url, policy, thread_state=None):
"""Handles the potential exception raised by the internal set function."""
try:
self._SetIamHelperInternal(storage_url,
policy,
thread_state=thread_state)
except ServiceException:
if self.continue_on_error:
self.everything_set_okay = False
else:
raise
def PatchIamHelper(self, storage_url, bindings_tuples, thread_state=None):
"""Patches an IAM policy for a single, resolved bucket / object URL.
The patch is applied by altering the policy from an IAM get request, and
setting the new IAM with the specified etag. Because concurrent IAM set
requests may alter the etag, we may need to retry this operation several
times before success.
Args:
storage_url: A CloudUrl instance with no wildcards, pointing to a
specific bucket or object.
bindings_tuples: A list of BindingsTuple instances.
thread_state: CloudApiDelegator instance which is passed from
command.WorkerThread.__init__() if the -m flag is
specified. Will use self.gsutil_api if thread_state is set
to None.
"""
try:
self._PatchIamHelperInternal(storage_url,
bindings_tuples,
thread_state=thread_state)
except ServiceException:
if self.continue_on_error:
self.everything_set_okay = False
else:
raise
except IamChOnResourceWithConditionsException as e:
if self.continue_on_error:
self.everything_set_okay = False
self.tried_ch_on_resource_with_conditions = True
self.logger.debug(e.message)
else:
raise CommandException(e.message)
@Retry(PreconditionException, tries=3, timeout_secs=1.0)
def _PatchIamHelperInternal(self,
storage_url,
bindings_tuples,
thread_state=None):
policy = self.GetIamHelper(storage_url, thread_state=thread_state)
(etag, bindings) = (policy.etag, policy.bindings)
# If any of the bindings have conditions present, raise an exception.
# See the docstring for the IamChOnResourceWithConditionsException class
# for more details on why we raise this exception.
for binding in bindings:
if binding.condition:
message = 'Could not patch IAM policy for %s.' % storage_url
message += '\n'
message += '\n'.join(
textwrap.wrap(
'The resource had conditions present in its IAM policy bindings, '
'which is not supported by "iam ch". %s' %
IAM_CH_CONDITIONS_WORKAROUND_MSG))
raise IamChOnResourceWithConditionsException(message)
# Create a backup which is untainted by any references to the original
# bindings.
orig_bindings = list(bindings)
for (is_grant, diff) in bindings_tuples:
bindings = PatchBindings(bindings, BindingsTuple(is_grant, diff))
if IsEqualBindings(bindings, orig_bindings):
self.logger.info('No changes made to %s', storage_url)
return
policy = apitools_messages.Policy(bindings=bindings, etag=etag)
# We explicitly wish for etag mismatches to raise an error and allow this
# function to error out, so we are bypassing the exception handling offered
# by IamCommand.SetIamHelper in lieu of our own handling (@Retry).
self._SetIamHelperInternal(storage_url,
policy,
thread_state=thread_state)
def _PatchIam(self):
self.continue_on_error = False
self.recursion_requested = False
patch_bindings_tuples = []
if self.sub_opts:
for o, a in self.sub_opts:
if o in ['-r', '-R']:
self.recursion_requested = True
elif o == '-f':
self.continue_on_error = True
elif o == '-d':
patch_bindings_tuples.append(BindingStringToTuple(
False, a))
patterns = []
# N.B.: self.sub_opts stops taking in options at the first non-flagged
# token. The rest of the tokens are sent to self.args. Thus, in order to
# handle input of the form "-d <binding> <binding> <url>", we will have to
# parse self.args for a mix of both bindings and CloudUrls. We are not
# expecting to come across the -r, -f flags here.
it = iter(self.args)
for token in it:
if STORAGE_URI_REGEX.match(token):
patterns.append(token)
break
if token == '-d':
patch_bindings_tuples.append(
BindingStringToTuple(False, it.next()))
else:
patch_bindings_tuples.append(BindingStringToTuple(True, token))
if not patch_bindings_tuples:
raise CommandException('Must specify at least one binding.')
# All following arguments are urls.
for token in it:
patterns.append(token)
self.everything_set_okay = True
self.tried_ch_on_resource_with_conditions = False
threaded_wildcards = []
for pattern in patterns:
surl = StorageUrlFromString(pattern)
try:
if surl.IsBucket():
if self.recursion_requested:
surl.object = '*'
threaded_wildcards.append(surl.url_string)
else:
self.PatchIamHelper(surl, patch_bindings_tuples)
else:
threaded_wildcards.append(surl.url_string)
except AttributeError:
error_msg = 'Invalid Cloud URL "%s".' % surl.object_name
if set(surl.object_name).issubset(set('-Rrf')):
error_msg += (
' This resource handle looks like a flag, which must appear '
'before all bindings. See "gsutil help iam ch" for more details.'
)
raise CommandException(error_msg)
if threaded_wildcards:
name_expansion_iterator = NameExpansionIterator(
self.command_name,
self.debug,
self.logger,
self.gsutil_api,
threaded_wildcards,
self.recursion_requested,
all_versions=self.all_versions,
continue_on_error=self.continue_on_error
or self.parallel_operations,
bucket_listing_fields=['name'])
seek_ahead_iterator = SeekAheadNameExpansionIterator(
self.command_name,
self.debug,
self.GetSeekAheadGsutilApi(),
threaded_wildcards,
self.recursion_requested,
all_versions=self.all_versions)
serialized_bindings_tuples_it = itertools.repeat(
[SerializeBindingsTuple(t) for t in patch_bindings_tuples])
self.Apply(_PatchIamWrapper,
itertools.izip(serialized_bindings_tuples_it,
name_expansion_iterator),
_PatchIamExceptionHandler,
fail_on_error=not self.continue_on_error,
seek_ahead_iterator=seek_ahead_iterator)
self.everything_set_okay &= not GetFailureCount() > 0
# TODO: Add an error counter for files and objects.
if not self.everything_set_okay:
msg = 'Some IAM policies could not be patched.'
if self.tried_ch_on_resource_with_conditions:
msg += '\n'
msg += '\n'.join(
textwrap.wrap(
'Some resources had conditions present in their IAM policy '
'bindings, which is not supported by "iam ch". %s' %
(IAM_CH_CONDITIONS_WORKAROUND_MSG)))
raise CommandException(msg)
# TODO(iam-beta): Add an optional flag to specify etag and edit the policy
# accordingly to be passed into the helper functions.
def _SetIam(self):
"""Set IAM policy for given wildcards on the command line."""
self.continue_on_error = False
self.recursion_requested = False
self.all_versions = False
force_etag = False
etag = ''
if self.sub_opts:
for o, arg in self.sub_opts:
if o in ['-r', '-R']:
self.recursion_requested = True
elif o == '-f':
self.continue_on_error = True
elif o == '-a':
self.all_versions = True
elif o == '-e':
etag = str(arg)
force_etag = True
else:
self.RaiseInvalidArgumentException()
file_url = self.args[0]
patterns = self.args[1:]
# Load the IAM policy file and raise error if the file is invalid JSON or
# does not exist.
try:
with open(file_url, 'r') as fp:
policy = json.loads(fp.read())
except IOError:
raise ArgumentException(
'Specified IAM policy file "%s" does not exist.' % file_url)
except ValueError as e:
self.logger.debug('Invalid IAM policy file, ValueError:\n', e)
raise ArgumentException('Invalid IAM policy file "%s".' % file_url)
bindings = policy.get('bindings', [])
if not force_etag:
etag = policy.get('etag', '')
policy_json = json.dumps({'bindings': bindings, 'etag': etag})
try:
policy = protojson.decode_message(apitools_messages.Policy,
policy_json)
except DecodeError:
raise ArgumentException(
'Invalid IAM policy file "%s" or etag "%s".' %
(file_url, etag))
self.everything_set_okay = True
# This list of wildcard strings will be handled by NameExpansionIterator.
threaded_wildcards = []
for pattern in patterns:
surl = StorageUrlFromString(pattern)
if surl.IsBucket():
if self.recursion_requested:
surl.object_name = '*'
threaded_wildcards.append(surl.url_string)
else:
self.SetIamHelper(surl, policy)
else:
threaded_wildcards.append(surl.url_string)
# N.B.: If threaded_wildcards contains a non-existent bucket
# (e.g. ["gs://non-existent", "gs://existent"]), NameExpansionIterator
# will raise an exception in iter.next. This halts all iteration, even
# when -f is set. This behavior is also evident in acl set. This behavior
# also appears for any exception that will be raised when iterating over
# wildcard expansions (access denied if bucket cannot be listed, etc.).
if threaded_wildcards:
name_expansion_iterator = NameExpansionIterator(
self.command_name,
self.debug,
self.logger,
self.gsutil_api,
threaded_wildcards,
self.recursion_requested,
all_versions=self.all_versions,
continue_on_error=self.continue_on_error
or self.parallel_operations,
bucket_listing_fields=['name'])
seek_ahead_iterator = SeekAheadNameExpansionIterator(
self.command_name,
self.debug,
self.GetSeekAheadGsutilApi(),
threaded_wildcards,
self.recursion_requested,
all_versions=self.all_versions)
policy_it = itertools.repeat(protojson.encode_message(policy))
self.Apply(_SetIamWrapper,
itertools.izip(policy_it, name_expansion_iterator),
_SetIamExceptionHandler,
fail_on_error=not self.continue_on_error,
seek_ahead_iterator=seek_ahead_iterator)
self.everything_set_okay &= not GetFailureCount() > 0
# TODO: Add an error counter for files and objects.
if not self.everything_set_okay:
raise CommandException('Some IAM policies could not be set.')
def RunCommand(self):
"""Command entry point for the acl command."""
action_subcommand = self.args.pop(0)
self.ParseSubOpts(check_args=True)
# Commands with both suboptions and subcommands need to reparse for
# suboptions, so we log again.
LogCommandParams(sub_opts=self.sub_opts)
self.def_acl = False
if action_subcommand == 'get':
LogCommandParams(subcommands=[action_subcommand])
self._GetIam()
elif action_subcommand == 'set':
LogCommandParams(subcommands=[action_subcommand])
self._SetIam()
elif action_subcommand == 'ch':
LogCommandParams(subcommands=[action_subcommand])
self._PatchIam()
else:
raise CommandException(
'Invalid subcommand "%s" for the %s command.\n'
'See "gsutil help iam".' %
(action_subcommand, self.command_name))
return 0
class LabelCommand(Command):
"""Implementation of gsutil label command."""
# Command specification. See base class for documentation.
command_spec = Command.CreateCommandSpec(
'label',
usage_synopsis=_SYNOPSIS,
min_args=2,
max_args=NO_MAX,
supported_sub_args='l:d:',
file_url_ok=False,
provider_url_ok=False,
urls_start_arg=1,
gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments={
'set': [
CommandArgument.MakeNFileURLsArgument(1),
CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(),
],
'get': [
CommandArgument.MakeNCloudURLsArgument(1),
],
'ch': [
CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(),
],
},
)
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='label',
help_name_aliases=[],
help_type='command_help',
help_one_line_summary=(
'Get, set, or change the label configuration of a bucket.'),
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={
'get': _get_help_text,
'set': _set_help_text,
'ch': _ch_help_text,
},
)
def _CalculateUrlsStartArg(self):
if not self.args:
self.RaiseWrongNumberOfArgumentsException()
if self.args[0].lower() == 'set':
return 2 # Filename comes before bucket arg(s).
return 1
def _SetLabel(self):
"""Parses options and sets labels on the specified buckets."""
# At this point, "set" has been popped off the front of self.args.
if len(self.args) < 2:
self.RaiseWrongNumberOfArgumentsException()
label_filename = self.args[0]
if not os.path.isfile(label_filename):
raise CommandException('Could not find the file "%s".' %
label_filename)
with codecs.open(label_filename, 'r', UTF8) as label_file:
label_text = label_file.read()
@Retry(PreconditionException, tries=3, timeout_secs=1)
def _SetLabelForBucket(blr):
url = blr.storage_url
self.logger.info('Setting label configuration on %s...', blr)
if url.scheme == 's3': # Uses only XML.
self.gsutil_api.XmlPassThroughSetTagging(label_text,
url,
provider=url.scheme)
else: # Must be a 'gs://' bucket.
labels_message = None
# When performing a read-modify-write cycle, include metageneration to
# avoid race conditions (supported for GS buckets only).
metageneration = None
new_label_json = json.loads(label_text)
if (self.gsutil_api.GetApiSelector(
url.scheme) == ApiSelector.JSON):
# Perform a read-modify-write so that we can specify which
# existing labels need to be deleted.
_, bucket_metadata = self.GetSingleBucketUrlFromArg(
url.url_string,
bucket_fields=['labels', 'metageneration'])
metageneration = bucket_metadata.metageneration
label_json = {}
if bucket_metadata.labels:
label_json = json.loads(
LabelTranslation.JsonFromMessage(
bucket_metadata.labels))
# Set all old keys' values to None; this will delete each key that
# is not included in the new set of labels.
merged_labels = dict(
(key, None) for key, _ in six.iteritems(label_json))
merged_labels.update(new_label_json)
labels_message = LabelTranslation.DictToMessage(
merged_labels)
else: # ApiSelector.XML
# No need to read-modify-write with the XML API.
labels_message = LabelTranslation.DictToMessage(
new_label_json)
preconditions = Preconditions(meta_gen_match=metageneration)
bucket_metadata = apitools_messages.Bucket(
labels=labels_message)
self.gsutil_api.PatchBucket(url.bucket_name,
bucket_metadata,
preconditions=preconditions,
provider=url.scheme,
fields=['id'])
some_matched = False
url_args = self.args[1:]
for url_str in url_args:
# Throws a CommandException if the argument is not a bucket.
bucket_iter = self.GetBucketUrlIterFromArg(url_str,
bucket_fields=['id'])
for bucket_listing_ref in bucket_iter:
some_matched = True
_SetLabelForBucket(bucket_listing_ref)
if not some_matched:
raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args))
def _ChLabel(self):
"""Parses options and changes labels on the specified buckets."""
self.label_changes = {}
self.num_deletions = 0
if self.sub_opts:
for o, a in self.sub_opts:
if o == '-l':
label_split = a.split(':')
if len(label_split) != 2:
raise CommandException(
'Found incorrectly formatted option for "gsutil label ch": '
'"%s". To add a label, please use the form <key>:<value>.'
% a)
self.label_changes[label_split[0]] = label_split[1]
elif o == '-d':
# Ensure only the key is supplied; stop if key:value was given.
val_split = a.split(':')
if len(val_split) != 1:
raise CommandException(
'Found incorrectly formatted option for "gsutil label ch": '
'"%s". To delete a label, provide only its key.' %
a)
self.label_changes[a] = None
self.num_deletions += 1
else:
self.RaiseInvalidArgumentException()
if not self.label_changes:
raise CommandException(
'Please specify at least one label change with the -l or -d flags.'
)
@Retry(PreconditionException, tries=3, timeout_secs=1)
def _ChLabelForBucket(blr):
url = blr.storage_url
self.logger.info('Setting label configuration on %s...', blr)
labels_message = None
# When performing a read-modify-write cycle, include metageneration to
# avoid race conditions (supported for GS buckets only).
metageneration = None
if (self.gsutil_api.GetApiSelector(
url.scheme) == ApiSelector.JSON):
# The JSON API's PATCH semantics allow us to skip read-modify-write,
# with the exception of one edge case - attempting to delete a
# nonexistent label returns an error iff no labels previously existed
corrected_changes = self.label_changes
if self.num_deletions:
(_, bucket_metadata) = self.GetSingleBucketUrlFromArg(
url.url_string,
bucket_fields=['labels', 'metageneration'])
if not bucket_metadata.labels:
metageneration = bucket_metadata.metageneration
# Remove each change that would try to delete a nonexistent key.
corrected_changes = dict(
(k, v)
for k, v in six.iteritems(self.label_changes) if v)
labels_message = LabelTranslation.DictToMessage(
corrected_changes)
else: # ApiSelector.XML
# Perform a read-modify-write cycle so that we can specify which
# existing labels need to be deleted.
(_, bucket_metadata) = self.GetSingleBucketUrlFromArg(
url.url_string, bucket_fields=['labels', 'metageneration'])
metageneration = bucket_metadata.metageneration
label_json = {}
if bucket_metadata.labels:
label_json = json.loads(
LabelTranslation.JsonFromMessage(
bucket_metadata.labels))
# Modify label_json such that all specified labels are added
# (overwriting old labels if necessary) and all specified deletions
# are removed from label_json if already present.
for key, value in six.iteritems(self.label_changes):
if not value and key in label_json:
del label_json[key]
else:
label_json[key] = value
labels_message = LabelTranslation.DictToMessage(label_json)
preconditions = Preconditions(meta_gen_match=metageneration)
bucket_metadata = apitools_messages.Bucket(labels=labels_message)
self.gsutil_api.PatchBucket(url.bucket_name,
bucket_metadata,
preconditions=preconditions,
provider=url.scheme,
fields=['id'])
some_matched = False
url_args = self.args
if not url_args:
self.RaiseWrongNumberOfArgumentsException()
for url_str in url_args:
# Throws a CommandException if the argument is not a bucket.
bucket_iter = self.GetBucketUrlIterFromArg(url_str)
for bucket_listing_ref in bucket_iter:
some_matched = True
_ChLabelForBucket(bucket_listing_ref)
if not some_matched:
raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args))
def _GetAndPrintLabel(self, bucket_arg):
"""Gets and prints the labels for a cloud bucket."""
bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg(
bucket_arg, bucket_fields=['labels'])
if bucket_url.scheme == 's3':
print((self.gsutil_api.XmlPassThroughGetTagging(
bucket_url, provider=bucket_url.scheme)))
else:
if bucket_metadata.labels:
print((LabelTranslation.JsonFromMessage(bucket_metadata.labels,
pretty_print=True)))
else:
print(('%s has no label configuration.' % bucket_url))
def RunCommand(self):
"""Command entry point for the label command."""
action_subcommand = self.args.pop(0)
self.ParseSubOpts(check_args=True)
# Commands with both suboptions and subcommands need to reparse for
# suboptions, so we log again.
metrics.LogCommandParams(sub_opts=self.sub_opts)
if action_subcommand == 'get':
metrics.LogCommandParams(subcommands=[action_subcommand])
self._GetAndPrintLabel(self.args[0])
elif action_subcommand == 'set':
metrics.LogCommandParams(subcommands=[action_subcommand])
self._SetLabel()
elif action_subcommand == 'ch':
metrics.LogCommandParams(subcommands=[action_subcommand])
self._ChLabel()
else:
raise CommandException(
'Invalid subcommand "%s" for the %s command.\nSee "gsutil help %s".'
% (action_subcommand, self.command_name, self.command_name))
return 0
class UrlSignCommand(Command):
"""Implementation of gsutil url_sign command."""
# Command specification. See base class for documentation.
command_spec = Command.CreateCommandSpec(
'signurl',
command_name_aliases=['signedurl', 'queryauth'],
usage_synopsis=_SYNOPSIS,
min_args=2,
max_args=NO_MAX,
supported_sub_args='m:d:c:p:',
file_url_ok=False,
provider_url_ok=False,
urls_start_arg=1,
gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments=[
CommandArgument.MakeNFileURLsArgument(1),
CommandArgument.MakeZeroOrMoreCloudURLsArgument()
])
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='signurl',
help_name_aliases=['signedurl', 'queryauth'],
help_type='command_help',
help_one_line_summary='Create a signed url',
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={},
)
def _ParseAndCheckSubOpts(self):
# Default argument values
delta = None
method = 'GET'
content_type = ''
passwd = None
for o, v in self.sub_opts:
if o == '-d':
if delta is not None:
delta += _DurationToTimeDelta(v)
else:
delta = _DurationToTimeDelta(v)
elif o == '-m':
method = v
elif o == '-c':
content_type = v
elif o == '-p':
passwd = v
else:
self.RaiseInvalidArgumentException()
if delta is None:
delta = timedelta(hours=1)
expiration = calendar.timegm(
(datetime.utcnow() + delta).utctimetuple())
if method not in ['GET', 'PUT', 'DELETE', 'HEAD', 'RESUMABLE']:
raise CommandException('HTTP method must be one of'
'[GET|HEAD|PUT|DELETE|RESUMABLE]')
return method, expiration, content_type, passwd
def _ProbeObjectAccessWithClient(self, key, client_email, gcs_path,
logger):
"""Performs a head request against a signed url to check for read access."""
# Choose a reasonable time in the future; if the user's system clock is
# 60 or more seconds behind the server's this will generate an error.
signed_url = _GenSignedUrl(key, client_email, 'HEAD', '', '',
int(time.time()) + 60, gcs_path, logger)
try:
h = GetNewHttp()
req = Request(signed_url, 'HEAD')
response = MakeRequest(h, req)
if response.status_code not in [200, 403, 404]:
raise HttpError.FromResponse(response)
return response.status_code
except HttpError:
error_string = (
'Unexpected HTTP response code %s while querying '
'object readability. Is your system clock accurate?' %
response.status_code)
if response.content:
error_string += ' Content: %s' % response.content
raise CommandException(error_string)
def _EnumerateStorageUrls(self, in_urls):
ret = []
for url_str in in_urls:
if ContainsWildcard(url_str):
ret.extend([
blr.storage_url for blr in self.WildcardIterator(url_str)
])
else:
ret.append(StorageUrlFromString(url_str))
return ret
def RunCommand(self):
"""Command entry point for signurl command."""
if not HAVE_OPENSSL:
raise CommandException(
'The signurl command requires the pyopenssl library (try pip '
'install pyopenssl or easy_install pyopenssl)')
method, expiration, content_type, passwd = self._ParseAndCheckSubOpts()
storage_urls = self._EnumerateStorageUrls(self.args[1:])
key = None
client_email = None
try:
key, client_email = _ReadJSONKeystore(
open(self.args[0], 'rb').read(), passwd)
except ValueError:
# Ignore and try parsing as a pkcs12.
if not passwd:
passwd = getpass.getpass('Keystore password:'******'rb').read(), passwd)
except ValueError:
raise CommandException(
'Unable to parse private key from {0}'.format(
self.args[0]))
print 'URL\tHTTP Method\tExpiration\tSigned URL'
for url in storage_urls:
if url.scheme != 'gs':
raise CommandException(
'Can only create signed urls from gs:// urls')
if url.IsBucket():
gcs_path = url.bucket_name
if method == 'RESUMABLE':
raise CommandException(
'Resumable signed URLs require an object '
'name.')
else:
# Need to url encode the object name as Google Cloud Storage does when
# computing the string to sign when checking the signature.
gcs_path = '{0}/{1}'.format(
url.bucket_name,
urllib.quote(url.object_name.encode(UTF8)))
final_url = _GenSignedUrl(key,
client_email,
method,
'',
content_type,
expiration,
gcs_path,
self.logger,
string_to_sign_debug=True)
expiration_dt = datetime.fromtimestamp(expiration)
print '{0}\t{1}\t{2}\t{3}'.format(
url.url_string.encode(UTF8), method,
(expiration_dt.strftime('%Y-%m-%d %H:%M:%S')),
final_url.encode(UTF8))
response_code = self._ProbeObjectAccessWithClient(
key, client_email, gcs_path, self.logger)
if response_code == 404:
if url.IsBucket() and method != 'PUT':
raise CommandException(
'Bucket {0} does not exist. Please create a bucket with '
'that name before a creating signed URL to access it.'.
format(url))
else:
if method != 'PUT' and method != 'RESUMABLE':
raise CommandException(
'Object {0} does not exist. Please create/upload an object '
'with that name before a creating signed URL to access it.'
.format(url))
elif response_code == 403:
self.logger.warn(
'%s does not have permissions on %s, using this link will likely '
'result in a 403 error until at least READ permissions are granted',
client_email, url)
return 0
class LifecycleCommand(Command):
"""Implementation of gsutil lifecycle command."""
# Command specification. See base class for documentation.
command_spec = Command.CreateCommandSpec(
'lifecycle',
command_name_aliases=['lifecycleconfig'],
usage_synopsis=_SYNOPSIS,
min_args=2,
max_args=NO_MAX,
supported_sub_args='',
file_url_ok=True,
provider_url_ok=False,
urls_start_arg=1,
gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments={
'set': [
CommandArgument.MakeNFileURLsArgument(1),
CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()
],
'get': [
CommandArgument.MakeNCloudBucketURLsArgument(1)
]
}
)
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='lifecycle',
help_name_aliases=['getlifecycle', 'setlifecycle'],
help_type='command_help',
help_one_line_summary=(
'Get or set lifecycle configuration for a bucket'),
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={'get': _get_help_text, 'set': _set_help_text},
)
def _SetLifecycleConfig(self):
"""Sets lifecycle configuration for a Google Cloud Storage bucket."""
lifecycle_arg = self.args[0]
url_args = self.args[1:]
# Disallow multi-provider 'lifecycle set' requests.
if not UrlsAreForSingleProvider(url_args):
raise CommandException('"%s" command spanning providers not allowed.' %
self.command_name)
# Open, read and parse file containing JSON document.
lifecycle_file = open(lifecycle_arg, 'r')
lifecycle_txt = lifecycle_file.read()
lifecycle_file.close()
# Iterate over URLs, expanding wildcards and setting the lifecycle on each.
some_matched = False
for url_str in url_args:
bucket_iter = self.GetBucketUrlIterFromArg(url_str,
bucket_fields=['lifecycle'])
for blr in bucket_iter:
url = blr.storage_url
some_matched = True
self.logger.info('Setting lifecycle configuration on %s...', blr)
if url.scheme == 's3':
self.gsutil_api.XmlPassThroughSetLifecycle(
lifecycle_txt, url, provider=url.scheme)
else:
lifecycle = LifecycleTranslation.JsonLifecycleToMessage(lifecycle_txt)
bucket_metadata = apitools_messages.Bucket(lifecycle=lifecycle)
self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata,
provider=url.scheme, fields=['id'])
if not some_matched:
raise CommandException('No URLs matched')
return 0
def _GetLifecycleConfig(self):
"""Gets lifecycle configuration for a Google Cloud Storage bucket."""
bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg(
self.args[0], bucket_fields=['lifecycle'])
if bucket_url.scheme == 's3':
sys.stdout.write(self.gsutil_api.XmlPassThroughGetLifecycle(
bucket_url, provider=bucket_url.scheme))
else:
if bucket_metadata.lifecycle and bucket_metadata.lifecycle.rule:
sys.stdout.write(LifecycleTranslation.JsonLifecycleFromMessage(
bucket_metadata.lifecycle))
else:
sys.stdout.write('%s has no lifecycle configuration.\n' % bucket_url)
return 0
def RunCommand(self):
"""Command entry point for the lifecycle command."""
subcommand = self.args.pop(0)
if subcommand == 'get':
return self._GetLifecycleConfig()
elif subcommand == 'set':
return self._SetLifecycleConfig()
else:
raise CommandException('Invalid subcommand "%s" for the %s command.' %
(subcommand, self.command_name))
class UrlSignCommand(Command):
"""Implementation of gsutil url_sign command."""
# Command specification. See base class for documentation.
command_spec = Command.CreateCommandSpec(
'signurl',
command_name_aliases=['signedurl', 'queryauth'],
usage_synopsis=_SYNOPSIS,
min_args=2,
max_args=NO_MAX,
supported_sub_args='m:d:c:p:r:',
file_url_ok=False,
provider_url_ok=False,
urls_start_arg=1,
gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments=[
CommandArgument.MakeNFileURLsArgument(1),
CommandArgument.MakeZeroOrMoreCloudURLsArgument(),
],
)
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='signurl',
help_name_aliases=[
'signedurl',
'queryauth',
],
help_type='command_help',
help_one_line_summary='Create a signed url',
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={},
)
def _ParseAndCheckSubOpts(self):
# Default argument values
delta = None
method = 'GET'
content_type = ''
passwd = None
region = _AUTO_DETECT_REGION
for o, v in self.sub_opts:
# TODO(PY3-ONLY): Delete this if block.
if six.PY2:
v = v.decode(sys.stdin.encoding or UTF8)
if o == '-d':
if delta is not None:
delta += _DurationToTimeDelta(v)
else:
delta = _DurationToTimeDelta(v)
elif o == '-m':
method = v
elif o == '-c':
content_type = v
elif o == '-p':
passwd = v
elif o == '-r':
region = v
else:
self.RaiseInvalidArgumentException()
if delta is None:
delta = timedelta(hours=1)
else:
if delta > _MAX_EXPIRATION_TIME:
raise CommandException('Max valid duration allowed is '
'%s' % _MAX_EXPIRATION_TIME)
if method not in ['GET', 'PUT', 'DELETE', 'HEAD', 'RESUMABLE']:
raise CommandException('HTTP method must be one of'
'[GET|HEAD|PUT|DELETE|RESUMABLE]')
return method, delta, content_type, passwd, region
def _ProbeObjectAccessWithClient(self, key, client_email, gcs_path, logger,
region):
"""Performs a head request against a signed url to check for read access."""
# Choose a reasonable time in the future; if the user's system clock is
# 60 or more seconds behind the server's this will generate an error.
signed_url = _GenSignedUrl(key, client_email, 'HEAD',
timedelta(seconds=60), gcs_path, logger,
region)
try:
h = GetNewHttp()
req = Request(signed_url, 'HEAD')
response = MakeRequest(h, req)
if response.status_code not in [200, 403, 404]:
raise HttpError.FromResponse(response)
return response.status_code
except HttpError as http_error:
if http_error.has_attr('response'):
error_response = http_error.response
error_string = (
'Unexpected HTTP response code %s while querying '
'object readability. Is your system clock accurate?' %
error_response.status_code)
if error_response.content:
error_string += ' Content: %s' % error_response.content
else:
error_string = (
'Expected an HTTP response code of '
'200 while querying object readability, but received '
'an error: %s' % http_error)
raise CommandException(error_string)
def _EnumerateStorageUrls(self, in_urls):
ret = []
for url_str in in_urls:
if ContainsWildcard(url_str):
ret.extend([
blr.storage_url for blr in self.WildcardIterator(url_str)
])
else:
ret.append(StorageUrlFromString(url_str))
return ret
def RunCommand(self):
"""Command entry point for signurl command."""
if not HAVE_OPENSSL:
raise CommandException(
'The signurl command requires the pyopenssl library (try pip '
'install pyopenssl or easy_install pyopenssl)')
method, delta, content_type, passwd, region = self._ParseAndCheckSubOpts(
)
storage_urls = self._EnumerateStorageUrls(self.args[1:])
region_cache = {}
key = None
client_email = None
try:
key, client_email = _ReadJSONKeystore(
open(self.args[0], 'rb').read(), passwd)
except ValueError:
# Ignore and try parsing as a pkcs12.
if not passwd:
passwd = getpass.getpass('Keystore password:'******'rb').read(), passwd)
except ValueError:
raise CommandException(
'Unable to parse private key from {0}'.format(
self.args[0]))
print('URL\tHTTP Method\tExpiration\tSigned URL')
for url in storage_urls:
if url.scheme != 'gs':
raise CommandException(
'Can only create signed urls from gs:// urls')
if url.IsBucket():
if region == _AUTO_DETECT_REGION:
raise CommandException(
'Generating signed URLs for creating buckets'
' requires a region be specified via the -r '
'option. Run `gsutil help signurl` for more '
'information about the \'-r\' option.')
gcs_path = url.bucket_name
if method == 'RESUMABLE':
raise CommandException(
'Resumable signed URLs require an object '
'name.')
else:
# Need to url encode the object name as Google Cloud Storage does when
# computing the string to sign when checking the signature.
gcs_path = '{0}/{1}'.format(
url.bucket_name,
urllib.parse.quote(url.object_name.encode(UTF8),
safe='/~'))
if region == _AUTO_DETECT_REGION:
if url.bucket_name in region_cache:
bucket_region = region_cache[url.bucket_name]
else:
try:
_, bucket = self.GetSingleBucketUrlFromArg(
'gs://{}'.format(url.bucket_name),
bucket_fields=['location'])
except Exception as e:
raise CommandException(
'{}: Failed to auto-detect location for bucket \'{}\'. Please '
'ensure you have storage.buckets.get permission on the bucket '
'or specify the bucket\'s location using the \'-r\' option.'
.format(e.__class__.__name__, url.bucket_name))
bucket_region = bucket.location.lower()
region_cache[url.bucket_name] = bucket_region
else:
bucket_region = region
final_url = _GenSignedUrl(key,
client_email,
method,
delta,
gcs_path,
self.logger,
bucket_region,
content_type,
string_to_sign_debug=True)
expiration = calendar.timegm(
(datetime.utcnow() + delta).utctimetuple())
expiration_dt = datetime.fromtimestamp(expiration)
time_str = expiration_dt.strftime('%Y-%m-%d %H:%M:%S')
# TODO(PY3-ONLY): Delete this if block.
if six.PY2:
time_str = time_str.decode(UTF8)
url_info_str = '{0}\t{1}\t{2}\t{3}'.format(url.url_string, method,
time_str, final_url)
# TODO(PY3-ONLY): Delete this if block.
if six.PY2:
url_info_str = url_info_str.encode(UTF8)
print(url_info_str)
response_code = self._ProbeObjectAccessWithClient(
key, client_email, gcs_path, self.logger, bucket_region)
if response_code == 404:
if url.IsBucket() and method != 'PUT':
raise CommandException(
'Bucket {0} does not exist. Please create a bucket with '
'that name before a creating signed URL to access it.'.
format(url))
else:
if method != 'PUT' and method != 'RESUMABLE':
raise CommandException(
'Object {0} does not exist. Please create/upload an object '
'with that name before a creating signed URL to access it.'
.format(url))
elif response_code == 403:
self.logger.warn(
'%s does not have permissions on %s, using this link will likely '
'result in a 403 error until at least READ permissions are granted',
client_email, url)
return 0
