class FakeCommandWithCompleters(Command):
"""Command with various completer types."""
command_spec = Command.CreateCommandSpec(
'fake2',
argparse_arguments=[
CommandArgument.MakeZeroOrMoreCloudURLsArgument(),
CommandArgument.MakeZeroOrMoreFileURLsArgument(),
CommandArgument.MakeZeroOrMoreCloudOrFileURLsArgument(),
CommandArgument.MakeFreeTextArgument(),
CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(),
CommandArgument.MakeFileURLOrCannedACLArgument(),
]
)
help_spec = Command.HelpSpec(
help_name='fake2',
help_name_aliases=[],
help_type='command_help',
help_one_line_summary='fake command for tests',
help_text='fake command for tests',
subcommand_help_text={}
)
def __init__(self):
pass
class NotificationCommand(Command):
"""Implementation of gsutil notification command."""
# Notification names might look like one of these:
# canonical form: projects/_/buckets/bucket/notificationConfigs/3
# JSON API form: b/bucket/notificationConfigs/5
# Either of the above might start with a / if a user is copying & pasting.
def _GetNotificationPathRegex(self):
if not NotificationCommand._notification_path_regex:
NotificationCommand._notification_path_regex = re.compile(
('/?(projects/[^/]+/)?b(uckets)?/(?P<bucket>[^/]+)/'
'notificationConfigs/(?P<notification>[0-9]+)'))
return NotificationCommand._notification_path_regex
_notification_path_regex = None
# Command specification. See base class for documentation.
command_spec = Command.CreateCommandSpec(
'notification',
command_name_aliases=[
'notify',
'notifyconfig',
'notifications',
'notif',
],
usage_synopsis=_SYNOPSIS,
min_args=2,
max_args=NO_MAX,
supported_sub_args='i:t:m:t:of:e:p:s',
file_url_ok=False,
provider_url_ok=False,
urls_start_arg=1,
gs_api_support=[ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments={
'watchbucket': [
CommandArgument.MakeFreeTextArgument(),
CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(),
],
'stopchannel': [],
'list': [
CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(),
],
'delete': [
# Takes a list of one of the following:
# notification: projects/_/buckets/bla/notificationConfigs/5,
# bucket: gs://foobar
CommandArgument.MakeZeroOrMoreCloudURLsArgument(),
],
'create': [
CommandArgument.MakeFreeTextArgument(), # Cloud Pub/Sub topic
CommandArgument.MakeNCloudBucketURLsArgument(1),
]
},
)
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='notification',
help_name_aliases=[
'watchbucket',
'stopchannel',
'notifyconfig',
],
help_type='command_help',
help_one_line_summary='Configure object change notification',
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={
'create': _create_help_text,
'list': _list_help_text,
'delete': _delete_help_text,
'watchbucket': _watchbucket_help_text,
'stopchannel': _stopchannel_help_text,
},
)
def _WatchBucket(self):
"""Creates a watch on a bucket given in self.args."""
self.CheckArguments()
identifier = None
client_token = None
if self.sub_opts:
for o, a in self.sub_opts:
if o == '-i':
identifier = a
if o == '-t':
client_token = a
identifier = identifier or str(uuid.uuid4())
watch_url = self.args[0]
bucket_arg = self.args[-1]
if not watch_url.lower().startswith('https://'):
raise CommandException(
'The application URL must be an https:// URL.')
bucket_url = StorageUrlFromString(bucket_arg)
if not (bucket_url.IsBucket() and bucket_url.scheme == 'gs'):
raise CommandException(
'The %s command can only be used with gs:// bucket URLs.' %
self.command_name)
if not bucket_url.IsBucket():
raise CommandException(
'URL must name a bucket for the %s command.' %
self.command_name)
self.logger.info('Watching bucket %s with application URL %s ...',
bucket_url, watch_url)
try:
channel = self.gsutil_api.WatchBucket(bucket_url.bucket_name,
watch_url,
identifier,
token=client_token,
provider=bucket_url.scheme)
except AccessDeniedException as e:
self.logger.warn(
NOTIFICATION_AUTHORIZATION_FAILED_MESSAGE.format(
watch_error=str(e), watch_url=watch_url))
raise
channel_id = channel.id
resource_id = channel.resourceId
client_token = channel.token
self.logger.info('Successfully created watch notification channel.')
self.logger.info('Watch channel identifier: %s', channel_id)
self.logger.info('Canonicalized resource identifier: %s', resource_id)
self.logger.info('Client state token: %s', client_token)
return 0
def _StopChannel(self):
channel_id = self.args[0]
resource_id = self.args[1]
self.logger.info('Removing channel %s with resource identifier %s ...',
channel_id, resource_id)
self.gsutil_api.StopChannel(channel_id, resource_id, provider='gs')
self.logger.info('Succesfully removed channel.')
return 0
def _ListChannels(self, bucket_arg):
"""Lists active channel watches on a bucket given in self.args."""
bucket_url = StorageUrlFromString(bucket_arg)
if not (bucket_url.IsBucket() and bucket_url.scheme == 'gs'):
raise CommandException(
'The %s command can only be used with gs:// bucket URLs.' %
self.command_name)
if not bucket_url.IsBucket():
raise CommandException(
'URL must name a bucket for the %s command.' %
self.command_name)
channels = self.gsutil_api.ListChannels(bucket_url.bucket_name,
provider='gs').items
self.logger.info(
'Bucket %s has the following active Object Change Notifications:',
bucket_url.bucket_name)
for idx, channel in enumerate(channels):
self.logger.info('\tNotification channel %d:', idx + 1)
self.logger.info('\t\tChannel identifier: %s', channel.channel_id)
self.logger.info('\t\tResource identifier: %s',
channel.resource_id)
self.logger.info('\t\tApplication URL: %s', channel.push_url)
self.logger.info('\t\tCreated by: %s', channel.subscriber_email)
self.logger.info(
'\t\tCreation time: %s',
str(datetime.fromtimestamp(channel.creation_time_ms / 1000)))
return 0
def _Create(self):
self.CheckArguments()
# User-specified options
pubsub_topic = None
payload_format = None
custom_attributes = {}
event_types = []
object_name_prefix = None
should_setup_topic = True
if self.sub_opts:
for o, a in self.sub_opts:
if o == '-e':
event_types.append(a)
elif o == '-f':
payload_format = a
elif o == '-m':
if ':' not in a:
raise CommandException(
'Custom attributes specified with -m should be of the form '
'key:value')
key, value = a.split(':')
custom_attributes[key] = value
elif o == '-p':
object_name_prefix = a
elif o == '-s':
should_setup_topic = False
elif o == '-t':
pubsub_topic = a
if payload_format not in PAYLOAD_FORMAT_MAP:
raise CommandException(
"Must provide a payload format with -f of either 'json' or 'none'"
)
payload_format = PAYLOAD_FORMAT_MAP[payload_format]
bucket_arg = self.args[-1]
bucket_url = StorageUrlFromString(bucket_arg)
if not bucket_url.IsCloudUrl() or not bucket_url.IsBucket():
raise CommandException(
"%s %s requires a GCS bucket name, but got '%s'" %
(self.command_name, self.subcommand_name, bucket_arg))
if bucket_url.scheme != 'gs':
raise CommandException(
'The %s command can only be used with gs:// bucket URLs.' %
self.command_name)
bucket_name = bucket_url.bucket_name
self.logger.debug('Creating notification for bucket %s', bucket_url)
# Find the project this bucket belongs to
bucket_metadata = self.gsutil_api.GetBucket(bucket_name,
fields=['projectNumber'],
provider=bucket_url.scheme)
bucket_project_number = bucket_metadata.projectNumber
# If not specified, choose a sensible default for the Cloud Pub/Sub topic
# name.
if not pubsub_topic:
pubsub_topic = 'projects/%s/topics/%s' % (PopulateProjectId(None),
bucket_name)
if not pubsub_topic.startswith('projects/'):
# If a user picks a topic ID (mytopic) but doesn't pass the whole name (
# projects/my-project/topics/mytopic ), pick a default project.
pubsub_topic = 'projects/%s/topics/%s' % (PopulateProjectId(None),
pubsub_topic)
self.logger.debug('Using Cloud Pub/Sub topic %s', pubsub_topic)
just_modified_topic_permissions = False
if should_setup_topic:
# Ask GCS for the email address that represents GCS's permission to
# publish to a Cloud Pub/Sub topic from this project.
service_account = self.gsutil_api.GetProjectServiceAccount(
bucket_project_number,
provider=bucket_url.scheme).email_address
self.logger.debug('Service account for project %d: %s',
bucket_project_number, service_account)
just_modified_topic_permissions = self._CreateTopic(
pubsub_topic, service_account)
for attempt_number in range(0, 2):
try:
create_response = self.gsutil_api.CreateNotificationConfig(
bucket_name,
pubsub_topic=pubsub_topic,
payload_format=payload_format,
custom_attributes=custom_attributes,
event_types=event_types if event_types else None,
object_name_prefix=object_name_prefix,
provider=bucket_url.scheme)
break
except PublishPermissionDeniedException:
if attempt_number == 0 and just_modified_topic_permissions:
# If we have just set the IAM policy, it may take up to 10 seconds to
# take effect.
self.logger.info(
'Retrying create notification in 10 seconds '
'(new permissions may take up to 10 seconds to take effect.)'
)
time.sleep(10)
else:
raise
notification_name = 'projects/_/buckets/%s/notificationConfigs/%s' % (
bucket_name, create_response.id)
self.logger.info('Created notification config %s', notification_name)
return 0
def _CreateTopic(self, pubsub_topic, service_account):
"""Assures that a topic exists, creating it if necessary.
Also adds GCS as a publisher on that bucket, if necessary.
Args:
pubsub_topic: name of the Cloud Pub/Sub topic to use/create.
service_account: the GCS service account that needs publish permission.
Returns:
true if we modified IAM permissions, otherwise false.
"""
pubsub_api = PubsubApi(logger=self.logger)
# Verify that the Pub/Sub topic exists. If it does not, create it.
try:
pubsub_api.GetTopic(topic_name=pubsub_topic)
self.logger.debug('Topic %s already exists', pubsub_topic)
except NotFoundException:
self.logger.debug('Creating topic %s', pubsub_topic)
pubsub_api.CreateTopic(topic_name=pubsub_topic)
self.logger.info('Created Cloud Pub/Sub topic %s', pubsub_topic)
# Verify that the service account is in the IAM policy.
policy = pubsub_api.GetTopicIamPolicy(topic_name=pubsub_topic)
binding = Binding(role='roles/pubsub.publisher',
members=['serviceAccount:%s' % service_account])
# This could be more extensive. We could, for instance, check for roles
# that are stronger that pubsub.publisher, like owner. We could also
# recurse up the hierarchy looking to see if there are project-level
# permissions. This can get very complex very quickly, as the caller
# may not necessarily have access to the project-level IAM policy.
# There's no danger in double-granting permission just to make sure it's
# there, though.
if binding not in policy.bindings:
policy.bindings.append(binding)
# transactional safety via etag field.
pubsub_api.SetTopicIamPolicy(topic_name=pubsub_topic,
policy=policy)
return True
else:
self.logger.debug(
'GCS already has publish permission to topic %s.',
pubsub_topic)
return False
def _EnumerateNotificationsFromArgs(self,
accept_notification_configs=True):
"""Yields bucket/notification tuples from command-line args.
Given a list of strings that are bucket names (gs://foo) or notification
config IDs, yield tuples of bucket names and their associated notifications.
Args:
accept_notification_configs: whether notification configs are valid args.
Yields:
Tuples of the form (bucket_name, Notification)
"""
path_regex = self._GetNotificationPathRegex()
for list_entry in self.args:
match = path_regex.match(list_entry)
if match:
if not accept_notification_configs:
raise CommandException(
'%s %s accepts only bucket names, but you provided %s'
%
(self.command_name, self.subcommand_name, list_entry))
bucket_name = match.group('bucket')
notification_id = match.group('notification')
found = False
for notification in self.gsutil_api.ListNotificationConfigs(
bucket_name, provider='gs'):
if notification.id == notification_id:
yield (bucket_name, notification)
found = True
break
if not found:
raise NotFoundException('Could not find notification %s' %
list_entry)
else:
storage_url = StorageUrlFromString(list_entry)
if not storage_url.IsCloudUrl():
raise CommandException(
'The %s command must be used on cloud buckets or notification '
'config names.' % self.command_name)
if storage_url.scheme != 'gs':
raise CommandException(
'The %s command only works on gs:// buckets.')
path = None
if storage_url.IsProvider():
path = 'gs://*'
elif storage_url.IsBucket():
path = list_entry
if not path:
raise CommandException(
'The %s command cannot be used on cloud objects, only buckets'
% self.command_name)
for blr in self.WildcardIterator(path).IterBuckets(
bucket_fields=['id']):
for notification in self.gsutil_api.ListNotificationConfigs(
blr.storage_url.bucket_name, provider='gs'):
yield (blr.storage_url.bucket_name, notification)
def _List(self):
self.CheckArguments()
if self.sub_opts:
if '-o' in dict(self.sub_opts):
for bucket_name in self.args:
self._ListChannels(bucket_name)
else:
for bucket_name, notification in self._EnumerateNotificationsFromArgs(
accept_notification_configs=False):
self._PrintNotificationDetails(bucket_name, notification)
return 0
def _PrintNotificationDetails(self, bucket, notification):
print(
'projects/_/buckets/{bucket}/notificationConfigs/{notification}\n'
'\tCloud Pub/Sub topic: {topic}'.format(
bucket=bucket,
notification=notification.id,
topic=notification.topic[len('//pubsub.googleapis.com/'):]))
if notification.custom_attributes:
print('\tCustom attributes:')
for attr in notification.custom_attributes.additionalProperties:
print('\t\t%s: %s' % (attr.key, attr.value))
filters = []
if notification.event_types:
filters.append('\t\tEvent Types: %s' %
', '.join(notification.event_types))
if notification.object_name_prefix:
filters.append("\t\tObject name prefix: '%s'" %
notification.object_name_prefix)
if filters:
print('\tFilters:')
for line in filters:
print(line)
self.logger.info('')
def _Delete(self):
for bucket_name, notification in self._EnumerateNotificationsFromArgs(
):
self._DeleteNotification(bucket_name, notification.id)
return 0
def _DeleteNotification(self, bucket_name, notification_id):
self.gsutil_api.DeleteNotificationConfig(bucket_name,
notification=notification_id,
provider='gs')
return 0
def _RunSubCommand(self, func):
try:
(self.sub_opts,
self.args) = getopt.getopt(self.args,
self.command_spec.supported_sub_args)
# Commands with both suboptions and subcommands need to reparse for
# suboptions, so we log again.
metrics.LogCommandParams(sub_opts=self.sub_opts)
return func(self)
except getopt.GetoptError:
self.RaiseInvalidArgumentException()
SUBCOMMANDS = {
'create': _Create,
'list': _List,
'delete': _Delete,
'watchbucket': _WatchBucket,
'stopchannel': _StopChannel
}
def RunCommand(self):
"""Command entry point for the notification command."""
self.subcommand_name = self.args.pop(0)
if self.subcommand_name in NotificationCommand.SUBCOMMANDS:
metrics.LogCommandParams(subcommands=[self.subcommand_name])
return self._RunSubCommand(
NotificationCommand.SUBCOMMANDS[self.subcommand_name])
else:
raise CommandException(
'Invalid subcommand "%s" for the %s command.' %
(self.subcommand_name, self.command_name))
class CorsCommand(Command):
"""Implementation of gsutil cors command."""
# Command specification. See base class for documentation.
command_spec = Command.CreateCommandSpec(
'cors',
command_name_aliases=['getcors', 'setcors'],
usage_synopsis=_SYNOPSIS,
min_args=2,
max_args=NO_MAX,
supported_sub_args='',
file_url_ok=False,
provider_url_ok=False,
urls_start_arg=1,
gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments={
'set': [
CommandArgument.MakeNFileURLsArgument(1),
CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()
],
'get': [CommandArgument.MakeNCloudBucketURLsArgument(1)]
})
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='cors',
help_name_aliases=['getcors', 'setcors', 'cross-origin'],
help_type='command_help',
help_one_line_summary=(
'Set a CORS JSON document for one or more buckets'),
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={
'get': _get_help_text,
'set': _set_help_text
},
)
def _CalculateUrlsStartArg(self):
if not self.args:
self.RaiseWrongNumberOfArgumentsException()
if self.args[0].lower() == 'set':
return 2
else:
return 1
def _SetCors(self):
"""Sets CORS configuration on a Google Cloud Storage bucket."""
cors_arg = self.args[0]
url_args = self.args[1:]
# Disallow multi-provider 'cors set' requests.
if not UrlsAreForSingleProvider(url_args):
raise CommandException(
'"%s" command spanning providers not allowed.' %
self.command_name)
# Open, read and parse file containing JSON document.
cors_file = open(cors_arg, 'r')
cors_txt = cors_file.read()
cors_file.close()
self.api = self.gsutil_api.GetApiSelector(
StorageUrlFromString(url_args[0]).scheme)
# Iterate over URLs, expanding wildcards and setting the CORS on each.
some_matched = False
for url_str in url_args:
bucket_iter = self.GetBucketUrlIterFromArg(url_str,
bucket_fields=['id'])
for blr in bucket_iter:
url = blr.storage_url
some_matched = True
self.logger.info('Setting CORS on %s...', blr)
if url.scheme == 's3':
self.gsutil_api.XmlPassThroughSetCors(cors_txt,
url,
provider=url.scheme)
else:
cors = CorsTranslation.JsonCorsToMessageEntries(cors_txt)
if not cors:
cors = REMOVE_CORS_CONFIG
bucket_metadata = apitools_messages.Bucket(cors=cors)
self.gsutil_api.PatchBucket(url.bucket_name,
bucket_metadata,
provider=url.scheme,
fields=['id'])
if not some_matched:
raise CommandException('No URLs matched')
return 0
def _GetCors(self):
"""Gets CORS configuration for a Google Cloud Storage bucket."""
bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg(
self.args[0], bucket_fields=['cors'])
if bucket_url.scheme == 's3':
sys.stdout.write(
self.gsutil_api.XmlPassThroughGetCors(
bucket_url, provider=bucket_url.scheme))
else:
if bucket_metadata.cors:
sys.stdout.write(
CorsTranslation.MessageEntriesToJson(bucket_metadata.cors))
else:
sys.stdout.write('%s has no CORS configuration.\n' %
bucket_url)
return 0
def RunCommand(self):
"""Command entry point for the cors command."""
action_subcommand = self.args.pop(0)
if action_subcommand == 'get':
func = self._GetCors
elif action_subcommand == 'set':
func = self._SetCors
else:
raise CommandException(
('Invalid subcommand "%s" for the %s command.\n'
'See "gsutil help cors".') %
(action_subcommand, self.command_name))
return func()
class DefStorageClassCommand(Command):
"""Implementation of gsutil defstorageclass command."""
# Command specification. See base class for documentation.
command_spec = Command.CreateCommandSpec(
'defstorageclass',
usage_synopsis=_SYNOPSIS,
min_args=2,
max_args=NO_MAX,
supported_sub_args='',
file_url_ok=False,
provider_url_ok=False,
urls_start_arg=2,
gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments={
'set': [
# FreeTextArgument allows for using storage class abbreviations.
CommandArgument.MakeFreeTextArgument(),
CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(),
],
'get': [
CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(),
],
},
)
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='defstorageclass',
help_name_aliases=['defaultstorageclass'],
help_type='command_help',
help_one_line_summary='Get or set the default storage class on buckets',
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={
'get': _get_help_text,
'set': _set_help_text,
},
)
def _CheckIsGsUrl(self, url_str):
if not url_str.startswith('gs://'):
raise CommandException(
'"%s" does not support the URL "%s". Did you mean to use a gs:// '
'URL?' % (self.command_name, url_str))
def _CalculateUrlsStartArg(self):
if not self.args:
self.RaiseWrongNumberOfArgumentsException()
if self.args[0].lower() == 'set':
return 2
else:
return 1
def _SetDefStorageClass(self):
"""Sets the default storage class for a bucket."""
# At this point, "set" has been popped off the front of self.args.
normalized_storage_class = NormalizeStorageClass(self.args[0])
url_args = self.args[1:]
if not url_args:
self.RaiseWrongNumberOfArgumentsException()
some_matched = False
for url_str in url_args:
self._CheckIsGsUrl(url_str)
# Throws a CommandException if the argument is not a bucket.
bucket_iter = self.GetBucketUrlIterFromArg(url_str,
bucket_fields=['id'])
for blr in bucket_iter:
some_matched = True
bucket_metadata = apitools_messages.Bucket()
self.logger.info(
'Setting default storage class to "%s" for bucket %s' %
(normalized_storage_class, blr.url_string.rstrip('/')))
bucket_metadata.storageClass = normalized_storage_class
self.gsutil_api.PatchBucket(blr.storage_url.bucket_name,
bucket_metadata,
provider=blr.storage_url.scheme,
fields=['id'])
if not some_matched:
raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args))
def _GetDefStorageClass(self):
"""Gets the default storage class for a bucket."""
# At this point, "get" has been popped off the front of self.args.
url_args = self.args
some_matched = False
for url_str in url_args:
self._CheckIsGsUrl(url_str)
bucket_iter = self.GetBucketUrlIterFromArg(
url_str, bucket_fields=['storageClass'])
for blr in bucket_iter:
some_matched = True
print(
'%s: %s' %
(blr.url_string.rstrip('/'), blr.root_object.storageClass))
if not some_matched:
raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args))
def RunCommand(self):
"""Command entry point for the defstorageclass command."""
action_subcommand = self.args.pop(0)
subcommand_args = [action_subcommand]
if action_subcommand == 'get':
func = self._GetDefStorageClass
elif action_subcommand == 'set':
func = self._SetDefStorageClass
normalized_storage_class = NormalizeStorageClass(self.args[0])
subcommand_args.append(normalized_storage_class)
else:
raise CommandException(
('Invalid subcommand "%s" for the %s command.\n'
'See "gsutil help %s".') %
(action_subcommand, self.command_name, self.command_name))
metrics.LogCommandParams(subcommands=subcommand_args)
func()
return 0
class VersioningCommand(Command):
"""Implementation of gsutil versioning command."""
# Command specification. See base class for documentation.
command_spec = Command.CreateCommandSpec(
'versioning',
command_name_aliases=['setversioning', 'getversioning'],
usage_synopsis=_SYNOPSIS,
min_args=2,
max_args=NO_MAX,
supported_sub_args='',
file_url_ok=False,
provider_url_ok=False,
urls_start_arg=2,
gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments={
'set': [
CommandArgument('mode', choices=['on', 'off']),
CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()
],
'get': [
CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()
]
}
)
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='versioning',
help_name_aliases=['getversioning', 'setversioning'],
help_type='command_help',
help_one_line_summary=(
'Enable or suspend versioning for one or more buckets'),
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={'get': _get_help_text, 'set': _set_help_text},
)
def _CalculateUrlsStartArg(self):
if not self.args:
self.RaiseWrongNumberOfArgumentsException()
if self.args[0].lower() == 'set':
return 2
else:
return 1
def _SetVersioning(self):
"""Gets versioning configuration for a bucket."""
versioning_arg = self.args[0].lower()
if versioning_arg not in ('on', 'off'):
raise CommandException('Argument to "%s set" must be either <on|off>'
% (self.command_name))
url_args = self.args[1:]
if not url_args:
self.RaiseWrongNumberOfArgumentsException()
# Iterate over URLs, expanding wildcards and set the versioning
# configuration on each.
some_matched = False
for url_str in url_args:
bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id'])
for blr in bucket_iter:
url = blr.storage_url
some_matched = True
bucket_metadata = apitools_messages.Bucket(
versioning=apitools_messages.Bucket.VersioningValue())
if versioning_arg == 'on':
self.logger.info('Enabling versioning for %s...', url)
bucket_metadata.versioning.enabled = True
else:
self.logger.info('Suspending versioning for %s...', url)
bucket_metadata.versioning.enabled = False
self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata,
provider=url.scheme, fields=['id'])
if not some_matched:
raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args))
def _GetVersioning(self):
"""Gets versioning configuration for one or more buckets."""
url_args = self.args
# Iterate over URLs, expanding wildcards and getting the versioning
# configuration on each.
some_matched = False
for url_str in url_args:
bucket_iter = self.GetBucketUrlIterFromArg(url_str,
bucket_fields=['versioning'])
for blr in bucket_iter:
some_matched = True
if blr.root_object.versioning and blr.root_object.versioning.enabled:
print('%s: Enabled' % blr.url_string.rstrip('/'))
else:
print('%s: Suspended' % blr.url_string.rstrip('/'))
if not some_matched:
raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args))
def RunCommand(self):
"""Command entry point for the versioning command."""
action_subcommand = self.args.pop(0)
if action_subcommand == 'get':
func = self._GetVersioning
metrics.LogCommandParams(subcommands=[action_subcommand])
elif action_subcommand == 'set':
func = self._SetVersioning
versioning_arg = self.args[0].lower()
if versioning_arg in ('on', 'off'):
metrics.LogCommandParams(
subcommands=[action_subcommand, versioning_arg])
else:
raise CommandException((
'Invalid subcommand "%s" for the %s command.\n'
'See "gsutil help %s".') % (
action_subcommand, self.command_name, self.command_name))
func()
return 0
class NotificationCommand(Command):
"""Implementation of gsutil notification command."""
# Notification names might look like one of these:
# canonical form: projects/_/buckets/bucket/notificationConfigs/3
# JSON API form: b/bucket/notificationConfigs/5
# Either of the above might start with a / if a user is copying & pasting.
def _GetNotificationPathRegex(self):
if not NotificationCommand._notification_path_regex:
NotificationCommand._notification_path_regex = re.compile(
('/?(projects/[^/]+/)?b(uckets)?/(?P<bucket>[^/]+)/'
'notificationConfigs/(?P<notification>[0-9]+)'))
return NotificationCommand._notification_path_regex
_notification_path_regex = None
# Command specification. See base class for documentation.
command_spec = Command.CreateCommandSpec(
'notification',
command_name_aliases=[
'notify', 'notifyconfig', 'notifications', 'notif'
],
usage_synopsis=_SYNOPSIS,
min_args=2,
max_args=NO_MAX,
supported_sub_args='i:t:m:t:o:f:e:p:s',
file_url_ok=False,
provider_url_ok=False,
urls_start_arg=1,
gs_api_support=[ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments={
'watchbucket': [
CommandArgument.MakeFreeTextArgument(),
CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()
],
'stopchannel': [],
'list': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()],
'delete': [
# Takes a list of one of the following:
# notification: projects/_/buckets/bla/notificationConfigs/5,
# bucket: gs://foobar
CommandArgument.MakeZeroOrMoreCloudURLsArgument()
],
'create': [
CommandArgument.MakeFreeTextArgument(), # Cloud Pub/Sub topic
CommandArgument.MakeNCloudBucketURLsArgument(1)
]
})
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='notification',
help_name_aliases=['watchbucket', 'stopchannel', 'notifyconfig'],
help_type='command_help',
help_one_line_summary='Configure object change notification',
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={
'create': _create_help_text,
'list': _list_help_text,
'delete': _delete_help_text,
'watchbucket': _watchbucket_help_text,
'stopchannel': _stopchannel_help_text
},
)
def _WatchBucket(self):
"""Creates a watch on a bucket given in self.args."""
self.CheckArguments()
identifier = None
client_token = None
if self.sub_opts:
for o, a in self.sub_opts:
if o == '-i':
identifier = a
if o == '-t':
client_token = a
identifier = identifier or str(uuid.uuid4())
watch_url = self.args[0]
bucket_arg = self.args[-1]
if not watch_url.lower().startswith('https://'):
raise CommandException(
'The application URL must be an https:// URL.')
bucket_url = StorageUrlFromString(bucket_arg)
if not (bucket_url.IsBucket() and bucket_url.scheme == 'gs'):
raise CommandException(
'The %s command can only be used with gs:// bucket URLs.' %
self.command_name)
if not bucket_url.IsBucket():
raise CommandException(
'URL must name a bucket for the %s command.' %
self.command_name)
self.logger.info('Watching bucket %s with application URL %s ...',
bucket_url, watch_url)
try:
channel = self.gsutil_api.WatchBucket(bucket_url.bucket_name,
watch_url,
identifier,
token=client_token,
provider=bucket_url.scheme)
except AccessDeniedException, e:
self.logger.warn(
NOTIFICATION_AUTHORIZATION_FAILED_MESSAGE.format(
watch_error=str(e), watch_url=watch_url))
raise
channel_id = channel.id
resource_id = channel.resourceId
client_token = channel.token
self.logger.info('Successfully created watch notification channel.')
self.logger.info('Watch channel identifier: %s', channel_id)
self.logger.info('Canonicalized resource identifier: %s', resource_id)
self.logger.info('Client state token: %s', client_token)
return 0
class LabelCommand(Command):
"""Implementation of gsutil label command."""
# Command specification. See base class for documentation.
command_spec = Command.CreateCommandSpec(
'label',
usage_synopsis=_SYNOPSIS,
min_args=2,
max_args=NO_MAX,
supported_sub_args='l:d:',
file_url_ok=False,
provider_url_ok=False,
urls_start_arg=1,
gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments={
'set': [
CommandArgument.MakeNFileURLsArgument(1),
CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(),
],
'get': [
CommandArgument.MakeNCloudURLsArgument(1),
],
'ch': [
CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(),
],
},
)
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='label',
help_name_aliases=[],
help_type='command_help',
help_one_line_summary=(
'Get, set, or change the label configuration of a bucket.'),
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={
'get': _get_help_text,
'set': _set_help_text,
'ch': _ch_help_text,
},
)
def _CalculateUrlsStartArg(self):
if not self.args:
self.RaiseWrongNumberOfArgumentsException()
if self.args[0].lower() == 'set':
return 2 # Filename comes before bucket arg(s).
return 1
def _SetLabel(self):
"""Parses options and sets labels on the specified buckets."""
# At this point, "set" has been popped off the front of self.args.
if len(self.args) < 2:
self.RaiseWrongNumberOfArgumentsException()
label_filename = self.args[0]
if not os.path.isfile(label_filename):
raise CommandException('Could not find the file "%s".' %
label_filename)
with codecs.open(label_filename, 'r', UTF8) as label_file:
label_text = label_file.read()
@Retry(PreconditionException, tries=3, timeout_secs=1)
def _SetLabelForBucket(blr):
url = blr.storage_url
self.logger.info('Setting label configuration on %s...', blr)
if url.scheme == 's3': # Uses only XML.
self.gsutil_api.XmlPassThroughSetTagging(label_text,
url,
provider=url.scheme)
else: # Must be a 'gs://' bucket.
labels_message = None
# When performing a read-modify-write cycle, include metageneration to
# avoid race conditions (supported for GS buckets only).
metageneration = None
new_label_json = json.loads(label_text)
if (self.gsutil_api.GetApiSelector(
url.scheme) == ApiSelector.JSON):
# Perform a read-modify-write so that we can specify which
# existing labels need to be deleted.
_, bucket_metadata = self.GetSingleBucketUrlFromArg(
url.url_string,
bucket_fields=['labels', 'metageneration'])
metageneration = bucket_metadata.metageneration
label_json = {}
if bucket_metadata.labels:
label_json = json.loads(
LabelTranslation.JsonFromMessage(
bucket_metadata.labels))
# Set all old keys' values to None; this will delete each key that
# is not included in the new set of labels.
merged_labels = dict(
(key, None) for key, _ in six.iteritems(label_json))
merged_labels.update(new_label_json)
labels_message = LabelTranslation.DictToMessage(
merged_labels)
else: # ApiSelector.XML
# No need to read-modify-write with the XML API.
labels_message = LabelTranslation.DictToMessage(
new_label_json)
preconditions = Preconditions(meta_gen_match=metageneration)
bucket_metadata = apitools_messages.Bucket(
labels=labels_message)
self.gsutil_api.PatchBucket(url.bucket_name,
bucket_metadata,
preconditions=preconditions,
provider=url.scheme,
fields=['id'])
some_matched = False
url_args = self.args[1:]
for url_str in url_args:
# Throws a CommandException if the argument is not a bucket.
bucket_iter = self.GetBucketUrlIterFromArg(url_str,
bucket_fields=['id'])
for bucket_listing_ref in bucket_iter:
some_matched = True
_SetLabelForBucket(bucket_listing_ref)
if not some_matched:
raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args))
def _ChLabel(self):
"""Parses options and changes labels on the specified buckets."""
self.label_changes = {}
self.num_deletions = 0
if self.sub_opts:
for o, a in self.sub_opts:
if o == '-l':
label_split = a.split(':')
if len(label_split) != 2:
raise CommandException(
'Found incorrectly formatted option for "gsutil label ch": '
'"%s". To add a label, please use the form <key>:<value>.'
% a)
self.label_changes[label_split[0]] = label_split[1]
elif o == '-d':
# Ensure only the key is supplied; stop if key:value was given.
val_split = a.split(':')
if len(val_split) != 1:
raise CommandException(
'Found incorrectly formatted option for "gsutil label ch": '
'"%s". To delete a label, provide only its key.' %
a)
self.label_changes[a] = None
self.num_deletions += 1
else:
self.RaiseInvalidArgumentException()
if not self.label_changes:
raise CommandException(
'Please specify at least one label change with the -l or -d flags.'
)
@Retry(PreconditionException, tries=3, timeout_secs=1)
def _ChLabelForBucket(blr):
url = blr.storage_url
self.logger.info('Setting label configuration on %s...', blr)
labels_message = None
# When performing a read-modify-write cycle, include metageneration to
# avoid race conditions (supported for GS buckets only).
metageneration = None
if (self.gsutil_api.GetApiSelector(
url.scheme) == ApiSelector.JSON):
# The JSON API's PATCH semantics allow us to skip read-modify-write,
# with the exception of one edge case - attempting to delete a
# nonexistent label returns an error iff no labels previously existed
corrected_changes = self.label_changes
if self.num_deletions:
(_, bucket_metadata) = self.GetSingleBucketUrlFromArg(
url.url_string,
bucket_fields=['labels', 'metageneration'])
if not bucket_metadata.labels:
metageneration = bucket_metadata.metageneration
# Remove each change that would try to delete a nonexistent key.
corrected_changes = dict(
(k, v)
for k, v in six.iteritems(self.label_changes) if v)
labels_message = LabelTranslation.DictToMessage(
corrected_changes)
else: # ApiSelector.XML
# Perform a read-modify-write cycle so that we can specify which
# existing labels need to be deleted.
(_, bucket_metadata) = self.GetSingleBucketUrlFromArg(
url.url_string, bucket_fields=['labels', 'metageneration'])
metageneration = bucket_metadata.metageneration
label_json = {}
if bucket_metadata.labels:
label_json = json.loads(
LabelTranslation.JsonFromMessage(
bucket_metadata.labels))
# Modify label_json such that all specified labels are added
# (overwriting old labels if necessary) and all specified deletions
# are removed from label_json if already present.
for key, value in six.iteritems(self.label_changes):
if not value and key in label_json:
del label_json[key]
else:
label_json[key] = value
labels_message = LabelTranslation.DictToMessage(label_json)
preconditions = Preconditions(meta_gen_match=metageneration)
bucket_metadata = apitools_messages.Bucket(labels=labels_message)
self.gsutil_api.PatchBucket(url.bucket_name,
bucket_metadata,
preconditions=preconditions,
provider=url.scheme,
fields=['id'])
some_matched = False
url_args = self.args
if not url_args:
self.RaiseWrongNumberOfArgumentsException()
for url_str in url_args:
# Throws a CommandException if the argument is not a bucket.
bucket_iter = self.GetBucketUrlIterFromArg(url_str)
for bucket_listing_ref in bucket_iter:
some_matched = True
_ChLabelForBucket(bucket_listing_ref)
if not some_matched:
raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args))
def _GetAndPrintLabel(self, bucket_arg):
"""Gets and prints the labels for a cloud bucket."""
bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg(
bucket_arg, bucket_fields=['labels'])
if bucket_url.scheme == 's3':
print((self.gsutil_api.XmlPassThroughGetTagging(
bucket_url, provider=bucket_url.scheme)))
else:
if bucket_metadata.labels:
print((LabelTranslation.JsonFromMessage(bucket_metadata.labels,
pretty_print=True)))
else:
print(('%s has no label configuration.' % bucket_url))
def RunCommand(self):
"""Command entry point for the label command."""
action_subcommand = self.args.pop(0)
self.ParseSubOpts(check_args=True)
# Commands with both suboptions and subcommands need to reparse for
# suboptions, so we log again.
metrics.LogCommandParams(sub_opts=self.sub_opts)
if action_subcommand == 'get':
metrics.LogCommandParams(subcommands=[action_subcommand])
self._GetAndPrintLabel(self.args[0])
elif action_subcommand == 'set':
metrics.LogCommandParams(subcommands=[action_subcommand])
self._SetLabel()
elif action_subcommand == 'ch':
metrics.LogCommandParams(subcommands=[action_subcommand])
self._ChLabel()
else:
raise CommandException(
'Invalid subcommand "%s" for the %s command.\nSee "gsutil help %s".'
% (action_subcommand, self.command_name, self.command_name))
return 0
class BucketPolicyOnlyCommand(Command):
"""Implements the gsutil bucketpolicyonly command."""
command_spec = Command.CreateCommandSpec(
'bucketpolicyonly',
usage_synopsis=_SYNOPSIS,
min_args=2,
max_args=NO_MAX,
supported_sub_args='',
file_url_ok=False,
provider_url_ok=False,
urls_start_arg=2,
gs_api_support=[ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments={
'get': [CommandArgument.MakeNCloudURLsArgument(1),],
'set': [
CommandArgument('mode', choices=['on', 'off']),
CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()
],
})
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='bucketpolicyonly',
help_name_aliases=[],
help_type='command_help',
help_one_line_summary='Configure uniform bucket-level access',
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={
'get': _get_help_text,
'set': _set_help_text,
},
)
def _ValidateBucketListingRefAndReturnBucketName(self, blr):
if blr.storage_url.scheme != 'gs':
raise CommandException(
'The %s command can only be used with gs:// bucket URLs.' %
self.command_name)
def _GetBucketPolicyOnly(self, blr):
"""Gets the Bucket Policy Only setting for a bucket."""
self._ValidateBucketListingRefAndReturnBucketName(blr)
bucket_url = blr.storage_url
bucket_metadata = self.gsutil_api.GetBucket(bucket_url.bucket_name,
fields=['iamConfiguration'],
provider=bucket_url.scheme)
iam_config = bucket_metadata.iamConfiguration
bucket_policy_only = iam_config.bucketPolicyOnly
fields = {
'bucket': str(bucket_url).rstrip('/'),
'enabled': bucket_policy_only.enabled
}
locked_time_line = ''
if bucket_policy_only.lockedTime:
fields['locked_time'] = bucket_policy_only.lockedTime
locked_time_line = ' LockedTime: {locked_time}\n'
if bucket_policy_only:
print(('Bucket Policy Only setting for {bucket}:\n'
' Enabled: {enabled}\n' + locked_time_line).format(**fields))
def _SetBucketPolicyOnly(self, blr, setting_arg):
"""Sets the Bucket Policy Only setting for a bucket on or off."""
self._ValidateBucketListingRefAndReturnBucketName(blr)
bucket_url = blr.storage_url
iam_config = IamConfigurationValue()
iam_config.bucketPolicyOnly = BucketPolicyOnlyValue()
iam_config.bucketPolicyOnly.enabled = (setting_arg == 'on')
bucket_metadata = apitools_messages.Bucket(iamConfiguration=iam_config)
setting_verb = 'Enabling' if setting_arg == 'on' else 'Disabling'
print('%s Bucket Policy Only for %s...' %
(setting_verb, str(bucket_url).rstrip('/')))
self.gsutil_api.PatchBucket(bucket_url.bucket_name,
bucket_metadata,
fields=['iamConfiguration'],
provider=bucket_url.scheme)
return 0
def _BucketPolicyOnly(self):
"""Handles bucketpolicyonly command on a Cloud Storage bucket."""
subcommand = self.args.pop(0)
if subcommand not in ('get', 'set'):
raise CommandException('bucketpolicyonly only supports get|set')
subcommand_func = None
subcommand_args = []
setting_arg = None
if subcommand == 'get':
subcommand_func = self._GetBucketPolicyOnly
elif subcommand == 'set':
subcommand_func = self._SetBucketPolicyOnly
setting_arg = self.args.pop(0)
InsistOnOrOff(setting_arg,
'Only on and off values allowed for set option')
subcommand_args.append(setting_arg)
# Iterate over bucket args, performing the specified subsubcommand.
some_matched = False
url_args = self.args
if not url_args:
self.RaiseWrongNumberOfArgumentsException()
for url_str in url_args:
# Throws a CommandException if the argument is not a bucket.
bucket_iter = self.GetBucketUrlIterFromArg(url_str)
for bucket_listing_ref in bucket_iter:
some_matched = True
subcommand_func(bucket_listing_ref, *subcommand_args)
if not some_matched:
raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args))
return 0
def RunCommand(self):
"""Command entry point for the bucketpolicyonly command."""
if self.gsutil_api.GetApiSelector(provider='gs') != ApiSelector.JSON:
raise CommandException('\n'.join(
textwrap.wrap(
'The "%s" command can only be used with the Cloud Storage JSON API.'
% self.command_name)))
action_subcommand = self.args[0]
self.ParseSubOpts(check_args=True)
if action_subcommand == 'get' or action_subcommand == 'set':
metrics.LogCommandParams(sub_opts=self.sub_opts)
metrics.LogCommandParams(subcommands=[action_subcommand])
self._BucketPolicyOnly()
else:
raise CommandException('Invalid subcommand "%s", use get|set instead.' %
action_subcommand)
class LifecycleCommand(Command):
"""Implementation of gsutil lifecycle command."""
# Command specification. See base class for documentation.
command_spec = Command.CreateCommandSpec(
'lifecycle',
command_name_aliases=['lifecycleconfig'],
usage_synopsis=_SYNOPSIS,
min_args=2,
max_args=NO_MAX,
supported_sub_args='',
file_url_ok=True,
provider_url_ok=False,
urls_start_arg=1,
gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments={
'set': [
CommandArgument.MakeNFileURLsArgument(1),
CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()
],
'get': [
CommandArgument.MakeNCloudBucketURLsArgument(1)
]
}
)
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='lifecycle',
help_name_aliases=['getlifecycle', 'setlifecycle'],
help_type='command_help',
help_one_line_summary=(
'Get or set lifecycle configuration for a bucket'),
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={'get': _get_help_text, 'set': _set_help_text},
)
def _SetLifecycleConfig(self):
"""Sets lifecycle configuration for a Google Cloud Storage bucket."""
lifecycle_arg = self.args[0]
url_args = self.args[1:]
# Disallow multi-provider 'lifecycle set' requests.
if not UrlsAreForSingleProvider(url_args):
raise CommandException('"%s" command spanning providers not allowed.' %
self.command_name)
# Open, read and parse file containing JSON document.
lifecycle_file = open(lifecycle_arg, 'r')
lifecycle_txt = lifecycle_file.read()
lifecycle_file.close()
# Iterate over URLs, expanding wildcards and setting the lifecycle on each.
some_matched = False
for url_str in url_args:
bucket_iter = self.GetBucketUrlIterFromArg(url_str,
bucket_fields=['lifecycle'])
for blr in bucket_iter:
url = blr.storage_url
some_matched = True
self.logger.info('Setting lifecycle configuration on %s...', blr)
if url.scheme == 's3':
self.gsutil_api.XmlPassThroughSetLifecycle(
lifecycle_txt, url, provider=url.scheme)
else:
lifecycle = LifecycleTranslation.JsonLifecycleToMessage(lifecycle_txt)
bucket_metadata = apitools_messages.Bucket(lifecycle=lifecycle)
self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata,
provider=url.scheme, fields=['id'])
if not some_matched:
raise CommandException('No URLs matched')
return 0
def _GetLifecycleConfig(self):
"""Gets lifecycle configuration for a Google Cloud Storage bucket."""
bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg(
self.args[0], bucket_fields=['lifecycle'])
if bucket_url.scheme == 's3':
sys.stdout.write(self.gsutil_api.XmlPassThroughGetLifecycle(
bucket_url, provider=bucket_url.scheme))
else:
if bucket_metadata.lifecycle and bucket_metadata.lifecycle.rule:
sys.stdout.write(LifecycleTranslation.JsonLifecycleFromMessage(
bucket_metadata.lifecycle))
else:
sys.stdout.write('%s has no lifecycle configuration.\n' % bucket_url)
return 0
def RunCommand(self):
"""Command entry point for the lifecycle command."""
subcommand = self.args.pop(0)
if subcommand == 'get':
return self._GetLifecycleConfig()
elif subcommand == 'set':
return self._SetLifecycleConfig()
else:
raise CommandException('Invalid subcommand "%s" for the %s command.' %
(subcommand, self.command_name))
class RbCommand(Command):
"""Implementation of gsutil rb command."""
# Command specification. See base class for documentation.
command_spec = Command.CreateCommandSpec(
'rb',
command_name_aliases=[
'deletebucket',
'removebucket',
'removebuckets',
'rmdir',
],
usage_synopsis=_SYNOPSIS,
min_args=1,
max_args=NO_MAX,
supported_sub_args='f',
file_url_ok=False,
provider_url_ok=False,
urls_start_arg=0,
gs_api_support=[
ApiSelector.XML,
ApiSelector.JSON,
],
gs_default_api=ApiSelector.JSON,
argparse_arguments=[
CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(),
],
)
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='rb',
help_name_aliases=[
'deletebucket',
'removebucket',
'removebuckets',
'rmdir',
],
help_type='command_help',
help_one_line_summary='Remove buckets',
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={},
)
def RunCommand(self):
"""Command entry point for the rb command."""
self.continue_on_error = False
if self.sub_opts:
for o, unused_a in self.sub_opts:
if o == '-f':
self.continue_on_error = True
did_some_work = False
some_failed = False
for url_str in self.args:
wildcard_url = StorageUrlFromString(url_str)
if wildcard_url.IsObject():
raise CommandException('"rb" command requires a provider or '
'bucket URL')
# Wrap WildcardIterator call in try/except so we can avoid printing errors
# with -f option if a non-existent URL listed, permission denial happens
# while listing, etc.
try:
# Materialize iterator results into a list to catch exceptions.
# Since this is listing buckets this list shouldn't be too large to fit
# in memory at once.
# Also, avoid listing all fields to avoid performing unnecessary bucket
# metadata GETs. These would also be problematic when billing is
# disabled, as deletes are allowed but GetBucket is not.
blrs = list(
self.WildcardIterator(url_str).IterBuckets(bucket_fields=['id']))
except: # pylint: disable=bare-except
some_failed = True
if self.continue_on_error:
continue
else:
raise
for blr in blrs:
url = blr.storage_url
self.logger.info('Removing %s...', url)
try:
self.gsutil_api.DeleteBucket(url.bucket_name, provider=url.scheme)
except NotEmptyException as e:
some_failed = True
if self.continue_on_error:
continue
elif 'VersionedBucketNotEmpty' in e.reason:
raise CommandException('Bucket is not empty. Note: this is a '
'versioned bucket, so to delete all '
'objects\nyou need to use:'
'\n\tgsutil rm -r %s' % url)
else:
raise
except: # pylint: disable=bare-except
some_failed = True
if not self.continue_on_error:
raise
did_some_work = True
if not did_some_work:
raise CommandException(NO_URLS_MATCHED_TARGET % list(self.args))
return 1 if some_failed else 0
class DefAclCommand(Command):
"""Implementation of gsutil defacl command."""
# Command specification. See base class for documentation.
command_spec = Command.CreateCommandSpec(
'defacl',
command_name_aliases=['setdefacl', 'getdefacl', 'chdefacl'],
usage_synopsis=_SYNOPSIS,
min_args=2,
max_args=NO_MAX,
supported_sub_args='fg:u:d:p:',
file_url_ok=False,
provider_url_ok=False,
urls_start_arg=1,
gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments={
'set': [
CommandArgument.MakeFileURLOrCannedACLArgument(),
CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()
],
'get': [CommandArgument.MakeNCloudBucketURLsArgument(1)],
'ch': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()],
})
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='defacl',
help_name_aliases=[
'default acl', 'setdefacl', 'getdefacl', 'chdefacl'
],
help_type='command_help',
help_one_line_summary='Get, set, or change default ACL on buckets',
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={
'get': _get_help_text,
'set': _set_help_text,
'ch': _ch_help_text
},
)
def _CalculateUrlsStartArg(self):
if not self.args:
self.RaiseWrongNumberOfArgumentsException()
if (self.args[0].lower() == 'set'
or self.command_alias_used == 'setdefacl'):
return 1
else:
return 0
def _SetDefAcl(self):
if not StorageUrlFromString(self.args[-1]).IsBucket():
raise CommandException(
'URL must name a bucket for the %s command' %
self.command_name)
try:
self.SetAclCommandHelper(SetAclFuncWrapper, SetAclExceptionHandler)
except AccessDeniedException:
self._WarnServiceAccounts()
raise
def _GetDefAcl(self):
if not StorageUrlFromString(self.args[0]).IsBucket():
raise CommandException(
'URL must name a bucket for the %s command' %
self.command_name)
self.GetAndPrintAcl(self.args[0])
def _ChDefAcl(self):
"""Parses options and changes default object ACLs on specified buckets."""
self.parse_versions = True
self.changes = []
if self.sub_opts:
for o, a in self.sub_opts:
if o == '-g':
self.changes.append(
aclhelpers.AclChange(
a, scope_type=aclhelpers.ChangeType.GROUP))
if o == '-u':
self.changes.append(
aclhelpers.AclChange(
a, scope_type=aclhelpers.ChangeType.USER))
if o == '-p':
self.changes.append(
aclhelpers.AclChange(
a, scope_type=aclhelpers.ChangeType.PROJECT))
if o == '-d':
self.changes.append(aclhelpers.AclDel(a))
if not self.changes:
raise CommandException('Please specify at least one access change '
'with the -g, -u, or -d flags')
if (not UrlsAreForSingleProvider(self.args)
or StorageUrlFromString(self.args[0]).scheme != 'gs'):
raise CommandException(
'The "{0}" command can only be used with gs:// URLs'.format(
self.command_name))
bucket_urls = set()
for url_arg in self.args:
for result in self.WildcardIterator(url_arg):
if not result.storage_url.IsBucket():
raise CommandException(
'The defacl ch command can only be applied to buckets.'
)
bucket_urls.add(result.storage_url)
for storage_url in bucket_urls:
self.ApplyAclChanges(storage_url)
@Retry(ServiceException, tries=3, timeout_secs=1)
def ApplyAclChanges(self, url):
"""Applies the changes in self.changes to the provided URL."""
bucket = self.gsutil_api.GetBucket(
url.bucket_name,
provider=url.scheme,
fields=['defaultObjectAcl', 'metageneration'])
# Default object ACLs can be blank if the ACL was set to private, or
# if the user doesn't have permission. We warn about this with defacl get,
# so just try the modification here and if the user doesn't have
# permission they'll get an AccessDeniedException.
current_acl = bucket.defaultObjectAcl
modification_count = 0
for change in self.changes:
modification_count += change.Execute(url, current_acl, 'defacl',
self.logger)
if modification_count == 0:
self.logger.info('No changes to %s', url)
return
if not current_acl:
# Use a sentinel value to indicate a private (no entries) default
# object ACL.
current_acl.append(PRIVATE_DEFAULT_OBJ_ACL)
try:
preconditions = Preconditions(meta_gen_match=bucket.metageneration)
bucket_metadata = apitools_messages.Bucket(
defaultObjectAcl=current_acl)
self.gsutil_api.PatchBucket(url.bucket_name,
bucket_metadata,
preconditions=preconditions,
provider=url.scheme,
fields=['id'])
except BadRequestException as e:
# Don't retry on bad requests, e.g. invalid email address.
raise CommandException('Received bad request from server: %s' %
str(e))
except AccessDeniedException:
self._WarnServiceAccounts()
raise CommandException(
'Failed to set acl for %s. Please ensure you have '
'OWNER-role access to this resource.' % url)
self.logger.info('Updated default ACL on %s', url)
def RunCommand(self):
"""Command entry point for the defacl command."""
action_subcommand = self.args.pop(0)
self.ParseSubOpts(check_args=True)
self.def_acl = True
self.continue_on_error = False
if action_subcommand == 'get':
func = self._GetDefAcl
elif action_subcommand == 'set':
func = self._SetDefAcl
elif action_subcommand in ('ch', 'change'):
func = self._ChDefAcl
else:
raise CommandException(
('Invalid subcommand "%s" for the %s command.\n'
'See "gsutil help defacl".') %
(action_subcommand, self.command_name))
func()
return 0
class NotificationCommand(Command):
"""Implementation of gsutil notification command."""
# Command specification. See base class for documentation.
command_spec = Command.CreateCommandSpec(
'notification',
command_name_aliases=[
'notify', 'notifyconfig', 'notifications', 'notif'
],
usage_synopsis=_SYNOPSIS,
min_args=3,
max_args=NO_MAX,
supported_sub_args='i:t:',
file_url_ok=False,
provider_url_ok=False,
urls_start_arg=1,
gs_api_support=[ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments={
'watchbucket': [
CommandArgument.MakeFreeTextArgument(),
CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()
],
'stopchannel': []
})
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='notification',
help_name_aliases=['watchbucket', 'stopchannel', 'notifyconfig'],
help_type='command_help',
help_one_line_summary='Configure object change notification',
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={
'watchbucket': _watchbucket_help_text,
'stopchannel': _stopchannel_help_text
},
)
def _WatchBucket(self):
"""Creates a watch on a bucket given in self.args."""
self.CheckArguments()
identifier = None
client_token = None
if self.sub_opts:
for o, a in self.sub_opts:
if o == '-i':
identifier = a
if o == '-t':
client_token = a
identifier = identifier or str(uuid.uuid4())
watch_url = self.args[0]
bucket_arg = self.args[-1]
if not watch_url.lower().startswith('https://'):
raise CommandException(
'The application URL must be an https:// URL.')
bucket_url = StorageUrlFromString(bucket_arg)
if not (bucket_url.IsBucket() and bucket_url.scheme == 'gs'):
raise CommandException(
'The %s command can only be used with gs:// bucket URLs.' %
self.command_name)
if not bucket_url.IsBucket():
raise CommandException(
'URL must name a bucket for the %s command.' %
self.command_name)
self.logger.info('Watching bucket %s with application URL %s ...',
bucket_url, watch_url)
try:
channel = self.gsutil_api.WatchBucket(bucket_url.bucket_name,
watch_url,
identifier,
token=client_token,
provider=bucket_url.scheme)
except AccessDeniedException, e:
self.logger.warn(
NOTIFICATION_AUTHORIZATION_FAILED_MESSAGE.format(
watch_error=str(e), watch_url=watch_url))
raise
channel_id = channel.id
resource_id = channel.resourceId
client_token = channel.token
self.logger.info('Successfully created watch notification channel.')
self.logger.info('Watch channel identifier: %s', channel_id)
self.logger.info('Canonicalized resource identifier: %s', resource_id)
self.logger.info('Client state token: %s', client_token)
return 0
class MbCommand(Command):
"""Implementation of gsutil mb command."""
# Command specification. See base class for documentation.
command_spec = Command.CreateCommandSpec(
'mb',
command_name_aliases=['makebucket', 'createbucket', 'md', 'mkdir'],
usage_synopsis=_SYNOPSIS,
min_args=1,
max_args=NO_MAX,
supported_sub_args='b:c:l:p:s:k:',
supported_private_args=['autoclass', 'retention=', 'pap=', 'rpo='],
file_url_ok=False,
provider_url_ok=False,
urls_start_arg=0,
gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments=[
CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()
],
)
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='mb',
help_name_aliases=[
'createbucket',
'makebucket',
'md',
'mkdir',
'location',
'dra',
'dras',
'reduced_availability',
'durable_reduced_availability',
'rr',
'reduced_redundancy',
'standard',
'storage class',
'nearline',
'nl',
],
help_type='command_help',
help_one_line_summary='Make buckets',
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={},
)
def RunCommand(self):
"""Command entry point for the mb command."""
autoclass = False
bucket_policy_only = None
kms_key = None
location = None
storage_class = None
seconds = None
public_access_prevention = None
rpo = None
json_only_flags_in_command = []
if self.sub_opts:
for o, a in self.sub_opts:
if o == '--autoclass':
autoclass = True
json_only_flags_in_command.append(o)
elif o == '-k':
kms_key = a
ValidateCMEK(kms_key)
json_only_flags_in_command.append(o)
elif o == '-l':
location = a
elif o == '-p':
# Project IDs are sent as header values when using gs and s3 XML APIs.
InsistAscii(
a, 'Invalid non-ASCII character found in project ID')
self.project_id = a
elif o == '-c' or o == '-s':
storage_class = NormalizeStorageClass(a)
elif o == '--retention':
seconds = RetentionInSeconds(a)
elif o == '--rpo':
rpo = a.strip()
if rpo not in VALID_RPO_VALUES:
raise CommandException(
'Invalid value for --rpo. Must be one of: {},'
' provided: {}'.format(VALID_RPO_VALUES_STRING, a))
json_only_flags_in_command.append(o)
elif o == '-b':
InsistOnOrOff(
a, 'Only on and off values allowed for -b option')
bucket_policy_only = (a == 'on')
json_only_flags_in_command.append(o)
elif o == '--pap':
public_access_prevention = a
json_only_flags_in_command.append(o)
bucket_metadata = apitools_messages.Bucket(location=location,
rpo=rpo,
storageClass=storage_class)
if autoclass:
bucket_metadata.autoclass = apitools_messages.Bucket.AutoclassValue(
enabled=autoclass)
if bucket_policy_only or public_access_prevention:
bucket_metadata.iamConfiguration = IamConfigurationValue()
iam_config = bucket_metadata.iamConfiguration
if bucket_policy_only:
iam_config.bucketPolicyOnly = BucketPolicyOnlyValue()
iam_config.bucketPolicyOnly.enabled = bucket_policy_only
if public_access_prevention:
iam_config.publicAccessPrevention = public_access_prevention
if kms_key:
encryption = apitools_messages.Bucket.EncryptionValue()
encryption.defaultKmsKeyName = kms_key
bucket_metadata.encryption = encryption
for bucket_url_str in self.args:
bucket_url = StorageUrlFromString(bucket_url_str)
if seconds is not None:
if bucket_url.scheme != 'gs':
raise CommandException(
'Retention policy can only be specified for '
'GCS buckets.')
retention_policy = (
apitools_messages.Bucket.RetentionPolicyValue(
retentionPeriod=seconds))
bucket_metadata.retentionPolicy = retention_policy
if json_only_flags_in_command and self.gsutil_api.GetApiSelector(
bucket_url.scheme) != ApiSelector.JSON:
raise CommandException(
'The {} option(s) can only be used for GCS'
' Buckets with the JSON API'.format(
', '.join(json_only_flags_in_command)))
if not bucket_url.IsBucket():
raise CommandException(
'The mb command requires a URL that specifies a '
'bucket.\n"%s" is not valid.' % bucket_url)
if (not BUCKET_NAME_RE.match(bucket_url.bucket_name)
or TOO_LONG_DNS_NAME_COMP.search(bucket_url.bucket_name)):
raise InvalidUrlError('Invalid bucket name in URL "%s"' %
bucket_url.bucket_name)
self.logger.info('Creating %s...', bucket_url)
# Pass storage_class param only if this is a GCS bucket. (In S3 the
# storage class is specified on the key object.)
try:
self.gsutil_api.CreateBucket(bucket_url.bucket_name,
project_id=self.project_id,
metadata=bucket_metadata,
provider=bucket_url.scheme)
except AccessDeniedException as e:
message = e.reason
if 'key' in message:
# This will print the error reason and append the following as a
# suggested next step:
#
# To authorize, run:
# gsutil kms authorize \
# -k <kms_key> \
# -p <project_id>
message += ' To authorize, run:\n gsutil kms authorize'
message += ' \\\n -k %s' % kms_key
if (self.project_id):
message += ' \\\n -p %s' % self.project_id
raise CommandException(message)
else:
raise
except BadRequestException as e:
if (e.status == 400
and e.reason == 'DotfulBucketNameNotUnderTld'
and bucket_url.scheme == 'gs'):
bucket_name = bucket_url.bucket_name
final_comp = bucket_name[bucket_name.rfind('.') + 1:]
raise CommandException('\n'.join(
textwrap.wrap(
'Buckets with "." in the name must be valid DNS names. The bucket'
' you are attempting to create (%s) is not a valid DNS name,'
' because the final component (%s) is not currently a valid part'
' of the top-level DNS tree.' %
(bucket_name, final_comp))))
else:
raise
return 0
class LoggingCommand(Command):
"""Implementation of gsutil logging command."""
# Command specification. See base class for documentation.
command_spec = Command.CreateCommandSpec(
'logging',
command_name_aliases=['disablelogging', 'enablelogging', 'getlogging'],
usage_synopsis=_SYNOPSIS,
min_args=2,
max_args=NO_MAX,
supported_sub_args='b:o:',
file_url_ok=False,
provider_url_ok=False,
urls_start_arg=0,
gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments=[
CommandArgument('mode', choices=['on', 'off']),
CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()
]
)
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='logging',
help_name_aliases=['loggingconfig', 'logs', 'log', 'getlogging',
'enablelogging', 'disablelogging'],
help_type='command_help',
help_one_line_summary='Configure or retrieve logging on buckets',
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={'get': _get_help_text, 'set': _set_help_text},
)
def _Get(self):
"""Gets logging configuration for a bucket."""
bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg(
self.args[0], bucket_fields=['logging'])
if bucket_url.scheme == 's3':
sys.stdout.write(self.gsutil_api.XmlPassThroughGetLogging(
bucket_url, provider=bucket_url.scheme))
else:
if (bucket_metadata.logging and bucket_metadata.logging.logBucket and
bucket_metadata.logging.logObjectPrefix):
sys.stdout.write(str(encoding.MessageToJson(
bucket_metadata.logging)) + '\n')
else:
sys.stdout.write('%s has no logging configuration.\n' % bucket_url)
return 0
def _Enable(self):
"""Enables logging configuration for a bucket."""
# Disallow multi-provider 'logging set on' calls, because the schemas
# differ.
if not UrlsAreForSingleProvider(self.args):
raise CommandException('"logging set on" command spanning providers not '
'allowed.')
target_bucket_url = None
target_prefix = None
for opt, opt_arg in self.sub_opts:
if opt == '-b':
target_bucket_url = StorageUrlFromString(opt_arg)
if opt == '-o':
target_prefix = opt_arg
if not target_bucket_url:
raise CommandException('"logging set on" requires \'-b <log_bucket>\' '
'option')
if not target_bucket_url.IsBucket():
raise CommandException('-b option must specify a bucket URL.')
# Iterate over URLs, expanding wildcards and setting logging on each.
some_matched = False
for url_str in self.args:
bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id'])
for blr in bucket_iter:
url = blr.storage_url
some_matched = True
self.logger.info('Enabling logging on %s...', blr)
logging = apitools_messages.Bucket.LoggingValue(
logBucket=target_bucket_url.bucket_name,
logObjectPrefix=target_prefix or url.bucket_name)
bucket_metadata = apitools_messages.Bucket(logging=logging)
self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata,
provider=url.scheme, fields=['id'])
if not some_matched:
raise CommandException(NO_URLS_MATCHED_TARGET % list(self.args))
return 0
def _Disable(self):
"""Disables logging configuration for a bucket."""
# Iterate over URLs, expanding wildcards, and disabling logging on each.
some_matched = False
for url_str in self.args:
bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id'])
for blr in bucket_iter:
url = blr.storage_url
some_matched = True
self.logger.info('Disabling logging on %s...', blr)
logging = apitools_messages.Bucket.LoggingValue()
bucket_metadata = apitools_messages.Bucket(logging=logging)
self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata,
provider=url.scheme, fields=['id'])
if not some_matched:
raise CommandException(NO_URLS_MATCHED_TARGET % list(self.args))
return 0
def RunCommand(self):
"""Command entry point for the logging command."""
# Parse the subcommand and alias for the new logging command.
action_subcommand = self.args.pop(0)
if action_subcommand == 'get':
func = self._Get
elif action_subcommand == 'set':
state_subcommand = self.args.pop(0)
if not self.args:
self.RaiseWrongNumberOfArgumentsException()
if state_subcommand == 'on':
func = self._Enable
elif state_subcommand == 'off':
func = self._Disable
else:
raise CommandException((
'Invalid subcommand "%s" for the "%s %s" command.\n'
'See "gsutil help logging".') % (
state_subcommand, self.command_name, action_subcommand))
else:
raise CommandException(('Invalid subcommand "%s" for the %s command.\n'
'See "gsutil help logging".') %
(action_subcommand, self.command_name))
self.ParseSubOpts(check_args=True)
func()
return 0
class RetentionCommand(Command):
"""Implementation of gsutil retention command."""
# Command specification. See base class for documentation.
command_spec = Command.CreateCommandSpec(
'retention',
command_name_aliases=[],
usage_synopsis=_SYNOPSIS,
min_args=2,
max_args=NO_MAX,
file_url_ok=False,
provider_url_ok=False,
urls_start_arg=1,
gs_api_support=[ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments={
'set': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()],
'clear': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()],
'get': [CommandArgument.MakeNCloudBucketURLsArgument(1)],
'lock': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()],
'event-default': {
'set': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()],
'release': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()]
},
'event': {
'set': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()],
'release': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()]
},
'temp': {
'set': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()],
'release': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()]
},
})
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='retention',
help_name_aliases=[],
help_type='command_help',
help_one_line_summary=(
'Provides utilities to interact with Retention Policy feature.'),
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={
'get': _get_help_text,
'set': _set_help_text,
'clear': _clear_help_text,
'lock': _lock_help_text,
'event-default': _event_default_help_text,
'event': _event_help_text,
'temp': _temp_help_text
},
)
def RunCommand(self):
"""Command entry point for the retention command."""
# If the only credential type the user supplies in their boto file is HMAC,
# GetApiSelector logic will force us to use the XML API, which bucket lock
# does not support at the moment.
if self.gsutil_api.GetApiSelector('gs') != ApiSelector.JSON:
raise CommandException(('The {} command can only be used with the GCS '
'JSON API. If you have only supplied hmac '
'credentials in your boto file, please instead '
'supply a credential type that can be used with '
'the JSON API.').format(self.command_name))
self.preconditions = PreconditionsFromHeaders(self.headers)
action_subcommand = self.args.pop(0)
self.ParseSubOpts(check_args=True)
if action_subcommand == 'set':
func = self._SetRetention
elif action_subcommand == 'clear':
func = self._ClearRetention
elif action_subcommand == 'get':
func = self._GetRetention
elif action_subcommand == 'lock':
func = self._LockRetention
elif action_subcommand == 'event-default':
func = self._DefaultEventHold
elif action_subcommand == 'event':
func = self._EventHold
elif action_subcommand == 'temp':
func = self._TempHold
else:
raise CommandException(
('Invalid subcommand "{}" for the {} command.\n'
'See "gsutil help retention".').format(action_subcommand,
self.command_name))
# Commands with both suboptions and subcommands need to reparse for
# suboptions, so we log again.
metrics.LogCommandParams(subcommands=[action_subcommand],
sub_opts=self.sub_opts)
return func()
def BucketUpdateFunc(self, url_args, bucket_metadata_update, fields,
log_msg_template):
preconditions = Preconditions(
meta_gen_match=self.preconditions.meta_gen_match)
# Iterate over URLs, expanding wildcards and setting the new bucket metadata
# on each bucket.
some_matched = False
for url_str in url_args:
bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id'])
for blr in bucket_iter:
url = blr.storage_url
some_matched = True
self.logger.info(log_msg_template, blr)
self.gsutil_api.PatchBucket(url.bucket_name,
bucket_metadata_update,
preconditions=preconditions,
provider=url.scheme,
fields=fields)
if not some_matched:
raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args))
def ObjectUpdateMetadataFunc(self,
patch_obj_metadata,
log_template,
name_expansion_result,
thread_state=None):
"""Updates metadata on an object using PatchObjectMetadata.
Args:
patch_obj_metadata: Metadata changes that should be applied to the
existing object.
log_template: The log template that should be printed for each object.
name_expansion_result: NameExpansionResult describing target object.
thread_state: gsutil Cloud API instance to use for the operation.
"""
gsutil_api = GetCloudApiInstance(self, thread_state=thread_state)
exp_src_url = name_expansion_result.expanded_storage_url
self.logger.info(log_template, exp_src_url)
cloud_obj_metadata = encoding.JsonToMessage(
apitools_messages.Object, name_expansion_result.expanded_result)
preconditions = Preconditions(
gen_match=self.preconditions.gen_match,
meta_gen_match=self.preconditions.meta_gen_match)
if preconditions.gen_match is None:
preconditions.gen_match = cloud_obj_metadata.generation
if preconditions.meta_gen_match is None:
preconditions.meta_gen_match = cloud_obj_metadata.metageneration
gsutil_api.PatchObjectMetadata(exp_src_url.bucket_name,
exp_src_url.object_name,
patch_obj_metadata,
generation=exp_src_url.generation,
preconditions=preconditions,
provider=exp_src_url.scheme,
fields=['id'])
PutToQueueWithTimeout(gsutil_api.status_queue,
MetadataMessage(message_time=time.time()))
def _GetObjectNameExpansionIterator(self, url_args):
return NameExpansionIterator(
self.command_name,
self.debug,
self.logger,
self.gsutil_api,
url_args,
self.recursion_requested,
all_versions=self.all_versions,
continue_on_error=self.parallel_operations,
bucket_listing_fields=['generation', 'metageneration'])
def _GetSeekAheadNameExpansionIterator(self, url_args):
return SeekAheadNameExpansionIterator(self.command_name,
self.debug,
self.GetSeekAheadGsutilApi(),
url_args,
self.recursion_requested,
all_versions=self.all_versions,
project_id=self.project_id)
def _SetRetention(self):
"""Set retention retention_period on one or more buckets."""
seconds = RetentionInSeconds(self.args[0])
retention_policy = (apitools_messages.Bucket.RetentionPolicyValue(
retentionPeriod=seconds))
log_msg_template = 'Setting Retention Policy on %s...'
bucket_metadata_update = apitools_messages.Bucket(
retentionPolicy=retention_policy)
url_args = self.args[1:]
self.BucketUpdateFunc(url_args,
bucket_metadata_update,
fields=['id', 'retentionPolicy'],
log_msg_template=log_msg_template)
return 0
def _ClearRetention(self):
"""Clear retention retention_period on one or more buckets."""
retention_policy = (apitools_messages.Bucket.RetentionPolicyValue(
retentionPeriod=None))
log_msg_template = 'Clearing Retention Policy on %s...'
bucket_metadata_update = apitools_messages.Bucket(
retentionPolicy=retention_policy)
url_args = self.args
self.BucketUpdateFunc(url_args,
bucket_metadata_update,
fields=['id', 'retentionPolicy'],
log_msg_template=log_msg_template)
return 0
def _GetRetention(self):
"""Get Retention Policy for a single bucket."""
bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg(
self.args[0], bucket_fields=['retentionPolicy'])
print(RetentionPolicyToString(bucket_metadata.retentionPolicy, bucket_url))
return 0
def _LockRetention(self):
"""Lock Retention Policy on one or more buckets."""
url_args = self.args
# Iterate over URLs, expanding wildcards and setting the Retention Policy
# configuration on each.
some_matched = False
for url_str in url_args:
bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id'])
for blr in bucket_iter:
url = blr.storage_url
some_matched = True
# Get bucket metadata to provide a precondition.
bucket_metadata = self.gsutil_api.GetBucket(
url.bucket_name,
provider=url.scheme,
fields=['id', 'metageneration', 'retentionPolicy'])
if (not (bucket_metadata.retentionPolicy and
bucket_metadata.retentionPolicy.retentionPeriod)):
# TODO: implement '-c' flag to continue_on_error
raise CommandException(
'Bucket "{}" does not have an Unlocked Retention Policy.'.format(
url.bucket_name))
elif bucket_metadata.retentionPolicy.isLocked is True:
self.logger.error('Retention Policy on "%s" is already locked.', blr)
elif ConfirmLockRequest(url.bucket_name,
bucket_metadata.retentionPolicy):
self.logger.info('Locking Retention Policy on %s...', blr)
self.gsutil_api.LockRetentionPolicy(url.bucket_name,
bucket_metadata.metageneration,
provider=url.scheme)
else:
self.logger.error(
' Abort Locking Retention Policy on {}'.format(blr))
if not some_matched:
raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args))
return 0
def _DefaultEventHold(self):
"""Sets default value for Event-Based Hold on one or more buckets."""
hold = None
if self.args:
if self.args[0].lower() == 'set':
hold = True
elif self.args[0].lower() == 'release':
hold = False
else:
raise CommandException(
('Invalid subcommand "{}" for the "retention event-default"'
' command.\nSee "gsutil help retention event".').format(
self.sub_opts))
verb = 'Setting' if hold else 'Releasing'
log_msg_template = '{} default Event-Based Hold on %s...'.format(verb)
bucket_metadata_update = apitools_messages.Bucket(
defaultEventBasedHold=hold)
url_args = self.args[1:]
self.BucketUpdateFunc(url_args,
bucket_metadata_update,
fields=['id', 'defaultEventBasedHold'],
log_msg_template=log_msg_template)
return 0
def _EventHold(self):
"""Sets or unsets Event-Based Hold on one or more objects."""
sub_command_name = 'event'
sub_command_full_name = 'Event-Based'
hold = self._ProcessHoldArgs(sub_command_name)
url_args = self.args[1:]
obj_metadata_update_wrapper = (SetEventHoldFuncWrapper
if hold else ReleaseEventHoldFuncWrapper)
self._SetHold(obj_metadata_update_wrapper, url_args, sub_command_full_name)
return 0
def _TempHold(self):
"""Sets or unsets Temporary Hold on one or more objects."""
sub_command_name = 'temp'
sub_command_full_name = 'Temporary'
hold = self._ProcessHoldArgs(sub_command_name)
url_args = self.args[1:]
obj_metadata_update_wrapper = (SetTempHoldFuncWrapper
if hold else ReleaseTempHoldFuncWrapper)
self._SetHold(obj_metadata_update_wrapper, url_args, sub_command_full_name)
return 0
def _ProcessHoldArgs(self, sub_command_name):
"""Processes command args for Temporary and Event-Based Hold sub-command.
Args:
sub_command_name: The name of the subcommand: "temp" / "event"
Returns:
Returns a boolean value indicating whether to set (True) or
release (False)the Hold.
"""
hold = None
if self.args[0].lower() == 'set':
hold = True
elif self.args[0].lower() == 'release':
hold = False
else:
raise CommandException(
('Invalid subcommand "{}" for the "retention {}" command.\n'
'See "gsutil help retention {}".').format(self.args[0],
sub_command_name,
sub_command_name))
return hold
def _SetHold(self, obj_metadata_update_wrapper, url_args,
sub_command_full_name):
"""Common logic to set or unset Event-Based/Temporary Hold on objects.
Args:
obj_metadata_update_wrapper: The function for updating related fields in
Object metadata.
url_args: List of object URIs.
sub_command_full_name: The full name for sub-command:
"Temporary" / "Event-Based"
"""
if len(url_args) == 1 and not self.recursion_requested:
url = StorageUrlFromString(url_args[0])
if not (url.IsCloudUrl() and url.IsObject()):
raise CommandException('URL ({}) must name an object'.format(
url_args[0]))
name_expansion_iterator = self._GetObjectNameExpansionIterator(url_args)
seek_ahead_iterator = self._GetSeekAheadNameExpansionIterator(url_args)
# Used to track if any objects' metadata failed to be set.
self.everything_set_okay = True
try:
# TODO: implement '-c' flag to continue_on_error
# Perform requests in parallel (-m) mode, if requested, using
# configured number of parallel processes and threads. Otherwise,
# perform requests with sequential function calls in current process.
self.Apply(obj_metadata_update_wrapper,
name_expansion_iterator,
UpdateObjectMetadataExceptionHandler,
fail_on_error=True,
seek_ahead_iterator=seek_ahead_iterator)
except AccessDeniedException as e:
if e.status == 403:
self._WarnServiceAccounts()
raise
if not self.everything_set_okay:
raise CommandException(
'{} Hold for some objects could not be set.'.format(
sub_command_full_name))
class MbCommand(Command):
"""Implementation of gsutil mb command."""
# Command specification. See base class for documentation.
command_spec = Command.CreateCommandSpec(
'mb',
command_name_aliases=['makebucket', 'createbucket', 'md', 'mkdir'],
usage_synopsis=_SYNOPSIS,
min_args=1,
max_args=NO_MAX,
supported_sub_args='c:l:p:s:',
file_url_ok=False,
provider_url_ok=False,
urls_start_arg=0,
gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments=[
CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()
]
)
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='mb',
help_name_aliases=[
'createbucket', 'makebucket', 'md', 'mkdir', 'location', 'dra',
'dras', 'reduced_availability', 'durable_reduced_availability', 'rr',
'reduced_redundancy', 'standard', 'storage class', 'nearline', 'nl'],
help_type='command_help',
help_one_line_summary='Make buckets',
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={},
)
def RunCommand(self):
"""Command entry point for the mb command."""
location = None
storage_class = None
if self.sub_opts:
for o, a in self.sub_opts:
if o == '-l':
location = a
elif o == '-p':
# Project IDs are sent as header values when using gs and s3 XML APIs.
InsistAscii(a, 'Invalid non-ASCII character found in project ID')
self.project_id = a
elif o == '-c' or o == '-s':
storage_class = NormalizeStorageClass(a)
bucket_metadata = apitools_messages.Bucket(location=location,
storageClass=storage_class)
for bucket_url_str in self.args:
bucket_url = StorageUrlFromString(bucket_url_str)
if not bucket_url.IsBucket():
raise CommandException('The mb command requires a URL that specifies a '
'bucket.\n"%s" is not valid.' % bucket_url)
if (not BUCKET_NAME_RE.match(bucket_url.bucket_name) or
TOO_LONG_DNS_NAME_COMP.search(bucket_url.bucket_name)):
raise InvalidUrlError(
'Invalid bucket name in URL "%s"' % bucket_url.bucket_name)
self.logger.info('Creating %s...', bucket_url)
# Pass storage_class param only if this is a GCS bucket. (In S3 the
# storage class is specified on the key object.)
try:
self.gsutil_api.CreateBucket(
bucket_url.bucket_name, project_id=self.project_id,
metadata=bucket_metadata, provider=bucket_url.scheme)
except BadRequestException as e:
if (e.status == 400 and e.reason == 'DotfulBucketNameNotUnderTld' and
bucket_url.scheme == 'gs'):
bucket_name = bucket_url.bucket_name
final_comp = bucket_name[bucket_name.rfind('.')+1:]
raise CommandException('\n'.join(textwrap.wrap(
'Buckets with "." in the name must be valid DNS names. The bucket'
' you are attempting to create (%s) is not a valid DNS name,'
' because the final component (%s) is not currently a valid part'
' of the top-level DNS tree.' % (bucket_name, final_comp))))
else:
raise
return 0
class PapCommand(Command):
"""Implements the gsutil pap command."""
command_spec = Command.CreateCommandSpec(
'pap',
command_name_aliases=['publicaccessprevention'],
usage_synopsis=_SYNOPSIS,
min_args=2,
max_args=NO_MAX,
supported_sub_args='',
file_url_ok=False,
provider_url_ok=False,
urls_start_arg=2,
gs_api_support=[ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments={
'get': [
CommandArgument.MakeNCloudURLsArgument(1),
],
'set': [
CommandArgument('mode', choices=['enforced', 'inherited']),
CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()
],
})
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='pap',
help_name_aliases=['publicaccessprevention'],
help_type='command_help',
help_one_line_summary='Configure public access prevention',
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={
'get': _get_help_text,
'set': _set_help_text,
},
)
def _ValidateBucketListingRefAndReturnBucketName(self, blr):
if blr.storage_url.scheme != 'gs':
raise CommandException(
'The %s command can only be used with gs:// bucket URLs.' %
self.command_name)
def _GetPublicAccessPrevention(self, blr):
"""Gets the public access prevention setting for a bucket."""
bucket_url = blr.storage_url
bucket_metadata = self.gsutil_api.GetBucket(
bucket_url.bucket_name,
fields=['iamConfiguration'],
provider=bucket_url.scheme)
iam_config = bucket_metadata.iamConfiguration
public_access_prevention = iam_config.publicAccessPrevention or 'inherited'
bucket = str(bucket_url).rstrip('/')
print('%s: %s' % (bucket, public_access_prevention))
def _SetPublicAccessPrevention(self, blr, setting_arg):
"""Sets the Public Access Prevention setting for a bucket enforced or inherited."""
bucket_url = blr.storage_url
iam_config = IamConfigurationValue()
iam_config.publicAccessPrevention = setting_arg
bucket_metadata = apitools_messages.Bucket(iamConfiguration=iam_config)
print('Setting Public Access Prevention %s for %s' %
(setting_arg, str(bucket_url).rstrip('/')))
self.gsutil_api.PatchBucket(bucket_url.bucket_name,
bucket_metadata,
fields=['iamConfiguration'],
provider=bucket_url.scheme)
return 0
def _Pap(self):
"""Handles pap command on Cloud Storage buckets."""
subcommand = self.args.pop(0)
if subcommand not in ('get', 'set'):
raise CommandException('pap only supports get|set')
subcommand_func = None
subcommand_args = []
setting_arg = None
if subcommand == 'get':
subcommand_func = self._GetPublicAccessPrevention
elif subcommand == 'set':
subcommand_func = self._SetPublicAccessPrevention
setting_arg = self.args.pop(0)
subcommand_args.append(setting_arg)
if self.gsutil_api.GetApiSelector('gs') != ApiSelector.JSON:
raise CommandException('\n'.join(
textwrap.wrap(
('The "%s" command can only be with the Cloud Storage '
'JSON API.') % self.command_name)))
# Iterate over bucket args, performing the specified subsubcommand.
some_matched = False
url_args = self.args
if not url_args:
self.RaiseWrongNumberOfArgumentsException()
for url_str in url_args:
# Throws a CommandException if the argument is not a bucket.
bucket_iter = self.GetBucketUrlIterFromArg(url_str)
for bucket_listing_ref in bucket_iter:
if self.gsutil_api.GetApiSelector(
bucket_listing_ref.storage_url.scheme
) != ApiSelector.JSON:
raise CommandException('\n'.join(
textwrap.wrap(
('The "%s" command can only be used for GCS '
'Buckets.') % self.command_name)))
some_matched = True
subcommand_func(bucket_listing_ref, *subcommand_args)
if not some_matched:
raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args))
return 0
def RunCommand(self):
"""Command entry point for the pap command."""
action_subcommand = self.args[0]
self.ParseSubOpts(check_args=True)
if action_subcommand == 'get' or action_subcommand == 'set':
metrics.LogCommandParams(sub_opts=self.sub_opts)
metrics.LogCommandParams(subcommands=[action_subcommand])
self._Pap()
else:
raise CommandException(
'Invalid subcommand "%s", use get|set instead.' %
action_subcommand)
class WebCommand(Command):
"""Implementation of gsutil web command."""
# Command specification. See base class for documentation.
command_spec = Command.CreateCommandSpec(
'web',
command_name_aliases=['setwebcfg', 'getwebcfg'],
usage_synopsis=_SYNOPSIS,
min_args=2,
max_args=NO_MAX,
supported_sub_args='m:e:',
file_url_ok=False,
provider_url_ok=False,
urls_start_arg=1,
gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments={
'set': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()],
'get': [CommandArgument.MakeNCloudBucketURLsArgument(1)]
})
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='web',
help_name_aliases=['getwebcfg', 'setwebcfg'],
help_type='command_help',
help_one_line_summary=(
'Set a main page and/or error page for one or more buckets'),
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={
'get': _get_help_text,
'set': _set_help_text
},
)
def _GetWeb(self):
"""Gets website configuration for a bucket."""
bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg(
self.args[0], bucket_fields=['website'])
if bucket_url.scheme == 's3':
sys.stdout.write(
self.gsutil_api.XmlPassThroughGetWebsite(
bucket_url, provider=bucket_url.scheme))
else:
if bucket_metadata.website and (
bucket_metadata.website.mainPageSuffix
or bucket_metadata.website.notFoundPage):
sys.stdout.write(
str(encoding.MessageToJson(bucket_metadata.website)) +
'\n')
else:
sys.stdout.write('%s has no website configuration.\n' %
bucket_url)
return 0
def _SetWeb(self):
"""Sets website configuration for a bucket."""
main_page_suffix = None
error_page = None
if self.sub_opts:
for o, a in self.sub_opts:
if o == '-m':
main_page_suffix = a
elif o == '-e':
error_page = a
url_args = self.args
website = apitools_messages.Bucket.WebsiteValue(
mainPageSuffix=main_page_suffix, notFoundPage=error_page)
# Iterate over URLs, expanding wildcards and setting the website
# configuration on each.
some_matched = False
for url_str in url_args:
bucket_iter = self.GetBucketUrlIterFromArg(url_str,
bucket_fields=['id'])
for blr in bucket_iter:
url = blr.storage_url
some_matched = True
self.logger.info('Setting website configuration on %s...', blr)
bucket_metadata = apitools_messages.Bucket(website=website)
self.gsutil_api.PatchBucket(url.bucket_name,
bucket_metadata,
provider=url.scheme,
fields=['id'])
if not some_matched:
raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args))
return 0
def RunCommand(self):
"""Command entry point for the web command."""
action_subcommand = self.args.pop(0)
self.ParseSubOpts(check_args=True)
if action_subcommand == 'get':
func = self._GetWeb
elif action_subcommand == 'set':
func = self._SetWeb
else:
raise CommandException(
('Invalid subcommand "%s" for the %s command.\n'
'See "gsutil help web".') %
(action_subcommand, self.command_name))
# Commands with both suboptions and subcommands need to reparse for
# suboptions, so we log again.
metrics.LogCommandParams(subcommands=[action_subcommand],
sub_opts=self.sub_opts)
return func()
class MbCommand(Command):
"""Implementation of gsutil mb command."""
# Command specification. See base class for documentation.
command_spec = Command.CreateCommandSpec(
'mb',
command_name_aliases=['makebucket', 'createbucket', 'md', 'mkdir'],
usage_synopsis=_SYNOPSIS,
min_args=1,
max_args=NO_MAX,
supported_sub_args='c:l:p:',
file_url_ok=False,
provider_url_ok=False,
urls_start_arg=0,
gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments=[
CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()
])
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='mb',
help_name_aliases=[
'createbucket', 'makebucket', 'md', 'mkdir', 'location', 'dra',
'dras', 'reduced_availability', 'durable_reduced_availability',
'rr', 'reduced_redundancy', 'standard', 'storage class'
],
help_type='command_help',
help_one_line_summary='Make buckets',
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={},
)
def RunCommand(self):
"""Command entry point for the mb command."""
location = None
storage_class = None
if self.sub_opts:
for o, a in self.sub_opts:
if o == '-l':
location = a
elif o == '-p':
self.project_id = a
elif o == '-c':
storage_class = self._Normalize_Storage_Class(a)
bucket_metadata = apitools_messages.Bucket(location=location,
storageClass=storage_class)
for bucket_uri_str in self.args:
bucket_uri = StorageUrlFromString(bucket_uri_str)
if not bucket_uri.IsBucket():
raise CommandException(
'The mb command requires a URI that specifies a '
'bucket.\n"%s" is not valid.' % bucket_uri)
self.logger.info('Creating %s...', bucket_uri)
# Pass storage_class param only if this is a GCS bucket. (In S3 the
# storage class is specified on the key object.)
try:
self.gsutil_api.CreateBucket(bucket_uri.bucket_name,
project_id=self.project_id,
metadata=bucket_metadata,
provider=bucket_uri.scheme)
except BadRequestException as e:
if (e.status == 400
and e.reason == 'DotfulBucketNameNotUnderTld'
and bucket_uri.scheme == 'gs'):
bucket_name = bucket_uri.bucket_name
final_comp = bucket_name[bucket_name.rfind('.') + 1:]
raise CommandException('\n'.join(
textwrap.wrap(
'Buckets with "." in the name must be valid DNS names. The bucket'
' you are attempting to create (%s) is not a valid DNS name,'
' because the final component (%s) is not currently a valid part'
' of the top-level DNS tree.' %
(bucket_name, final_comp))))
else:
raise
return 0
def _Normalize_Storage_Class(self, sc):
sc = sc.upper()
if sc in ('DRA', 'DURABLE_REDUCED_AVAILABILITY'):
return 'DURABLE_REDUCED_AVAILABILITY'
if sc in ('S', 'STD', 'STANDARD'):
return 'STANDARD'
return sc
