class KmsCommand(Command):
"""Implements of gsutil kms command."""
command_spec = Command.CreateCommandSpec(
'kms',
usage_synopsis=_SYNOPSIS,
min_args=1,
max_args=NO_MAX,
supported_sub_args='dk:p:w',
file_url_ok=False,
provider_url_ok=False,
urls_start_arg=1,
gs_api_support=[ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments={
'authorize': [],
'encryption': [CommandArgument.MakeNCloudBucketURLsArgument(1)],
'serviceaccount': [],
})
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='kms',
help_name_aliases=[],
help_type='command_help',
help_one_line_summary='Configure Cloud KMS encryption',
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={
'authorize': _authorize_help_text,
'encryption': _encryption_help_text,
'serviceaccount': _serviceaccount_help_text
},
)
def _GatherSubOptions(self, subcommand_name):
self.CheckArguments()
self.clear_kms_key = False
self.kms_key = None
self.warn_on_key_authorize_failure = False
if self.sub_opts:
for o, a in self.sub_opts:
if o == '-p':
self.project_id = a
elif o == '-k':
self.kms_key = a
ValidateCMEK(self.kms_key)
elif o == '-d':
self.clear_kms_key = True
elif o == '-w':
self.warn_on_key_authorize_failure = True
if self.warn_on_key_authorize_failure and (
self.subcommand_name != 'encryption' or not self.kms_key):
raise CommandException('\n'.join(
textwrap.wrap(
'The "-w" option should only be specified for the "encryption" '
'subcommand and must be used with the "-k" option.')))
# Determine the project (used in the serviceaccount and authorize
# subcommands), either from the "-p" option's value or the default specified
# in the user's Boto config file.
if not self.project_id:
self.project_id = PopulateProjectId(None)
def _AuthorizeProject(self, project_id, kms_key):
"""Authorizes a project's service account to be used with a KMS key.
Authorizes the Cloud Storage-owned service account for project_id to be used
with kms_key.
Args:
project_id: (str) Project id string (not number).
kms_key: (str) Fully qualified resource name for the KMS key.
Returns:
(str, bool) A 2-tuple consisting of:
1) The email address for the service account associated with the project,
which is authorized to encrypt/decrypt with the specified key.
2) A bool value - True if we had to grant the service account permission
to encrypt/decrypt with the given key; False if the required permission
was already present.
"""
# Request the Cloud Storage-owned service account for project_id, creating
# it if it does not exist.
service_account = self.gsutil_api.GetProjectServiceAccount(
project_id, provider='gs').email_address
kms_api = KmsApi(logger=self.logger)
self.logger.debug('Getting IAM policy for %s', kms_key)
try:
policy = kms_api.GetKeyIamPolicy(kms_key)
self.logger.debug('Current policy is %s', policy)
# Check if the required binding is already present; if not, add it and
# update the key's IAM policy.
added_new_binding = False
binding = Binding(role='roles/cloudkms.cryptoKeyEncrypterDecrypter',
members=['serviceAccount:%s' % service_account])
if binding not in policy.bindings:
policy.bindings.append(binding)
kms_api.SetKeyIamPolicy(kms_key, policy)
added_new_binding = True
return (service_account, added_new_binding)
except AccessDeniedException:
if self.warn_on_key_authorize_failure:
text_util.print_to_fd('\n'.join(
textwrap.wrap(
'Warning: Check that your Cloud Platform project\'s service '
'account has the "cloudkms.cryptoKeyEncrypterDecrypter" role '
'for the specified key. Without this role, you may not be '
'able to encrypt or decrypt objects using the key which will '
'prevent you from uploading or downloading objects.')))
return (service_account, False)
else:
raise
def _Authorize(self):
self._GatherSubOptions('authorize')
if not self.kms_key:
raise CommandException('%s %s requires a key to be specified with -k' %
(self.command_name, self.subcommand_name))
_, newly_authorized = self._AuthorizeProject(self.project_id, self.kms_key)
if newly_authorized:
print('Authorized project %s to encrypt and decrypt with key:\n%s' %
(self.project_id, self.kms_key))
else:
print('Project %s was already authorized to encrypt and decrypt with '
'key:\n%s.' % (self.project_id, self.kms_key))
return 0
def _EncryptionClearKey(self, bucket_metadata, bucket_url):
"""Clears the defaultKmsKeyName on a Cloud Storage bucket.
Args:
bucket_metadata: (apitools_messages.Bucket) Metadata for the given bucket.
bucket_url: (gslib.storage_url.StorageUrl) StorageUrl of the given bucket.
"""
bucket_metadata.encryption = apitools_messages.Bucket.EncryptionValue()
print('Clearing default encryption key for %s...' %
str(bucket_url).rstrip('/'))
self.gsutil_api.PatchBucket(bucket_url.bucket_name,
bucket_metadata,
fields=['encryption'],
provider=bucket_url.scheme)
def _EncryptionSetKey(self, bucket_metadata, bucket_url,
svc_acct_for_project_num):
"""Sets defaultKmsKeyName on a Cloud Storage bucket.
Args:
bucket_metadata: (apitools_messages.Bucket) Metadata for the given bucket.
bucket_url: (gslib.storage_url.StorageUrl) StorageUrl of the given bucket.
svc_acct_for_project_num: (Dict[int, str]) Mapping of project numbers to
their corresponding service account.
"""
bucket_project_number = bucket_metadata.projectNumber
try:
# newly_authorized will always be False if the project number is in our
# cache dict, since we've already called _AuthorizeProject on it.
service_account, newly_authorized = (
svc_acct_for_project_num[bucket_project_number], False)
except KeyError:
service_account, newly_authorized = self._AuthorizeProject(
bucket_project_number, self.kms_key)
svc_acct_for_project_num[bucket_project_number] = service_account
if newly_authorized:
text_util.print_to_fd('Authorized service account %s to use key:\n%s' %
(service_account, self.kms_key))
bucket_metadata.encryption = apitools_messages.Bucket.EncryptionValue(
defaultKmsKeyName=self.kms_key)
print('Setting default KMS key for bucket %s...' %
str(bucket_url).rstrip('/'))
self.gsutil_api.PatchBucket(bucket_url.bucket_name,
bucket_metadata,
fields=['encryption'],
provider=bucket_url.scheme)
def _Encryption(self):
self._GatherSubOptions('encryption')
# For each project, we should only make one API call to look up its
# associated Cloud Storage-owned service account; subsequent lookups can be
# pulled from this cache dict.
svc_acct_for_project_num = {}
def _EncryptionForBucket(blr):
"""Set, clear, or get the defaultKmsKeyName for a bucket."""
bucket_url = blr.storage_url
if bucket_url.scheme != 'gs':
raise CommandException(
'The %s command can only be used with gs:// bucket URLs.' %
self.command_name)
# Determine the project from the provided bucket.
bucket_metadata = self.gsutil_api.GetBucket(
bucket_url.bucket_name,
fields=['encryption', 'projectNumber'],
provider=bucket_url.scheme)
# "-d" flag was specified, so clear the default KMS key and return.
if self.clear_kms_key:
self._EncryptionClearKey(bucket_metadata, bucket_url)
return 0
# "-k" flag was specified, so set the default KMS key and return.
if self.kms_key:
self._EncryptionSetKey(bucket_metadata, bucket_url,
svc_acct_for_project_num)
return 0
# Neither "-d" nor "-k" was specified, so emit the default KMS key and
# return.
bucket_url_string = str(bucket_url).rstrip('/')
if (bucket_metadata.encryption and
bucket_metadata.encryption.defaultKmsKeyName):
print('Default encryption key for %s:\n%s' %
(bucket_url_string, bucket_metadata.encryption.defaultKmsKeyName))
else:
print('Bucket %s has no default encryption key' % bucket_url_string)
return 0
# Iterate over bucket args, performing the specified encryption operation
# for each.
some_matched = False
url_args = self.args
if not url_args:
self.RaiseWrongNumberOfArgumentsException()
for url_str in url_args:
# Throws a CommandException if the argument is not a bucket.
bucket_iter = self.GetBucketUrlIterFromArg(url_str)
for bucket_listing_ref in bucket_iter:
some_matched = True
_EncryptionForBucket(bucket_listing_ref)
if not some_matched:
raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args))
return 0
def _ServiceAccount(self):
self.CheckArguments()
if not self.args:
self.args = ['gs://']
if self.sub_opts:
for o, a in self.sub_opts:
if o == '-p':
self.project_id = a
if not self.project_id:
self.project_id = PopulateProjectId(None)
# Request the service account for that project; this might create the
# service account if it doesn't already exist.
self.logger.debug('Checking service account for project %s',
self.project_id)
service_account = self.gsutil_api.GetProjectServiceAccount(
self.project_id, provider='gs').email_address
print(service_account)
return 0
def _RunSubCommand(self, func):
try:
self.sub_opts, self.args = getopt.getopt(
self.args, self.command_spec.supported_sub_args)
# Commands with both suboptions and subcommands need to reparse for
# suboptions, so we log again.
metrics.LogCommandParams(sub_opts=self.sub_opts)
return func(self)
except getopt.GetoptError:
self.RaiseInvalidArgumentException()
def RunCommand(self):
"""Command entry point for the kms command."""
# If the only credential type the user supplies in their boto file is hmac,
# GetApiSelector logic will force us to use the XML API. As the XML API does
# not support all the operations needed for kms subcommands, fail early.
if self.gsutil_api.GetApiSelector(provider='gs') != ApiSelector.JSON:
raise CommandException('\n'.join(
textwrap.wrap(
'The "%s" command can only be used with the GCS JSON API. If you '
'have only supplied hmac credentials in your boto file, please '
'instead supply a credential type that can be used with the JSON '
'API.' % self.command_name)))
def RunCommand(self):
"""Command entry point for the kms command."""
# If the only credential type the user supplies in their boto file is hmac,
# GetApiSelector logic will force us to use the XML API. As the XML API does
# not support all the operations needed for kms subcommands, fail early.
if self.gsutil_api.GetApiSelector(provider='gs') != ApiSelector.JSON:
raise CommandException('\n'.join(
textwrap.wrap(
'The "%s" command can only be used with the GCS JSON API, which '
'cannot use HMAC credentials. Please supply a credential '
'type that is compatible with the JSON API (e.g. OAuth2) in your '
'boto config file.' % self.command_name)))
method_for_subcommand = {
'authorize': KmsCommand._Authorize,
'encryption': KmsCommand._Encryption,
'serviceaccount': KmsCommand._ServiceAccount
}
self.subcommand_name = self.args.pop(0)
if self.subcommand_name in method_for_subcommand:
metrics.LogCommandParams(subcommands=[self.subcommand_name])
return self._RunSubCommand(method_for_subcommand[self.subcommand_name])
else:
raise CommandException('Invalid subcommand "%s" for the %s command.' %
(self.subcommand_name, self.command_name))
class NotificationCommand(Command):
"""Implementation of gsutil notification command."""
# Notification names might look like one of these:
# canonical form: projects/_/buckets/bucket/notificationConfigs/3
# JSON API form: b/bucket/notificationConfigs/5
# Either of the above might start with a / if a user is copying & pasting.
def _GetNotificationPathRegex(self):
if not NotificationCommand._notification_path_regex:
NotificationCommand._notification_path_regex = re.compile(
('/?(projects/[^/]+/)?b(uckets)?/(?P<bucket>[^/]+)/'
'notificationConfigs/(?P<notification>[0-9]+)'))
return NotificationCommand._notification_path_regex
_notification_path_regex = None
# Command specification. See base class for documentation.
command_spec = Command.CreateCommandSpec(
'notification',
command_name_aliases=[
'notify',
'notifyconfig',
'notifications',
'notif',
],
usage_synopsis=_SYNOPSIS,
min_args=2,
max_args=NO_MAX,
supported_sub_args='i:t:m:t:of:e:p:s',
file_url_ok=False,
provider_url_ok=False,
urls_start_arg=1,
gs_api_support=[ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments={
'watchbucket': [
CommandArgument.MakeFreeTextArgument(),
CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(),
],
'stopchannel': [],
'list': [
CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(),
],
'delete': [
# Takes a list of one of the following:
# notification: projects/_/buckets/bla/notificationConfigs/5,
# bucket: gs://foobar
CommandArgument.MakeZeroOrMoreCloudURLsArgument(),
],
'create': [
CommandArgument.MakeFreeTextArgument(), # Cloud Pub/Sub topic
CommandArgument.MakeNCloudBucketURLsArgument(1),
]
},
)
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='notification',
help_name_aliases=[
'watchbucket',
'stopchannel',
'notifyconfig',
],
help_type='command_help',
help_one_line_summary='Configure object change notification',
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={
'create': _create_help_text,
'list': _list_help_text,
'delete': _delete_help_text,
'watchbucket': _watchbucket_help_text,
'stopchannel': _stopchannel_help_text,
},
)
def _WatchBucket(self):
"""Creates a watch on a bucket given in self.args."""
self.CheckArguments()
identifier = None
client_token = None
if self.sub_opts:
for o, a in self.sub_opts:
if o == '-i':
identifier = a
if o == '-t':
client_token = a
identifier = identifier or str(uuid.uuid4())
watch_url = self.args[0]
bucket_arg = self.args[-1]
if not watch_url.lower().startswith('https://'):
raise CommandException(
'The application URL must be an https:// URL.')
bucket_url = StorageUrlFromString(bucket_arg)
if not (bucket_url.IsBucket() and bucket_url.scheme == 'gs'):
raise CommandException(
'The %s command can only be used with gs:// bucket URLs.' %
self.command_name)
if not bucket_url.IsBucket():
raise CommandException(
'URL must name a bucket for the %s command.' %
self.command_name)
self.logger.info('Watching bucket %s with application URL %s ...',
bucket_url, watch_url)
try:
channel = self.gsutil_api.WatchBucket(bucket_url.bucket_name,
watch_url,
identifier,
token=client_token,
provider=bucket_url.scheme)
except AccessDeniedException as e:
self.logger.warn(
NOTIFICATION_AUTHORIZATION_FAILED_MESSAGE.format(
watch_error=str(e), watch_url=watch_url))
raise
channel_id = channel.id
resource_id = channel.resourceId
client_token = channel.token
self.logger.info('Successfully created watch notification channel.')
self.logger.info('Watch channel identifier: %s', channel_id)
self.logger.info('Canonicalized resource identifier: %s', resource_id)
self.logger.info('Client state token: %s', client_token)
return 0
def _StopChannel(self):
channel_id = self.args[0]
resource_id = self.args[1]
self.logger.info('Removing channel %s with resource identifier %s ...',
channel_id, resource_id)
self.gsutil_api.StopChannel(channel_id, resource_id, provider='gs')
self.logger.info('Succesfully removed channel.')
return 0
def _ListChannels(self, bucket_arg):
"""Lists active channel watches on a bucket given in self.args."""
bucket_url = StorageUrlFromString(bucket_arg)
if not (bucket_url.IsBucket() and bucket_url.scheme == 'gs'):
raise CommandException(
'The %s command can only be used with gs:// bucket URLs.' %
self.command_name)
if not bucket_url.IsBucket():
raise CommandException(
'URL must name a bucket for the %s command.' %
self.command_name)
channels = self.gsutil_api.ListChannels(bucket_url.bucket_name,
provider='gs').items
self.logger.info(
'Bucket %s has the following active Object Change Notifications:',
bucket_url.bucket_name)
for idx, channel in enumerate(channels):
self.logger.info('\tNotification channel %d:', idx + 1)
self.logger.info('\t\tChannel identifier: %s', channel.channel_id)
self.logger.info('\t\tResource identifier: %s',
channel.resource_id)
self.logger.info('\t\tApplication URL: %s', channel.push_url)
self.logger.info('\t\tCreated by: %s', channel.subscriber_email)
self.logger.info(
'\t\tCreation time: %s',
str(datetime.fromtimestamp(channel.creation_time_ms / 1000)))
return 0
def _Create(self):
self.CheckArguments()
# User-specified options
pubsub_topic = None
payload_format = None
custom_attributes = {}
event_types = []
object_name_prefix = None
should_setup_topic = True
if self.sub_opts:
for o, a in self.sub_opts:
if o == '-e':
event_types.append(a)
elif o == '-f':
payload_format = a
elif o == '-m':
if ':' not in a:
raise CommandException(
'Custom attributes specified with -m should be of the form '
'key:value')
key, value = a.split(':')
custom_attributes[key] = value
elif o == '-p':
object_name_prefix = a
elif o == '-s':
should_setup_topic = False
elif o == '-t':
pubsub_topic = a
if payload_format not in PAYLOAD_FORMAT_MAP:
raise CommandException(
"Must provide a payload format with -f of either 'json' or 'none'"
)
payload_format = PAYLOAD_FORMAT_MAP[payload_format]
bucket_arg = self.args[-1]
bucket_url = StorageUrlFromString(bucket_arg)
if not bucket_url.IsCloudUrl() or not bucket_url.IsBucket():
raise CommandException(
"%s %s requires a GCS bucket name, but got '%s'" %
(self.command_name, self.subcommand_name, bucket_arg))
if bucket_url.scheme != 'gs':
raise CommandException(
'The %s command can only be used with gs:// bucket URLs.' %
self.command_name)
bucket_name = bucket_url.bucket_name
self.logger.debug('Creating notification for bucket %s', bucket_url)
# Find the project this bucket belongs to
bucket_metadata = self.gsutil_api.GetBucket(bucket_name,
fields=['projectNumber'],
provider=bucket_url.scheme)
bucket_project_number = bucket_metadata.projectNumber
# If not specified, choose a sensible default for the Cloud Pub/Sub topic
# name.
if not pubsub_topic:
pubsub_topic = 'projects/%s/topics/%s' % (PopulateProjectId(None),
bucket_name)
if not pubsub_topic.startswith('projects/'):
# If a user picks a topic ID (mytopic) but doesn't pass the whole name (
# projects/my-project/topics/mytopic ), pick a default project.
pubsub_topic = 'projects/%s/topics/%s' % (PopulateProjectId(None),
pubsub_topic)
self.logger.debug('Using Cloud Pub/Sub topic %s', pubsub_topic)
just_modified_topic_permissions = False
if should_setup_topic:
# Ask GCS for the email address that represents GCS's permission to
# publish to a Cloud Pub/Sub topic from this project.
service_account = self.gsutil_api.GetProjectServiceAccount(
bucket_project_number,
provider=bucket_url.scheme).email_address
self.logger.debug('Service account for project %d: %s',
bucket_project_number, service_account)
just_modified_topic_permissions = self._CreateTopic(
pubsub_topic, service_account)
for attempt_number in range(0, 2):
try:
create_response = self.gsutil_api.CreateNotificationConfig(
bucket_name,
pubsub_topic=pubsub_topic,
payload_format=payload_format,
custom_attributes=custom_attributes,
event_types=event_types if event_types else None,
object_name_prefix=object_name_prefix,
provider=bucket_url.scheme)
break
except PublishPermissionDeniedException:
if attempt_number == 0 and just_modified_topic_permissions:
# If we have just set the IAM policy, it may take up to 10 seconds to
# take effect.
self.logger.info(
'Retrying create notification in 10 seconds '
'(new permissions may take up to 10 seconds to take effect.)'
)
time.sleep(10)
else:
raise
notification_name = 'projects/_/buckets/%s/notificationConfigs/%s' % (
bucket_name, create_response.id)
self.logger.info('Created notification config %s', notification_name)
return 0
def _CreateTopic(self, pubsub_topic, service_account):
"""Assures that a topic exists, creating it if necessary.
Also adds GCS as a publisher on that bucket, if necessary.
Args:
pubsub_topic: name of the Cloud Pub/Sub topic to use/create.
service_account: the GCS service account that needs publish permission.
Returns:
true if we modified IAM permissions, otherwise false.
"""
pubsub_api = PubsubApi(logger=self.logger)
# Verify that the Pub/Sub topic exists. If it does not, create it.
try:
pubsub_api.GetTopic(topic_name=pubsub_topic)
self.logger.debug('Topic %s already exists', pubsub_topic)
except NotFoundException:
self.logger.debug('Creating topic %s', pubsub_topic)
pubsub_api.CreateTopic(topic_name=pubsub_topic)
self.logger.info('Created Cloud Pub/Sub topic %s', pubsub_topic)
# Verify that the service account is in the IAM policy.
policy = pubsub_api.GetTopicIamPolicy(topic_name=pubsub_topic)
binding = Binding(role='roles/pubsub.publisher',
members=['serviceAccount:%s' % service_account])
# This could be more extensive. We could, for instance, check for roles
# that are stronger that pubsub.publisher, like owner. We could also
# recurse up the hierarchy looking to see if there are project-level
# permissions. This can get very complex very quickly, as the caller
# may not necessarily have access to the project-level IAM policy.
# There's no danger in double-granting permission just to make sure it's
# there, though.
if binding not in policy.bindings:
policy.bindings.append(binding)
# transactional safety via etag field.
pubsub_api.SetTopicIamPolicy(topic_name=pubsub_topic,
policy=policy)
return True
else:
self.logger.debug(
'GCS already has publish permission to topic %s.',
pubsub_topic)
return False
def _EnumerateNotificationsFromArgs(self,
accept_notification_configs=True):
"""Yields bucket/notification tuples from command-line args.
Given a list of strings that are bucket names (gs://foo) or notification
config IDs, yield tuples of bucket names and their associated notifications.
Args:
accept_notification_configs: whether notification configs are valid args.
Yields:
Tuples of the form (bucket_name, Notification)
"""
path_regex = self._GetNotificationPathRegex()
for list_entry in self.args:
match = path_regex.match(list_entry)
if match:
if not accept_notification_configs:
raise CommandException(
'%s %s accepts only bucket names, but you provided %s'
%
(self.command_name, self.subcommand_name, list_entry))
bucket_name = match.group('bucket')
notification_id = match.group('notification')
found = False
for notification in self.gsutil_api.ListNotificationConfigs(
bucket_name, provider='gs'):
if notification.id == notification_id:
yield (bucket_name, notification)
found = True
break
if not found:
raise NotFoundException('Could not find notification %s' %
list_entry)
else:
storage_url = StorageUrlFromString(list_entry)
if not storage_url.IsCloudUrl():
raise CommandException(
'The %s command must be used on cloud buckets or notification '
'config names.' % self.command_name)
if storage_url.scheme != 'gs':
raise CommandException(
'The %s command only works on gs:// buckets.')
path = None
if storage_url.IsProvider():
path = 'gs://*'
elif storage_url.IsBucket():
path = list_entry
if not path:
raise CommandException(
'The %s command cannot be used on cloud objects, only buckets'
% self.command_name)
for blr in self.WildcardIterator(path).IterBuckets(
bucket_fields=['id']):
for notification in self.gsutil_api.ListNotificationConfigs(
blr.storage_url.bucket_name, provider='gs'):
yield (blr.storage_url.bucket_name, notification)
def _List(self):
self.CheckArguments()
if self.sub_opts:
if '-o' in dict(self.sub_opts):
for bucket_name in self.args:
self._ListChannels(bucket_name)
else:
for bucket_name, notification in self._EnumerateNotificationsFromArgs(
accept_notification_configs=False):
self._PrintNotificationDetails(bucket_name, notification)
return 0
def _PrintNotificationDetails(self, bucket, notification):
print(
'projects/_/buckets/{bucket}/notificationConfigs/{notification}\n'
'\tCloud Pub/Sub topic: {topic}'.format(
bucket=bucket,
notification=notification.id,
topic=notification.topic[len('//pubsub.googleapis.com/'):]))
if notification.custom_attributes:
print('\tCustom attributes:')
for attr in notification.custom_attributes.additionalProperties:
print('\t\t%s: %s' % (attr.key, attr.value))
filters = []
if notification.event_types:
filters.append('\t\tEvent Types: %s' %
', '.join(notification.event_types))
if notification.object_name_prefix:
filters.append("\t\tObject name prefix: '%s'" %
notification.object_name_prefix)
if filters:
print('\tFilters:')
for line in filters:
print(line)
self.logger.info('')
def _Delete(self):
for bucket_name, notification in self._EnumerateNotificationsFromArgs(
):
self._DeleteNotification(bucket_name, notification.id)
return 0
def _DeleteNotification(self, bucket_name, notification_id):
self.gsutil_api.DeleteNotificationConfig(bucket_name,
notification=notification_id,
provider='gs')
return 0
def _RunSubCommand(self, func):
try:
(self.sub_opts,
self.args) = getopt.getopt(self.args,
self.command_spec.supported_sub_args)
# Commands with both suboptions and subcommands need to reparse for
# suboptions, so we log again.
metrics.LogCommandParams(sub_opts=self.sub_opts)
return func(self)
except getopt.GetoptError:
self.RaiseInvalidArgumentException()
SUBCOMMANDS = {
'create': _Create,
'list': _List,
'delete': _Delete,
'watchbucket': _WatchBucket,
'stopchannel': _StopChannel
}
def RunCommand(self):
"""Command entry point for the notification command."""
self.subcommand_name = self.args.pop(0)
if self.subcommand_name in NotificationCommand.SUBCOMMANDS:
metrics.LogCommandParams(subcommands=[self.subcommand_name])
return self._RunSubCommand(
NotificationCommand.SUBCOMMANDS[self.subcommand_name])
else:
raise CommandException(
'Invalid subcommand "%s" for the %s command.' %
(self.subcommand_name, self.command_name))
class WebCommand(Command):
"""Implementation of gsutil web command."""
# Command specification. See base class for documentation.
command_spec = Command.CreateCommandSpec(
'web',
command_name_aliases=['setwebcfg', 'getwebcfg'],
usage_synopsis=_SYNOPSIS,
min_args=2,
max_args=NO_MAX,
supported_sub_args='m:e:',
file_url_ok=False,
provider_url_ok=False,
urls_start_arg=1,
gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments={
'set': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()],
'get': [CommandArgument.MakeNCloudBucketURLsArgument(1)]
})
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='web',
help_name_aliases=['getwebcfg', 'setwebcfg'],
help_type='command_help',
help_one_line_summary=(
'Set a main page and/or error page for one or more buckets'),
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={
'get': _get_help_text,
'set': _set_help_text
},
)
def _GetWeb(self):
"""Gets website configuration for a bucket."""
bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg(
self.args[0], bucket_fields=['website'])
if bucket_url.scheme == 's3':
sys.stdout.write(
self.gsutil_api.XmlPassThroughGetWebsite(
bucket_url, provider=bucket_url.scheme))
else:
if bucket_metadata.website and (
bucket_metadata.website.mainPageSuffix
or bucket_metadata.website.notFoundPage):
sys.stdout.write(
str(encoding.MessageToJson(bucket_metadata.website)) +
'\n')
else:
sys.stdout.write('%s has no website configuration.\n' %
bucket_url)
return 0
def _SetWeb(self):
"""Sets website configuration for a bucket."""
main_page_suffix = None
error_page = None
if self.sub_opts:
for o, a in self.sub_opts:
if o == '-m':
main_page_suffix = a
elif o == '-e':
error_page = a
url_args = self.args
website = apitools_messages.Bucket.WebsiteValue(
mainPageSuffix=main_page_suffix, notFoundPage=error_page)
# Iterate over URLs, expanding wildcards and setting the website
# configuration on each.
some_matched = False
for url_str in url_args:
bucket_iter = self.GetBucketUrlIterFromArg(url_str,
bucket_fields=['id'])
for blr in bucket_iter:
url = blr.storage_url
some_matched = True
self.logger.info('Setting website configuration on %s...', blr)
bucket_metadata = apitools_messages.Bucket(website=website)
self.gsutil_api.PatchBucket(url.bucket_name,
bucket_metadata,
provider=url.scheme,
fields=['id'])
if not some_matched:
raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args))
return 0
def RunCommand(self):
"""Command entry point for the web command."""
action_subcommand = self.args.pop(0)
self.ParseSubOpts(check_args=True)
if action_subcommand == 'get':
func = self._GetWeb
elif action_subcommand == 'set':
func = self._SetWeb
else:
raise CommandException(
('Invalid subcommand "%s" for the %s command.\n'
'See "gsutil help web".') %
(action_subcommand, self.command_name))
# Commands with both suboptions and subcommands need to reparse for
# suboptions, so we log again.
metrics.LogCommandParams(subcommands=[action_subcommand],
sub_opts=self.sub_opts)
return func()
class CorsCommand(Command):
"""Implementation of gsutil cors command."""
# Command specification. See base class for documentation.
command_spec = Command.CreateCommandSpec(
'cors',
command_name_aliases=['getcors', 'setcors'],
usage_synopsis=_SYNOPSIS,
min_args=2,
max_args=NO_MAX,
supported_sub_args='',
file_url_ok=False,
provider_url_ok=False,
urls_start_arg=1,
gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments={
'set': [
CommandArgument.MakeNFileURLsArgument(1),
CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()
],
'get': [CommandArgument.MakeNCloudBucketURLsArgument(1)]
})
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='cors',
help_name_aliases=['getcors', 'setcors', 'cross-origin'],
help_type='command_help',
help_one_line_summary=(
'Set a CORS JSON document for one or more buckets'),
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={
'get': _get_help_text,
'set': _set_help_text
},
)
def _CalculateUrlsStartArg(self):
if not self.args:
self.RaiseWrongNumberOfArgumentsException()
if self.args[0].lower() == 'set':
return 2
else:
return 1
def _SetCors(self):
"""Sets CORS configuration on a Google Cloud Storage bucket."""
cors_arg = self.args[0]
url_args = self.args[1:]
# Disallow multi-provider 'cors set' requests.
if not UrlsAreForSingleProvider(url_args):
raise CommandException(
'"%s" command spanning providers not allowed.' %
self.command_name)
# Open, read and parse file containing JSON document.
cors_file = open(cors_arg, 'r')
cors_txt = cors_file.read()
cors_file.close()
self.api = self.gsutil_api.GetApiSelector(
StorageUrlFromString(url_args[0]).scheme)
# Iterate over URLs, expanding wildcards and setting the CORS on each.
some_matched = False
for url_str in url_args:
bucket_iter = self.GetBucketUrlIterFromArg(url_str,
bucket_fields=['id'])
for blr in bucket_iter:
url = blr.storage_url
some_matched = True
self.logger.info('Setting CORS on %s...', blr)
if url.scheme == 's3':
self.gsutil_api.XmlPassThroughSetCors(cors_txt,
url,
provider=url.scheme)
else:
cors = CorsTranslation.JsonCorsToMessageEntries(cors_txt)
if not cors:
cors = REMOVE_CORS_CONFIG
bucket_metadata = apitools_messages.Bucket(cors=cors)
self.gsutil_api.PatchBucket(url.bucket_name,
bucket_metadata,
provider=url.scheme,
fields=['id'])
if not some_matched:
raise CommandException('No URLs matched')
return 0
def _GetCors(self):
"""Gets CORS configuration for a Google Cloud Storage bucket."""
bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg(
self.args[0], bucket_fields=['cors'])
if bucket_url.scheme == 's3':
sys.stdout.write(
self.gsutil_api.XmlPassThroughGetCors(
bucket_url, provider=bucket_url.scheme))
else:
if bucket_metadata.cors:
sys.stdout.write(
CorsTranslation.MessageEntriesToJson(bucket_metadata.cors))
else:
sys.stdout.write('%s has no CORS configuration.\n' %
bucket_url)
return 0
def RunCommand(self):
"""Command entry point for the cors command."""
action_subcommand = self.args.pop(0)
if action_subcommand == 'get':
func = self._GetCors
elif action_subcommand == 'set':
func = self._SetCors
else:
raise CommandException(
('Invalid subcommand "%s" for the %s command.\n'
'See "gsutil help cors".') %
(action_subcommand, self.command_name))
return func()
class NotificationCommand(Command):
"""Implementation of gsutil notification command."""
# Notification names might look like one of these:
# canonical form: projects/_/buckets/bucket/notificationConfigs/3
# JSON API form: b/bucket/notificationConfigs/5
# Either of the above might start with a / if a user is copying & pasting.
def _GetNotificationPathRegex(self):
if not NotificationCommand._notification_path_regex:
NotificationCommand._notification_path_regex = re.compile(
('/?(projects/[^/]+/)?b(uckets)?/(?P<bucket>[^/]+)/'
'notificationConfigs/(?P<notification>[0-9]+)'))
return NotificationCommand._notification_path_regex
_notification_path_regex = None
# Command specification. See base class for documentation.
command_spec = Command.CreateCommandSpec(
'notification',
command_name_aliases=[
'notify', 'notifyconfig', 'notifications', 'notif'
],
usage_synopsis=_SYNOPSIS,
min_args=2,
max_args=NO_MAX,
supported_sub_args='i:t:m:t:o:f:e:p:s',
file_url_ok=False,
provider_url_ok=False,
urls_start_arg=1,
gs_api_support=[ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments={
'watchbucket': [
CommandArgument.MakeFreeTextArgument(),
CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()
],
'stopchannel': [],
'list': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()],
'delete': [
# Takes a list of one of the following:
# notification: projects/_/buckets/bla/notificationConfigs/5,
# bucket: gs://foobar
CommandArgument.MakeZeroOrMoreCloudURLsArgument()
],
'create': [
CommandArgument.MakeFreeTextArgument(), # Cloud Pub/Sub topic
CommandArgument.MakeNCloudBucketURLsArgument(1)
]
})
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='notification',
help_name_aliases=['watchbucket', 'stopchannel', 'notifyconfig'],
help_type='command_help',
help_one_line_summary='Configure object change notification',
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={
'create': _create_help_text,
'list': _list_help_text,
'delete': _delete_help_text,
'watchbucket': _watchbucket_help_text,
'stopchannel': _stopchannel_help_text
},
)
def _WatchBucket(self):
"""Creates a watch on a bucket given in self.args."""
self.CheckArguments()
identifier = None
client_token = None
if self.sub_opts:
for o, a in self.sub_opts:
if o == '-i':
identifier = a
if o == '-t':
client_token = a
identifier = identifier or str(uuid.uuid4())
watch_url = self.args[0]
bucket_arg = self.args[-1]
if not watch_url.lower().startswith('https://'):
raise CommandException(
'The application URL must be an https:// URL.')
bucket_url = StorageUrlFromString(bucket_arg)
if not (bucket_url.IsBucket() and bucket_url.scheme == 'gs'):
raise CommandException(
'The %s command can only be used with gs:// bucket URLs.' %
self.command_name)
if not bucket_url.IsBucket():
raise CommandException(
'URL must name a bucket for the %s command.' %
self.command_name)
self.logger.info('Watching bucket %s with application URL %s ...',
bucket_url, watch_url)
try:
channel = self.gsutil_api.WatchBucket(bucket_url.bucket_name,
watch_url,
identifier,
token=client_token,
provider=bucket_url.scheme)
except AccessDeniedException, e:
self.logger.warn(
NOTIFICATION_AUTHORIZATION_FAILED_MESSAGE.format(
watch_error=str(e), watch_url=watch_url))
raise
channel_id = channel.id
resource_id = channel.resourceId
client_token = channel.token
self.logger.info('Successfully created watch notification channel.')
self.logger.info('Watch channel identifier: %s', channel_id)
self.logger.info('Canonicalized resource identifier: %s', resource_id)
self.logger.info('Client state token: %s', client_token)
return 0
class RetentionCommand(Command):
"""Implementation of gsutil retention command."""
# Command specification. See base class for documentation.
command_spec = Command.CreateCommandSpec(
'retention',
command_name_aliases=[],
usage_synopsis=_SYNOPSIS,
min_args=2,
max_args=NO_MAX,
file_url_ok=False,
provider_url_ok=False,
urls_start_arg=1,
gs_api_support=[ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments={
'set': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()],
'clear': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()],
'get': [CommandArgument.MakeNCloudBucketURLsArgument(1)],
'lock': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()],
'event-default': {
'set': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()],
'release': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()]
},
'event': {
'set': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()],
'release': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()]
},
'temp': {
'set': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()],
'release': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()]
},
})
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='retention',
help_name_aliases=[],
help_type='command_help',
help_one_line_summary=(
'Provides utilities to interact with Retention Policy feature.'),
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={
'get': _get_help_text,
'set': _set_help_text,
'clear': _clear_help_text,
'lock': _lock_help_text,
'event-default': _event_default_help_text,
'event': _event_help_text,
'temp': _temp_help_text
},
)
def RunCommand(self):
"""Command entry point for the retention command."""
# If the only credential type the user supplies in their boto file is HMAC,
# GetApiSelector logic will force us to use the XML API, which bucket lock
# does not support at the moment.
if self.gsutil_api.GetApiSelector('gs') != ApiSelector.JSON:
raise CommandException(('The {} command can only be used with the GCS '
'JSON API. If you have only supplied hmac '
'credentials in your boto file, please instead '
'supply a credential type that can be used with '
'the JSON API.').format(self.command_name))
self.preconditions = PreconditionsFromHeaders(self.headers)
action_subcommand = self.args.pop(0)
self.ParseSubOpts(check_args=True)
if action_subcommand == 'set':
func = self._SetRetention
elif action_subcommand == 'clear':
func = self._ClearRetention
elif action_subcommand == 'get':
func = self._GetRetention
elif action_subcommand == 'lock':
func = self._LockRetention
elif action_subcommand == 'event-default':
func = self._DefaultEventHold
elif action_subcommand == 'event':
func = self._EventHold
elif action_subcommand == 'temp':
func = self._TempHold
else:
raise CommandException(
('Invalid subcommand "{}" for the {} command.\n'
'See "gsutil help retention".').format(action_subcommand,
self.command_name))
# Commands with both suboptions and subcommands need to reparse for
# suboptions, so we log again.
metrics.LogCommandParams(subcommands=[action_subcommand],
sub_opts=self.sub_opts)
return func()
def BucketUpdateFunc(self, url_args, bucket_metadata_update, fields,
log_msg_template):
preconditions = Preconditions(
meta_gen_match=self.preconditions.meta_gen_match)
# Iterate over URLs, expanding wildcards and setting the new bucket metadata
# on each bucket.
some_matched = False
for url_str in url_args:
bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id'])
for blr in bucket_iter:
url = blr.storage_url
some_matched = True
self.logger.info(log_msg_template, blr)
self.gsutil_api.PatchBucket(url.bucket_name,
bucket_metadata_update,
preconditions=preconditions,
provider=url.scheme,
fields=fields)
if not some_matched:
raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args))
def ObjectUpdateMetadataFunc(self,
patch_obj_metadata,
log_template,
name_expansion_result,
thread_state=None):
"""Updates metadata on an object using PatchObjectMetadata.
Args:
patch_obj_metadata: Metadata changes that should be applied to the
existing object.
log_template: The log template that should be printed for each object.
name_expansion_result: NameExpansionResult describing target object.
thread_state: gsutil Cloud API instance to use for the operation.
"""
gsutil_api = GetCloudApiInstance(self, thread_state=thread_state)
exp_src_url = name_expansion_result.expanded_storage_url
self.logger.info(log_template, exp_src_url)
cloud_obj_metadata = encoding.JsonToMessage(
apitools_messages.Object, name_expansion_result.expanded_result)
preconditions = Preconditions(
gen_match=self.preconditions.gen_match,
meta_gen_match=self.preconditions.meta_gen_match)
if preconditions.gen_match is None:
preconditions.gen_match = cloud_obj_metadata.generation
if preconditions.meta_gen_match is None:
preconditions.meta_gen_match = cloud_obj_metadata.metageneration
gsutil_api.PatchObjectMetadata(exp_src_url.bucket_name,
exp_src_url.object_name,
patch_obj_metadata,
generation=exp_src_url.generation,
preconditions=preconditions,
provider=exp_src_url.scheme,
fields=['id'])
PutToQueueWithTimeout(gsutil_api.status_queue,
MetadataMessage(message_time=time.time()))
def _GetObjectNameExpansionIterator(self, url_args):
return NameExpansionIterator(
self.command_name,
self.debug,
self.logger,
self.gsutil_api,
url_args,
self.recursion_requested,
all_versions=self.all_versions,
continue_on_error=self.parallel_operations,
bucket_listing_fields=['generation', 'metageneration'])
def _GetSeekAheadNameExpansionIterator(self, url_args):
return SeekAheadNameExpansionIterator(self.command_name,
self.debug,
self.GetSeekAheadGsutilApi(),
url_args,
self.recursion_requested,
all_versions=self.all_versions,
project_id=self.project_id)
def _SetRetention(self):
"""Set retention retention_period on one or more buckets."""
seconds = RetentionInSeconds(self.args[0])
retention_policy = (apitools_messages.Bucket.RetentionPolicyValue(
retentionPeriod=seconds))
log_msg_template = 'Setting Retention Policy on %s...'
bucket_metadata_update = apitools_messages.Bucket(
retentionPolicy=retention_policy)
url_args = self.args[1:]
self.BucketUpdateFunc(url_args,
bucket_metadata_update,
fields=['id', 'retentionPolicy'],
log_msg_template=log_msg_template)
return 0
def _ClearRetention(self):
"""Clear retention retention_period on one or more buckets."""
retention_policy = (apitools_messages.Bucket.RetentionPolicyValue(
retentionPeriod=None))
log_msg_template = 'Clearing Retention Policy on %s...'
bucket_metadata_update = apitools_messages.Bucket(
retentionPolicy=retention_policy)
url_args = self.args
self.BucketUpdateFunc(url_args,
bucket_metadata_update,
fields=['id', 'retentionPolicy'],
log_msg_template=log_msg_template)
return 0
def _GetRetention(self):
"""Get Retention Policy for a single bucket."""
bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg(
self.args[0], bucket_fields=['retentionPolicy'])
print(RetentionPolicyToString(bucket_metadata.retentionPolicy, bucket_url))
return 0
def _LockRetention(self):
"""Lock Retention Policy on one or more buckets."""
url_args = self.args
# Iterate over URLs, expanding wildcards and setting the Retention Policy
# configuration on each.
some_matched = False
for url_str in url_args:
bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id'])
for blr in bucket_iter:
url = blr.storage_url
some_matched = True
# Get bucket metadata to provide a precondition.
bucket_metadata = self.gsutil_api.GetBucket(
url.bucket_name,
provider=url.scheme,
fields=['id', 'metageneration', 'retentionPolicy'])
if (not (bucket_metadata.retentionPolicy and
bucket_metadata.retentionPolicy.retentionPeriod)):
# TODO: implement '-c' flag to continue_on_error
raise CommandException(
'Bucket "{}" does not have an Unlocked Retention Policy.'.format(
url.bucket_name))
elif bucket_metadata.retentionPolicy.isLocked is True:
self.logger.error('Retention Policy on "%s" is already locked.', blr)
elif ConfirmLockRequest(url.bucket_name,
bucket_metadata.retentionPolicy):
self.logger.info('Locking Retention Policy on %s...', blr)
self.gsutil_api.LockRetentionPolicy(url.bucket_name,
bucket_metadata.metageneration,
provider=url.scheme)
else:
self.logger.error(
' Abort Locking Retention Policy on {}'.format(blr))
if not some_matched:
raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args))
return 0
def _DefaultEventHold(self):
"""Sets default value for Event-Based Hold on one or more buckets."""
hold = None
if self.args:
if self.args[0].lower() == 'set':
hold = True
elif self.args[0].lower() == 'release':
hold = False
else:
raise CommandException(
('Invalid subcommand "{}" for the "retention event-default"'
' command.\nSee "gsutil help retention event".').format(
self.sub_opts))
verb = 'Setting' if hold else 'Releasing'
log_msg_template = '{} default Event-Based Hold on %s...'.format(verb)
bucket_metadata_update = apitools_messages.Bucket(
defaultEventBasedHold=hold)
url_args = self.args[1:]
self.BucketUpdateFunc(url_args,
bucket_metadata_update,
fields=['id', 'defaultEventBasedHold'],
log_msg_template=log_msg_template)
return 0
def _EventHold(self):
"""Sets or unsets Event-Based Hold on one or more objects."""
sub_command_name = 'event'
sub_command_full_name = 'Event-Based'
hold = self._ProcessHoldArgs(sub_command_name)
url_args = self.args[1:]
obj_metadata_update_wrapper = (SetEventHoldFuncWrapper
if hold else ReleaseEventHoldFuncWrapper)
self._SetHold(obj_metadata_update_wrapper, url_args, sub_command_full_name)
return 0
def _TempHold(self):
"""Sets or unsets Temporary Hold on one or more objects."""
sub_command_name = 'temp'
sub_command_full_name = 'Temporary'
hold = self._ProcessHoldArgs(sub_command_name)
url_args = self.args[1:]
obj_metadata_update_wrapper = (SetTempHoldFuncWrapper
if hold else ReleaseTempHoldFuncWrapper)
self._SetHold(obj_metadata_update_wrapper, url_args, sub_command_full_name)
return 0
def _ProcessHoldArgs(self, sub_command_name):
"""Processes command args for Temporary and Event-Based Hold sub-command.
Args:
sub_command_name: The name of the subcommand: "temp" / "event"
Returns:
Returns a boolean value indicating whether to set (True) or
release (False)the Hold.
"""
hold = None
if self.args[0].lower() == 'set':
hold = True
elif self.args[0].lower() == 'release':
hold = False
else:
raise CommandException(
('Invalid subcommand "{}" for the "retention {}" command.\n'
'See "gsutil help retention {}".').format(self.args[0],
sub_command_name,
sub_command_name))
return hold
def _SetHold(self, obj_metadata_update_wrapper, url_args,
sub_command_full_name):
"""Common logic to set or unset Event-Based/Temporary Hold on objects.
Args:
obj_metadata_update_wrapper: The function for updating related fields in
Object metadata.
url_args: List of object URIs.
sub_command_full_name: The full name for sub-command:
"Temporary" / "Event-Based"
"""
if len(url_args) == 1 and not self.recursion_requested:
url = StorageUrlFromString(url_args[0])
if not (url.IsCloudUrl() and url.IsObject()):
raise CommandException('URL ({}) must name an object'.format(
url_args[0]))
name_expansion_iterator = self._GetObjectNameExpansionIterator(url_args)
seek_ahead_iterator = self._GetSeekAheadNameExpansionIterator(url_args)
# Used to track if any objects' metadata failed to be set.
self.everything_set_okay = True
try:
# TODO: implement '-c' flag to continue_on_error
# Perform requests in parallel (-m) mode, if requested, using
# configured number of parallel processes and threads. Otherwise,
# perform requests with sequential function calls in current process.
self.Apply(obj_metadata_update_wrapper,
name_expansion_iterator,
UpdateObjectMetadataExceptionHandler,
fail_on_error=True,
seek_ahead_iterator=seek_ahead_iterator)
except AccessDeniedException as e:
if e.status == 403:
self._WarnServiceAccounts()
raise
if not self.everything_set_okay:
raise CommandException(
'{} Hold for some objects could not be set.'.format(
sub_command_full_name))
class LifecycleCommand(Command):
"""Implementation of gsutil lifecycle command."""
# Command specification. See base class for documentation.
command_spec = Command.CreateCommandSpec(
'lifecycle',
command_name_aliases=['lifecycleconfig'],
usage_synopsis=_SYNOPSIS,
min_args=2,
max_args=NO_MAX,
supported_sub_args='',
file_url_ok=True,
provider_url_ok=False,
urls_start_arg=1,
gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments={
'set': [
CommandArgument.MakeNFileURLsArgument(1),
CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()
],
'get': [
CommandArgument.MakeNCloudBucketURLsArgument(1)
]
}
)
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='lifecycle',
help_name_aliases=['getlifecycle', 'setlifecycle'],
help_type='command_help',
help_one_line_summary=(
'Get or set lifecycle configuration for a bucket'),
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={'get': _get_help_text, 'set': _set_help_text},
)
def _SetLifecycleConfig(self):
"""Sets lifecycle configuration for a Google Cloud Storage bucket."""
lifecycle_arg = self.args[0]
url_args = self.args[1:]
# Disallow multi-provider 'lifecycle set' requests.
if not UrlsAreForSingleProvider(url_args):
raise CommandException('"%s" command spanning providers not allowed.' %
self.command_name)
# Open, read and parse file containing JSON document.
lifecycle_file = open(lifecycle_arg, 'r')
lifecycle_txt = lifecycle_file.read()
lifecycle_file.close()
# Iterate over URLs, expanding wildcards and setting the lifecycle on each.
some_matched = False
for url_str in url_args:
bucket_iter = self.GetBucketUrlIterFromArg(url_str,
bucket_fields=['lifecycle'])
for blr in bucket_iter:
url = blr.storage_url
some_matched = True
self.logger.info('Setting lifecycle configuration on %s...', blr)
if url.scheme == 's3':
self.gsutil_api.XmlPassThroughSetLifecycle(
lifecycle_txt, url, provider=url.scheme)
else:
lifecycle = LifecycleTranslation.JsonLifecycleToMessage(lifecycle_txt)
bucket_metadata = apitools_messages.Bucket(lifecycle=lifecycle)
self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata,
provider=url.scheme, fields=['id'])
if not some_matched:
raise CommandException('No URLs matched')
return 0
def _GetLifecycleConfig(self):
"""Gets lifecycle configuration for a Google Cloud Storage bucket."""
bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg(
self.args[0], bucket_fields=['lifecycle'])
if bucket_url.scheme == 's3':
sys.stdout.write(self.gsutil_api.XmlPassThroughGetLifecycle(
bucket_url, provider=bucket_url.scheme))
else:
if bucket_metadata.lifecycle and bucket_metadata.lifecycle.rule:
sys.stdout.write(LifecycleTranslation.JsonLifecycleFromMessage(
bucket_metadata.lifecycle))
else:
sys.stdout.write('%s has no lifecycle configuration.\n' % bucket_url)
return 0
def RunCommand(self):
"""Command entry point for the lifecycle command."""
subcommand = self.args.pop(0)
if subcommand == 'get':
return self._GetLifecycleConfig()
elif subcommand == 'set':
return self._SetLifecycleConfig()
else:
raise CommandException('Invalid subcommand "%s" for the %s command.' %
(subcommand, self.command_name))
class DefAclCommand(Command):
"""Implementation of gsutil defacl command."""
# Command specification. See base class for documentation.
command_spec = Command.CreateCommandSpec(
'defacl',
command_name_aliases=['setdefacl', 'getdefacl', 'chdefacl'],
usage_synopsis=_SYNOPSIS,
min_args=2,
max_args=NO_MAX,
supported_sub_args='fg:u:d:p:',
file_url_ok=False,
provider_url_ok=False,
urls_start_arg=1,
gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments={
'set': [
CommandArgument.MakeFileURLOrCannedACLArgument(),
CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()
],
'get': [CommandArgument.MakeNCloudBucketURLsArgument(1)],
'ch': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()],
})
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='defacl',
help_name_aliases=[
'default acl', 'setdefacl', 'getdefacl', 'chdefacl'
],
help_type='command_help',
help_one_line_summary='Get, set, or change default ACL on buckets',
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={
'get': _get_help_text,
'set': _set_help_text,
'ch': _ch_help_text
},
)
def _CalculateUrlsStartArg(self):
if not self.args:
self.RaiseWrongNumberOfArgumentsException()
if (self.args[0].lower() == 'set'
or self.command_alias_used == 'setdefacl'):
return 1
else:
return 0
def _SetDefAcl(self):
if not StorageUrlFromString(self.args[-1]).IsBucket():
raise CommandException(
'URL must name a bucket for the %s command' %
self.command_name)
try:
self.SetAclCommandHelper(SetAclFuncWrapper, SetAclExceptionHandler)
except AccessDeniedException:
self._WarnServiceAccounts()
raise
def _GetDefAcl(self):
if not StorageUrlFromString(self.args[0]).IsBucket():
raise CommandException(
'URL must name a bucket for the %s command' %
self.command_name)
self.GetAndPrintAcl(self.args[0])
def _ChDefAcl(self):
"""Parses options and changes default object ACLs on specified buckets."""
self.parse_versions = True
self.changes = []
if self.sub_opts:
for o, a in self.sub_opts:
if o == '-g':
self.changes.append(
aclhelpers.AclChange(
a, scope_type=aclhelpers.ChangeType.GROUP))
if o == '-u':
self.changes.append(
aclhelpers.AclChange(
a, scope_type=aclhelpers.ChangeType.USER))
if o == '-p':
self.changes.append(
aclhelpers.AclChange(
a, scope_type=aclhelpers.ChangeType.PROJECT))
if o == '-d':
self.changes.append(aclhelpers.AclDel(a))
if not self.changes:
raise CommandException('Please specify at least one access change '
'with the -g, -u, or -d flags')
if (not UrlsAreForSingleProvider(self.args)
or StorageUrlFromString(self.args[0]).scheme != 'gs'):
raise CommandException(
'The "{0}" command can only be used with gs:// URLs'.format(
self.command_name))
bucket_urls = set()
for url_arg in self.args:
for result in self.WildcardIterator(url_arg):
if not result.storage_url.IsBucket():
raise CommandException(
'The defacl ch command can only be applied to buckets.'
)
bucket_urls.add(result.storage_url)
for storage_url in bucket_urls:
self.ApplyAclChanges(storage_url)
@Retry(ServiceException, tries=3, timeout_secs=1)
def ApplyAclChanges(self, url):
"""Applies the changes in self.changes to the provided URL."""
bucket = self.gsutil_api.GetBucket(
url.bucket_name,
provider=url.scheme,
fields=['defaultObjectAcl', 'metageneration'])
# Default object ACLs can be blank if the ACL was set to private, or
# if the user doesn't have permission. We warn about this with defacl get,
# so just try the modification here and if the user doesn't have
# permission they'll get an AccessDeniedException.
current_acl = bucket.defaultObjectAcl
modification_count = 0
for change in self.changes:
modification_count += change.Execute(url, current_acl, 'defacl',
self.logger)
if modification_count == 0:
self.logger.info('No changes to %s', url)
return
if not current_acl:
# Use a sentinel value to indicate a private (no entries) default
# object ACL.
current_acl.append(PRIVATE_DEFAULT_OBJ_ACL)
try:
preconditions = Preconditions(meta_gen_match=bucket.metageneration)
bucket_metadata = apitools_messages.Bucket(
defaultObjectAcl=current_acl)
self.gsutil_api.PatchBucket(url.bucket_name,
bucket_metadata,
preconditions=preconditions,
provider=url.scheme,
fields=['id'])
except BadRequestException as e:
# Don't retry on bad requests, e.g. invalid email address.
raise CommandException('Received bad request from server: %s' %
str(e))
except AccessDeniedException:
self._WarnServiceAccounts()
raise CommandException(
'Failed to set acl for %s. Please ensure you have '
'OWNER-role access to this resource.' % url)
self.logger.info('Updated default ACL on %s', url)
def RunCommand(self):
"""Command entry point for the defacl command."""
action_subcommand = self.args.pop(0)
self.ParseSubOpts(check_args=True)
self.def_acl = True
self.continue_on_error = False
if action_subcommand == 'get':
func = self._GetDefAcl
elif action_subcommand == 'set':
func = self._SetDefAcl
elif action_subcommand in ('ch', 'change'):
func = self._ChDefAcl
else:
raise CommandException(
('Invalid subcommand "%s" for the %s command.\n'
'See "gsutil help defacl".') %
(action_subcommand, self.command_name))
func()
return 0
