class KmsCommand(Command): """Implements of gsutil kms command.""" command_spec = Command.CreateCommandSpec( 'kms', usage_synopsis=_SYNOPSIS, min_args=1, max_args=NO_MAX, supported_sub_args='dk:p:w', file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'authorize': [], 'encryption': [CommandArgument.MakeNCloudBucketURLsArgument(1)], 'serviceaccount': [], }) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='kms', help_name_aliases=[], help_type='command_help', help_one_line_summary='Configure Cloud KMS encryption', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'authorize': _authorize_help_text, 'encryption': _encryption_help_text, 'serviceaccount': _serviceaccount_help_text }, ) def _GatherSubOptions(self, subcommand_name): self.CheckArguments() self.clear_kms_key = False self.kms_key = None self.warn_on_key_authorize_failure = False if self.sub_opts: for o, a in self.sub_opts: if o == '-p': self.project_id = a elif o == '-k': self.kms_key = a ValidateCMEK(self.kms_key) elif o == '-d': self.clear_kms_key = True elif o == '-w': self.warn_on_key_authorize_failure = True if self.warn_on_key_authorize_failure and ( self.subcommand_name != 'encryption' or not self.kms_key): raise CommandException('\n'.join( textwrap.wrap( 'The "-w" option should only be specified for the "encryption" ' 'subcommand and must be used with the "-k" option.'))) # Determine the project (used in the serviceaccount and authorize # subcommands), either from the "-p" option's value or the default specified # in the user's Boto config file. if not self.project_id: self.project_id = PopulateProjectId(None) def _AuthorizeProject(self, project_id, kms_key): """Authorizes a project's service account to be used with a KMS key. Authorizes the Cloud Storage-owned service account for project_id to be used with kms_key. Args: project_id: (str) Project id string (not number). kms_key: (str) Fully qualified resource name for the KMS key. Returns: (str, bool) A 2-tuple consisting of: 1) The email address for the service account associated with the project, which is authorized to encrypt/decrypt with the specified key. 2) A bool value - True if we had to grant the service account permission to encrypt/decrypt with the given key; False if the required permission was already present. """ # Request the Cloud Storage-owned service account for project_id, creating # it if it does not exist. service_account = self.gsutil_api.GetProjectServiceAccount( project_id, provider='gs').email_address kms_api = KmsApi(logger=self.logger) self.logger.debug('Getting IAM policy for %s', kms_key) try: policy = kms_api.GetKeyIamPolicy(kms_key) self.logger.debug('Current policy is %s', policy) # Check if the required binding is already present; if not, add it and # update the key's IAM policy. added_new_binding = False binding = Binding(role='roles/cloudkms.cryptoKeyEncrypterDecrypter', members=['serviceAccount:%s' % service_account]) if binding not in policy.bindings: policy.bindings.append(binding) kms_api.SetKeyIamPolicy(kms_key, policy) added_new_binding = True return (service_account, added_new_binding) except AccessDeniedException: if self.warn_on_key_authorize_failure: text_util.print_to_fd('\n'.join( textwrap.wrap( 'Warning: Check that your Cloud Platform project\'s service ' 'account has the "cloudkms.cryptoKeyEncrypterDecrypter" role ' 'for the specified key. Without this role, you may not be ' 'able to encrypt or decrypt objects using the key which will ' 'prevent you from uploading or downloading objects.'))) return (service_account, False) else: raise def _Authorize(self): self._GatherSubOptions('authorize') if not self.kms_key: raise CommandException('%s %s requires a key to be specified with -k' % (self.command_name, self.subcommand_name)) _, newly_authorized = self._AuthorizeProject(self.project_id, self.kms_key) if newly_authorized: print('Authorized project %s to encrypt and decrypt with key:\n%s' % (self.project_id, self.kms_key)) else: print('Project %s was already authorized to encrypt and decrypt with ' 'key:\n%s.' % (self.project_id, self.kms_key)) return 0 def _EncryptionClearKey(self, bucket_metadata, bucket_url): """Clears the defaultKmsKeyName on a Cloud Storage bucket. Args: bucket_metadata: (apitools_messages.Bucket) Metadata for the given bucket. bucket_url: (gslib.storage_url.StorageUrl) StorageUrl of the given bucket. """ bucket_metadata.encryption = apitools_messages.Bucket.EncryptionValue() print('Clearing default encryption key for %s...' % str(bucket_url).rstrip('/')) self.gsutil_api.PatchBucket(bucket_url.bucket_name, bucket_metadata, fields=['encryption'], provider=bucket_url.scheme) def _EncryptionSetKey(self, bucket_metadata, bucket_url, svc_acct_for_project_num): """Sets defaultKmsKeyName on a Cloud Storage bucket. Args: bucket_metadata: (apitools_messages.Bucket) Metadata for the given bucket. bucket_url: (gslib.storage_url.StorageUrl) StorageUrl of the given bucket. svc_acct_for_project_num: (Dict[int, str]) Mapping of project numbers to their corresponding service account. """ bucket_project_number = bucket_metadata.projectNumber try: # newly_authorized will always be False if the project number is in our # cache dict, since we've already called _AuthorizeProject on it. service_account, newly_authorized = ( svc_acct_for_project_num[bucket_project_number], False) except KeyError: service_account, newly_authorized = self._AuthorizeProject( bucket_project_number, self.kms_key) svc_acct_for_project_num[bucket_project_number] = service_account if newly_authorized: text_util.print_to_fd('Authorized service account %s to use key:\n%s' % (service_account, self.kms_key)) bucket_metadata.encryption = apitools_messages.Bucket.EncryptionValue( defaultKmsKeyName=self.kms_key) print('Setting default KMS key for bucket %s...' % str(bucket_url).rstrip('/')) self.gsutil_api.PatchBucket(bucket_url.bucket_name, bucket_metadata, fields=['encryption'], provider=bucket_url.scheme) def _Encryption(self): self._GatherSubOptions('encryption') # For each project, we should only make one API call to look up its # associated Cloud Storage-owned service account; subsequent lookups can be # pulled from this cache dict. svc_acct_for_project_num = {} def _EncryptionForBucket(blr): """Set, clear, or get the defaultKmsKeyName for a bucket.""" bucket_url = blr.storage_url if bucket_url.scheme != 'gs': raise CommandException( 'The %s command can only be used with gs:// bucket URLs.' % self.command_name) # Determine the project from the provided bucket. bucket_metadata = self.gsutil_api.GetBucket( bucket_url.bucket_name, fields=['encryption', 'projectNumber'], provider=bucket_url.scheme) # "-d" flag was specified, so clear the default KMS key and return. if self.clear_kms_key: self._EncryptionClearKey(bucket_metadata, bucket_url) return 0 # "-k" flag was specified, so set the default KMS key and return. if self.kms_key: self._EncryptionSetKey(bucket_metadata, bucket_url, svc_acct_for_project_num) return 0 # Neither "-d" nor "-k" was specified, so emit the default KMS key and # return. bucket_url_string = str(bucket_url).rstrip('/') if (bucket_metadata.encryption and bucket_metadata.encryption.defaultKmsKeyName): print('Default encryption key for %s:\n%s' % (bucket_url_string, bucket_metadata.encryption.defaultKmsKeyName)) else: print('Bucket %s has no default encryption key' % bucket_url_string) return 0 # Iterate over bucket args, performing the specified encryption operation # for each. some_matched = False url_args = self.args if not url_args: self.RaiseWrongNumberOfArgumentsException() for url_str in url_args: # Throws a CommandException if the argument is not a bucket. bucket_iter = self.GetBucketUrlIterFromArg(url_str) for bucket_listing_ref in bucket_iter: some_matched = True _EncryptionForBucket(bucket_listing_ref) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) return 0 def _ServiceAccount(self): self.CheckArguments() if not self.args: self.args = ['gs://'] if self.sub_opts: for o, a in self.sub_opts: if o == '-p': self.project_id = a if not self.project_id: self.project_id = PopulateProjectId(None) # Request the service account for that project; this might create the # service account if it doesn't already exist. self.logger.debug('Checking service account for project %s', self.project_id) service_account = self.gsutil_api.GetProjectServiceAccount( self.project_id, provider='gs').email_address print(service_account) return 0 def _RunSubCommand(self, func): try: self.sub_opts, self.args = getopt.getopt( self.args, self.command_spec.supported_sub_args) # Commands with both suboptions and subcommands need to reparse for # suboptions, so we log again. metrics.LogCommandParams(sub_opts=self.sub_opts) return func(self) except getopt.GetoptError: self.RaiseInvalidArgumentException() def RunCommand(self): """Command entry point for the kms command.""" # If the only credential type the user supplies in their boto file is hmac, # GetApiSelector logic will force us to use the XML API. As the XML API does # not support all the operations needed for kms subcommands, fail early. if self.gsutil_api.GetApiSelector(provider='gs') != ApiSelector.JSON: raise CommandException('\n'.join( textwrap.wrap( 'The "%s" command can only be used with the GCS JSON API. If you ' 'have only supplied hmac credentials in your boto file, please ' 'instead supply a credential type that can be used with the JSON ' 'API.' % self.command_name))) def RunCommand(self): """Command entry point for the kms command.""" # If the only credential type the user supplies in their boto file is hmac, # GetApiSelector logic will force us to use the XML API. As the XML API does # not support all the operations needed for kms subcommands, fail early. if self.gsutil_api.GetApiSelector(provider='gs') != ApiSelector.JSON: raise CommandException('\n'.join( textwrap.wrap( 'The "%s" command can only be used with the GCS JSON API, which ' 'cannot use HMAC credentials. Please supply a credential ' 'type that is compatible with the JSON API (e.g. OAuth2) in your ' 'boto config file.' % self.command_name))) method_for_subcommand = { 'authorize': KmsCommand._Authorize, 'encryption': KmsCommand._Encryption, 'serviceaccount': KmsCommand._ServiceAccount } self.subcommand_name = self.args.pop(0) if self.subcommand_name in method_for_subcommand: metrics.LogCommandParams(subcommands=[self.subcommand_name]) return self._RunSubCommand(method_for_subcommand[self.subcommand_name]) else: raise CommandException('Invalid subcommand "%s" for the %s command.' % (self.subcommand_name, self.command_name))
class NotificationCommand(Command): """Implementation of gsutil notification command.""" # Notification names might look like one of these: # canonical form: projects/_/buckets/bucket/notificationConfigs/3 # JSON API form: b/bucket/notificationConfigs/5 # Either of the above might start with a / if a user is copying & pasting. def _GetNotificationPathRegex(self): if not NotificationCommand._notification_path_regex: NotificationCommand._notification_path_regex = re.compile( ('/?(projects/[^/]+/)?b(uckets)?/(?P<bucket>[^/]+)/' 'notificationConfigs/(?P<notification>[0-9]+)')) return NotificationCommand._notification_path_regex _notification_path_regex = None # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'notification', command_name_aliases=[ 'notify', 'notifyconfig', 'notifications', 'notif', ], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='i:t:m:t:of:e:p:s', file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'watchbucket': [ CommandArgument.MakeFreeTextArgument(), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(), ], 'stopchannel': [], 'list': [ CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(), ], 'delete': [ # Takes a list of one of the following: # notification: projects/_/buckets/bla/notificationConfigs/5, # bucket: gs://foobar CommandArgument.MakeZeroOrMoreCloudURLsArgument(), ], 'create': [ CommandArgument.MakeFreeTextArgument(), # Cloud Pub/Sub topic CommandArgument.MakeNCloudBucketURLsArgument(1), ] }, ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='notification', help_name_aliases=[ 'watchbucket', 'stopchannel', 'notifyconfig', ], help_type='command_help', help_one_line_summary='Configure object change notification', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'create': _create_help_text, 'list': _list_help_text, 'delete': _delete_help_text, 'watchbucket': _watchbucket_help_text, 'stopchannel': _stopchannel_help_text, }, ) def _WatchBucket(self): """Creates a watch on a bucket given in self.args.""" self.CheckArguments() identifier = None client_token = None if self.sub_opts: for o, a in self.sub_opts: if o == '-i': identifier = a if o == '-t': client_token = a identifier = identifier or str(uuid.uuid4()) watch_url = self.args[0] bucket_arg = self.args[-1] if not watch_url.lower().startswith('https://'): raise CommandException( 'The application URL must be an https:// URL.') bucket_url = StorageUrlFromString(bucket_arg) if not (bucket_url.IsBucket() and bucket_url.scheme == 'gs'): raise CommandException( 'The %s command can only be used with gs:// bucket URLs.' % self.command_name) if not bucket_url.IsBucket(): raise CommandException( 'URL must name a bucket for the %s command.' % self.command_name) self.logger.info('Watching bucket %s with application URL %s ...', bucket_url, watch_url) try: channel = self.gsutil_api.WatchBucket(bucket_url.bucket_name, watch_url, identifier, token=client_token, provider=bucket_url.scheme) except AccessDeniedException as e: self.logger.warn( NOTIFICATION_AUTHORIZATION_FAILED_MESSAGE.format( watch_error=str(e), watch_url=watch_url)) raise channel_id = channel.id resource_id = channel.resourceId client_token = channel.token self.logger.info('Successfully created watch notification channel.') self.logger.info('Watch channel identifier: %s', channel_id) self.logger.info('Canonicalized resource identifier: %s', resource_id) self.logger.info('Client state token: %s', client_token) return 0 def _StopChannel(self): channel_id = self.args[0] resource_id = self.args[1] self.logger.info('Removing channel %s with resource identifier %s ...', channel_id, resource_id) self.gsutil_api.StopChannel(channel_id, resource_id, provider='gs') self.logger.info('Succesfully removed channel.') return 0 def _ListChannels(self, bucket_arg): """Lists active channel watches on a bucket given in self.args.""" bucket_url = StorageUrlFromString(bucket_arg) if not (bucket_url.IsBucket() and bucket_url.scheme == 'gs'): raise CommandException( 'The %s command can only be used with gs:// bucket URLs.' % self.command_name) if not bucket_url.IsBucket(): raise CommandException( 'URL must name a bucket for the %s command.' % self.command_name) channels = self.gsutil_api.ListChannels(bucket_url.bucket_name, provider='gs').items self.logger.info( 'Bucket %s has the following active Object Change Notifications:', bucket_url.bucket_name) for idx, channel in enumerate(channels): self.logger.info('\tNotification channel %d:', idx + 1) self.logger.info('\t\tChannel identifier: %s', channel.channel_id) self.logger.info('\t\tResource identifier: %s', channel.resource_id) self.logger.info('\t\tApplication URL: %s', channel.push_url) self.logger.info('\t\tCreated by: %s', channel.subscriber_email) self.logger.info( '\t\tCreation time: %s', str(datetime.fromtimestamp(channel.creation_time_ms / 1000))) return 0 def _Create(self): self.CheckArguments() # User-specified options pubsub_topic = None payload_format = None custom_attributes = {} event_types = [] object_name_prefix = None should_setup_topic = True if self.sub_opts: for o, a in self.sub_opts: if o == '-e': event_types.append(a) elif o == '-f': payload_format = a elif o == '-m': if ':' not in a: raise CommandException( 'Custom attributes specified with -m should be of the form ' 'key:value') key, value = a.split(':') custom_attributes[key] = value elif o == '-p': object_name_prefix = a elif o == '-s': should_setup_topic = False elif o == '-t': pubsub_topic = a if payload_format not in PAYLOAD_FORMAT_MAP: raise CommandException( "Must provide a payload format with -f of either 'json' or 'none'" ) payload_format = PAYLOAD_FORMAT_MAP[payload_format] bucket_arg = self.args[-1] bucket_url = StorageUrlFromString(bucket_arg) if not bucket_url.IsCloudUrl() or not bucket_url.IsBucket(): raise CommandException( "%s %s requires a GCS bucket name, but got '%s'" % (self.command_name, self.subcommand_name, bucket_arg)) if bucket_url.scheme != 'gs': raise CommandException( 'The %s command can only be used with gs:// bucket URLs.' % self.command_name) bucket_name = bucket_url.bucket_name self.logger.debug('Creating notification for bucket %s', bucket_url) # Find the project this bucket belongs to bucket_metadata = self.gsutil_api.GetBucket(bucket_name, fields=['projectNumber'], provider=bucket_url.scheme) bucket_project_number = bucket_metadata.projectNumber # If not specified, choose a sensible default for the Cloud Pub/Sub topic # name. if not pubsub_topic: pubsub_topic = 'projects/%s/topics/%s' % (PopulateProjectId(None), bucket_name) if not pubsub_topic.startswith('projects/'): # If a user picks a topic ID (mytopic) but doesn't pass the whole name ( # projects/my-project/topics/mytopic ), pick a default project. pubsub_topic = 'projects/%s/topics/%s' % (PopulateProjectId(None), pubsub_topic) self.logger.debug('Using Cloud Pub/Sub topic %s', pubsub_topic) just_modified_topic_permissions = False if should_setup_topic: # Ask GCS for the email address that represents GCS's permission to # publish to a Cloud Pub/Sub topic from this project. service_account = self.gsutil_api.GetProjectServiceAccount( bucket_project_number, provider=bucket_url.scheme).email_address self.logger.debug('Service account for project %d: %s', bucket_project_number, service_account) just_modified_topic_permissions = self._CreateTopic( pubsub_topic, service_account) for attempt_number in range(0, 2): try: create_response = self.gsutil_api.CreateNotificationConfig( bucket_name, pubsub_topic=pubsub_topic, payload_format=payload_format, custom_attributes=custom_attributes, event_types=event_types if event_types else None, object_name_prefix=object_name_prefix, provider=bucket_url.scheme) break except PublishPermissionDeniedException: if attempt_number == 0 and just_modified_topic_permissions: # If we have just set the IAM policy, it may take up to 10 seconds to # take effect. self.logger.info( 'Retrying create notification in 10 seconds ' '(new permissions may take up to 10 seconds to take effect.)' ) time.sleep(10) else: raise notification_name = 'projects/_/buckets/%s/notificationConfigs/%s' % ( bucket_name, create_response.id) self.logger.info('Created notification config %s', notification_name) return 0 def _CreateTopic(self, pubsub_topic, service_account): """Assures that a topic exists, creating it if necessary. Also adds GCS as a publisher on that bucket, if necessary. Args: pubsub_topic: name of the Cloud Pub/Sub topic to use/create. service_account: the GCS service account that needs publish permission. Returns: true if we modified IAM permissions, otherwise false. """ pubsub_api = PubsubApi(logger=self.logger) # Verify that the Pub/Sub topic exists. If it does not, create it. try: pubsub_api.GetTopic(topic_name=pubsub_topic) self.logger.debug('Topic %s already exists', pubsub_topic) except NotFoundException: self.logger.debug('Creating topic %s', pubsub_topic) pubsub_api.CreateTopic(topic_name=pubsub_topic) self.logger.info('Created Cloud Pub/Sub topic %s', pubsub_topic) # Verify that the service account is in the IAM policy. policy = pubsub_api.GetTopicIamPolicy(topic_name=pubsub_topic) binding = Binding(role='roles/pubsub.publisher', members=['serviceAccount:%s' % service_account]) # This could be more extensive. We could, for instance, check for roles # that are stronger that pubsub.publisher, like owner. We could also # recurse up the hierarchy looking to see if there are project-level # permissions. This can get very complex very quickly, as the caller # may not necessarily have access to the project-level IAM policy. # There's no danger in double-granting permission just to make sure it's # there, though. if binding not in policy.bindings: policy.bindings.append(binding) # transactional safety via etag field. pubsub_api.SetTopicIamPolicy(topic_name=pubsub_topic, policy=policy) return True else: self.logger.debug( 'GCS already has publish permission to topic %s.', pubsub_topic) return False def _EnumerateNotificationsFromArgs(self, accept_notification_configs=True): """Yields bucket/notification tuples from command-line args. Given a list of strings that are bucket names (gs://foo) or notification config IDs, yield tuples of bucket names and their associated notifications. Args: accept_notification_configs: whether notification configs are valid args. Yields: Tuples of the form (bucket_name, Notification) """ path_regex = self._GetNotificationPathRegex() for list_entry in self.args: match = path_regex.match(list_entry) if match: if not accept_notification_configs: raise CommandException( '%s %s accepts only bucket names, but you provided %s' % (self.command_name, self.subcommand_name, list_entry)) bucket_name = match.group('bucket') notification_id = match.group('notification') found = False for notification in self.gsutil_api.ListNotificationConfigs( bucket_name, provider='gs'): if notification.id == notification_id: yield (bucket_name, notification) found = True break if not found: raise NotFoundException('Could not find notification %s' % list_entry) else: storage_url = StorageUrlFromString(list_entry) if not storage_url.IsCloudUrl(): raise CommandException( 'The %s command must be used on cloud buckets or notification ' 'config names.' % self.command_name) if storage_url.scheme != 'gs': raise CommandException( 'The %s command only works on gs:// buckets.') path = None if storage_url.IsProvider(): path = 'gs://*' elif storage_url.IsBucket(): path = list_entry if not path: raise CommandException( 'The %s command cannot be used on cloud objects, only buckets' % self.command_name) for blr in self.WildcardIterator(path).IterBuckets( bucket_fields=['id']): for notification in self.gsutil_api.ListNotificationConfigs( blr.storage_url.bucket_name, provider='gs'): yield (blr.storage_url.bucket_name, notification) def _List(self): self.CheckArguments() if self.sub_opts: if '-o' in dict(self.sub_opts): for bucket_name in self.args: self._ListChannels(bucket_name) else: for bucket_name, notification in self._EnumerateNotificationsFromArgs( accept_notification_configs=False): self._PrintNotificationDetails(bucket_name, notification) return 0 def _PrintNotificationDetails(self, bucket, notification): print( 'projects/_/buckets/{bucket}/notificationConfigs/{notification}\n' '\tCloud Pub/Sub topic: {topic}'.format( bucket=bucket, notification=notification.id, topic=notification.topic[len('//pubsub.googleapis.com/'):])) if notification.custom_attributes: print('\tCustom attributes:') for attr in notification.custom_attributes.additionalProperties: print('\t\t%s: %s' % (attr.key, attr.value)) filters = [] if notification.event_types: filters.append('\t\tEvent Types: %s' % ', '.join(notification.event_types)) if notification.object_name_prefix: filters.append("\t\tObject name prefix: '%s'" % notification.object_name_prefix) if filters: print('\tFilters:') for line in filters: print(line) self.logger.info('') def _Delete(self): for bucket_name, notification in self._EnumerateNotificationsFromArgs( ): self._DeleteNotification(bucket_name, notification.id) return 0 def _DeleteNotification(self, bucket_name, notification_id): self.gsutil_api.DeleteNotificationConfig(bucket_name, notification=notification_id, provider='gs') return 0 def _RunSubCommand(self, func): try: (self.sub_opts, self.args) = getopt.getopt(self.args, self.command_spec.supported_sub_args) # Commands with both suboptions and subcommands need to reparse for # suboptions, so we log again. metrics.LogCommandParams(sub_opts=self.sub_opts) return func(self) except getopt.GetoptError: self.RaiseInvalidArgumentException() SUBCOMMANDS = { 'create': _Create, 'list': _List, 'delete': _Delete, 'watchbucket': _WatchBucket, 'stopchannel': _StopChannel } def RunCommand(self): """Command entry point for the notification command.""" self.subcommand_name = self.args.pop(0) if self.subcommand_name in NotificationCommand.SUBCOMMANDS: metrics.LogCommandParams(subcommands=[self.subcommand_name]) return self._RunSubCommand( NotificationCommand.SUBCOMMANDS[self.subcommand_name]) else: raise CommandException( 'Invalid subcommand "%s" for the %s command.' % (self.subcommand_name, self.command_name))
class WebCommand(Command): """Implementation of gsutil web command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'web', command_name_aliases=['setwebcfg', 'getwebcfg'], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='m:e:', file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'set': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()], 'get': [CommandArgument.MakeNCloudBucketURLsArgument(1)] }) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='web', help_name_aliases=['getwebcfg', 'setwebcfg'], help_type='command_help', help_one_line_summary=( 'Set a main page and/or error page for one or more buckets'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text }, ) def _GetWeb(self): """Gets website configuration for a bucket.""" bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg( self.args[0], bucket_fields=['website']) if bucket_url.scheme == 's3': sys.stdout.write( self.gsutil_api.XmlPassThroughGetWebsite( bucket_url, provider=bucket_url.scheme)) else: if bucket_metadata.website and ( bucket_metadata.website.mainPageSuffix or bucket_metadata.website.notFoundPage): sys.stdout.write( str(encoding.MessageToJson(bucket_metadata.website)) + '\n') else: sys.stdout.write('%s has no website configuration.\n' % bucket_url) return 0 def _SetWeb(self): """Sets website configuration for a bucket.""" main_page_suffix = None error_page = None if self.sub_opts: for o, a in self.sub_opts: if o == '-m': main_page_suffix = a elif o == '-e': error_page = a url_args = self.args website = apitools_messages.Bucket.WebsiteValue( mainPageSuffix=main_page_suffix, notFoundPage=error_page) # Iterate over URLs, expanding wildcards and setting the website # configuration on each. some_matched = False for url_str in url_args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id']) for blr in bucket_iter: url = blr.storage_url some_matched = True self.logger.info('Setting website configuration on %s...', blr) bucket_metadata = apitools_messages.Bucket(website=website) self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, provider=url.scheme, fields=['id']) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) return 0 def RunCommand(self): """Command entry point for the web command.""" action_subcommand = self.args.pop(0) self.ParseSubOpts(check_args=True) if action_subcommand == 'get': func = self._GetWeb elif action_subcommand == 'set': func = self._SetWeb else: raise CommandException( ('Invalid subcommand "%s" for the %s command.\n' 'See "gsutil help web".') % (action_subcommand, self.command_name)) # Commands with both suboptions and subcommands need to reparse for # suboptions, so we log again. metrics.LogCommandParams(subcommands=[action_subcommand], sub_opts=self.sub_opts) return func()
class CorsCommand(Command): """Implementation of gsutil cors command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'cors', command_name_aliases=['getcors', 'setcors'], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='', file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'set': [ CommandArgument.MakeNFileURLsArgument(1), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument() ], 'get': [CommandArgument.MakeNCloudBucketURLsArgument(1)] }) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='cors', help_name_aliases=['getcors', 'setcors', 'cross-origin'], help_type='command_help', help_one_line_summary=( 'Set a CORS JSON document for one or more buckets'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text }, ) def _CalculateUrlsStartArg(self): if not self.args: self.RaiseWrongNumberOfArgumentsException() if self.args[0].lower() == 'set': return 2 else: return 1 def _SetCors(self): """Sets CORS configuration on a Google Cloud Storage bucket.""" cors_arg = self.args[0] url_args = self.args[1:] # Disallow multi-provider 'cors set' requests. if not UrlsAreForSingleProvider(url_args): raise CommandException( '"%s" command spanning providers not allowed.' % self.command_name) # Open, read and parse file containing JSON document. cors_file = open(cors_arg, 'r') cors_txt = cors_file.read() cors_file.close() self.api = self.gsutil_api.GetApiSelector( StorageUrlFromString(url_args[0]).scheme) # Iterate over URLs, expanding wildcards and setting the CORS on each. some_matched = False for url_str in url_args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id']) for blr in bucket_iter: url = blr.storage_url some_matched = True self.logger.info('Setting CORS on %s...', blr) if url.scheme == 's3': self.gsutil_api.XmlPassThroughSetCors(cors_txt, url, provider=url.scheme) else: cors = CorsTranslation.JsonCorsToMessageEntries(cors_txt) if not cors: cors = REMOVE_CORS_CONFIG bucket_metadata = apitools_messages.Bucket(cors=cors) self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, provider=url.scheme, fields=['id']) if not some_matched: raise CommandException('No URLs matched') return 0 def _GetCors(self): """Gets CORS configuration for a Google Cloud Storage bucket.""" bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg( self.args[0], bucket_fields=['cors']) if bucket_url.scheme == 's3': sys.stdout.write( self.gsutil_api.XmlPassThroughGetCors( bucket_url, provider=bucket_url.scheme)) else: if bucket_metadata.cors: sys.stdout.write( CorsTranslation.MessageEntriesToJson(bucket_metadata.cors)) else: sys.stdout.write('%s has no CORS configuration.\n' % bucket_url) return 0 def RunCommand(self): """Command entry point for the cors command.""" action_subcommand = self.args.pop(0) if action_subcommand == 'get': func = self._GetCors elif action_subcommand == 'set': func = self._SetCors else: raise CommandException( ('Invalid subcommand "%s" for the %s command.\n' 'See "gsutil help cors".') % (action_subcommand, self.command_name)) return func()
class NotificationCommand(Command): """Implementation of gsutil notification command.""" # Notification names might look like one of these: # canonical form: projects/_/buckets/bucket/notificationConfigs/3 # JSON API form: b/bucket/notificationConfigs/5 # Either of the above might start with a / if a user is copying & pasting. def _GetNotificationPathRegex(self): if not NotificationCommand._notification_path_regex: NotificationCommand._notification_path_regex = re.compile( ('/?(projects/[^/]+/)?b(uckets)?/(?P<bucket>[^/]+)/' 'notificationConfigs/(?P<notification>[0-9]+)')) return NotificationCommand._notification_path_regex _notification_path_regex = None # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'notification', command_name_aliases=[ 'notify', 'notifyconfig', 'notifications', 'notif' ], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='i:t:m:t:o:f:e:p:s', file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'watchbucket': [ CommandArgument.MakeFreeTextArgument(), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument() ], 'stopchannel': [], 'list': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()], 'delete': [ # Takes a list of one of the following: # notification: projects/_/buckets/bla/notificationConfigs/5, # bucket: gs://foobar CommandArgument.MakeZeroOrMoreCloudURLsArgument() ], 'create': [ CommandArgument.MakeFreeTextArgument(), # Cloud Pub/Sub topic CommandArgument.MakeNCloudBucketURLsArgument(1) ] }) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='notification', help_name_aliases=['watchbucket', 'stopchannel', 'notifyconfig'], help_type='command_help', help_one_line_summary='Configure object change notification', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'create': _create_help_text, 'list': _list_help_text, 'delete': _delete_help_text, 'watchbucket': _watchbucket_help_text, 'stopchannel': _stopchannel_help_text }, ) def _WatchBucket(self): """Creates a watch on a bucket given in self.args.""" self.CheckArguments() identifier = None client_token = None if self.sub_opts: for o, a in self.sub_opts: if o == '-i': identifier = a if o == '-t': client_token = a identifier = identifier or str(uuid.uuid4()) watch_url = self.args[0] bucket_arg = self.args[-1] if not watch_url.lower().startswith('https://'): raise CommandException( 'The application URL must be an https:// URL.') bucket_url = StorageUrlFromString(bucket_arg) if not (bucket_url.IsBucket() and bucket_url.scheme == 'gs'): raise CommandException( 'The %s command can only be used with gs:// bucket URLs.' % self.command_name) if not bucket_url.IsBucket(): raise CommandException( 'URL must name a bucket for the %s command.' % self.command_name) self.logger.info('Watching bucket %s with application URL %s ...', bucket_url, watch_url) try: channel = self.gsutil_api.WatchBucket(bucket_url.bucket_name, watch_url, identifier, token=client_token, provider=bucket_url.scheme) except AccessDeniedException, e: self.logger.warn( NOTIFICATION_AUTHORIZATION_FAILED_MESSAGE.format( watch_error=str(e), watch_url=watch_url)) raise channel_id = channel.id resource_id = channel.resourceId client_token = channel.token self.logger.info('Successfully created watch notification channel.') self.logger.info('Watch channel identifier: %s', channel_id) self.logger.info('Canonicalized resource identifier: %s', resource_id) self.logger.info('Client state token: %s', client_token) return 0
class RetentionCommand(Command): """Implementation of gsutil retention command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'retention', command_name_aliases=[], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'set': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()], 'clear': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()], 'get': [CommandArgument.MakeNCloudBucketURLsArgument(1)], 'lock': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()], 'event-default': { 'set': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()], 'release': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()] }, 'event': { 'set': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()], 'release': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()] }, 'temp': { 'set': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()], 'release': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()] }, }) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='retention', help_name_aliases=[], help_type='command_help', help_one_line_summary=( 'Provides utilities to interact with Retention Policy feature.'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, 'clear': _clear_help_text, 'lock': _lock_help_text, 'event-default': _event_default_help_text, 'event': _event_help_text, 'temp': _temp_help_text }, ) def RunCommand(self): """Command entry point for the retention command.""" # If the only credential type the user supplies in their boto file is HMAC, # GetApiSelector logic will force us to use the XML API, which bucket lock # does not support at the moment. if self.gsutil_api.GetApiSelector('gs') != ApiSelector.JSON: raise CommandException(('The {} command can only be used with the GCS ' 'JSON API. If you have only supplied hmac ' 'credentials in your boto file, please instead ' 'supply a credential type that can be used with ' 'the JSON API.').format(self.command_name)) self.preconditions = PreconditionsFromHeaders(self.headers) action_subcommand = self.args.pop(0) self.ParseSubOpts(check_args=True) if action_subcommand == 'set': func = self._SetRetention elif action_subcommand == 'clear': func = self._ClearRetention elif action_subcommand == 'get': func = self._GetRetention elif action_subcommand == 'lock': func = self._LockRetention elif action_subcommand == 'event-default': func = self._DefaultEventHold elif action_subcommand == 'event': func = self._EventHold elif action_subcommand == 'temp': func = self._TempHold else: raise CommandException( ('Invalid subcommand "{}" for the {} command.\n' 'See "gsutil help retention".').format(action_subcommand, self.command_name)) # Commands with both suboptions and subcommands need to reparse for # suboptions, so we log again. metrics.LogCommandParams(subcommands=[action_subcommand], sub_opts=self.sub_opts) return func() def BucketUpdateFunc(self, url_args, bucket_metadata_update, fields, log_msg_template): preconditions = Preconditions( meta_gen_match=self.preconditions.meta_gen_match) # Iterate over URLs, expanding wildcards and setting the new bucket metadata # on each bucket. some_matched = False for url_str in url_args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id']) for blr in bucket_iter: url = blr.storage_url some_matched = True self.logger.info(log_msg_template, blr) self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata_update, preconditions=preconditions, provider=url.scheme, fields=fields) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) def ObjectUpdateMetadataFunc(self, patch_obj_metadata, log_template, name_expansion_result, thread_state=None): """Updates metadata on an object using PatchObjectMetadata. Args: patch_obj_metadata: Metadata changes that should be applied to the existing object. log_template: The log template that should be printed for each object. name_expansion_result: NameExpansionResult describing target object. thread_state: gsutil Cloud API instance to use for the operation. """ gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) exp_src_url = name_expansion_result.expanded_storage_url self.logger.info(log_template, exp_src_url) cloud_obj_metadata = encoding.JsonToMessage( apitools_messages.Object, name_expansion_result.expanded_result) preconditions = Preconditions( gen_match=self.preconditions.gen_match, meta_gen_match=self.preconditions.meta_gen_match) if preconditions.gen_match is None: preconditions.gen_match = cloud_obj_metadata.generation if preconditions.meta_gen_match is None: preconditions.meta_gen_match = cloud_obj_metadata.metageneration gsutil_api.PatchObjectMetadata(exp_src_url.bucket_name, exp_src_url.object_name, patch_obj_metadata, generation=exp_src_url.generation, preconditions=preconditions, provider=exp_src_url.scheme, fields=['id']) PutToQueueWithTimeout(gsutil_api.status_queue, MetadataMessage(message_time=time.time())) def _GetObjectNameExpansionIterator(self, url_args): return NameExpansionIterator( self.command_name, self.debug, self.logger, self.gsutil_api, url_args, self.recursion_requested, all_versions=self.all_versions, continue_on_error=self.parallel_operations, bucket_listing_fields=['generation', 'metageneration']) def _GetSeekAheadNameExpansionIterator(self, url_args): return SeekAheadNameExpansionIterator(self.command_name, self.debug, self.GetSeekAheadGsutilApi(), url_args, self.recursion_requested, all_versions=self.all_versions, project_id=self.project_id) def _SetRetention(self): """Set retention retention_period on one or more buckets.""" seconds = RetentionInSeconds(self.args[0]) retention_policy = (apitools_messages.Bucket.RetentionPolicyValue( retentionPeriod=seconds)) log_msg_template = 'Setting Retention Policy on %s...' bucket_metadata_update = apitools_messages.Bucket( retentionPolicy=retention_policy) url_args = self.args[1:] self.BucketUpdateFunc(url_args, bucket_metadata_update, fields=['id', 'retentionPolicy'], log_msg_template=log_msg_template) return 0 def _ClearRetention(self): """Clear retention retention_period on one or more buckets.""" retention_policy = (apitools_messages.Bucket.RetentionPolicyValue( retentionPeriod=None)) log_msg_template = 'Clearing Retention Policy on %s...' bucket_metadata_update = apitools_messages.Bucket( retentionPolicy=retention_policy) url_args = self.args self.BucketUpdateFunc(url_args, bucket_metadata_update, fields=['id', 'retentionPolicy'], log_msg_template=log_msg_template) return 0 def _GetRetention(self): """Get Retention Policy for a single bucket.""" bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg( self.args[0], bucket_fields=['retentionPolicy']) print(RetentionPolicyToString(bucket_metadata.retentionPolicy, bucket_url)) return 0 def _LockRetention(self): """Lock Retention Policy on one or more buckets.""" url_args = self.args # Iterate over URLs, expanding wildcards and setting the Retention Policy # configuration on each. some_matched = False for url_str in url_args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id']) for blr in bucket_iter: url = blr.storage_url some_matched = True # Get bucket metadata to provide a precondition. bucket_metadata = self.gsutil_api.GetBucket( url.bucket_name, provider=url.scheme, fields=['id', 'metageneration', 'retentionPolicy']) if (not (bucket_metadata.retentionPolicy and bucket_metadata.retentionPolicy.retentionPeriod)): # TODO: implement '-c' flag to continue_on_error raise CommandException( 'Bucket "{}" does not have an Unlocked Retention Policy.'.format( url.bucket_name)) elif bucket_metadata.retentionPolicy.isLocked is True: self.logger.error('Retention Policy on "%s" is already locked.', blr) elif ConfirmLockRequest(url.bucket_name, bucket_metadata.retentionPolicy): self.logger.info('Locking Retention Policy on %s...', blr) self.gsutil_api.LockRetentionPolicy(url.bucket_name, bucket_metadata.metageneration, provider=url.scheme) else: self.logger.error( ' Abort Locking Retention Policy on {}'.format(blr)) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) return 0 def _DefaultEventHold(self): """Sets default value for Event-Based Hold on one or more buckets.""" hold = None if self.args: if self.args[0].lower() == 'set': hold = True elif self.args[0].lower() == 'release': hold = False else: raise CommandException( ('Invalid subcommand "{}" for the "retention event-default"' ' command.\nSee "gsutil help retention event".').format( self.sub_opts)) verb = 'Setting' if hold else 'Releasing' log_msg_template = '{} default Event-Based Hold on %s...'.format(verb) bucket_metadata_update = apitools_messages.Bucket( defaultEventBasedHold=hold) url_args = self.args[1:] self.BucketUpdateFunc(url_args, bucket_metadata_update, fields=['id', 'defaultEventBasedHold'], log_msg_template=log_msg_template) return 0 def _EventHold(self): """Sets or unsets Event-Based Hold on one or more objects.""" sub_command_name = 'event' sub_command_full_name = 'Event-Based' hold = self._ProcessHoldArgs(sub_command_name) url_args = self.args[1:] obj_metadata_update_wrapper = (SetEventHoldFuncWrapper if hold else ReleaseEventHoldFuncWrapper) self._SetHold(obj_metadata_update_wrapper, url_args, sub_command_full_name) return 0 def _TempHold(self): """Sets or unsets Temporary Hold on one or more objects.""" sub_command_name = 'temp' sub_command_full_name = 'Temporary' hold = self._ProcessHoldArgs(sub_command_name) url_args = self.args[1:] obj_metadata_update_wrapper = (SetTempHoldFuncWrapper if hold else ReleaseTempHoldFuncWrapper) self._SetHold(obj_metadata_update_wrapper, url_args, sub_command_full_name) return 0 def _ProcessHoldArgs(self, sub_command_name): """Processes command args for Temporary and Event-Based Hold sub-command. Args: sub_command_name: The name of the subcommand: "temp" / "event" Returns: Returns a boolean value indicating whether to set (True) or release (False)the Hold. """ hold = None if self.args[0].lower() == 'set': hold = True elif self.args[0].lower() == 'release': hold = False else: raise CommandException( ('Invalid subcommand "{}" for the "retention {}" command.\n' 'See "gsutil help retention {}".').format(self.args[0], sub_command_name, sub_command_name)) return hold def _SetHold(self, obj_metadata_update_wrapper, url_args, sub_command_full_name): """Common logic to set or unset Event-Based/Temporary Hold on objects. Args: obj_metadata_update_wrapper: The function for updating related fields in Object metadata. url_args: List of object URIs. sub_command_full_name: The full name for sub-command: "Temporary" / "Event-Based" """ if len(url_args) == 1 and not self.recursion_requested: url = StorageUrlFromString(url_args[0]) if not (url.IsCloudUrl() and url.IsObject()): raise CommandException('URL ({}) must name an object'.format( url_args[0])) name_expansion_iterator = self._GetObjectNameExpansionIterator(url_args) seek_ahead_iterator = self._GetSeekAheadNameExpansionIterator(url_args) # Used to track if any objects' metadata failed to be set. self.everything_set_okay = True try: # TODO: implement '-c' flag to continue_on_error # Perform requests in parallel (-m) mode, if requested, using # configured number of parallel processes and threads. Otherwise, # perform requests with sequential function calls in current process. self.Apply(obj_metadata_update_wrapper, name_expansion_iterator, UpdateObjectMetadataExceptionHandler, fail_on_error=True, seek_ahead_iterator=seek_ahead_iterator) except AccessDeniedException as e: if e.status == 403: self._WarnServiceAccounts() raise if not self.everything_set_okay: raise CommandException( '{} Hold for some objects could not be set.'.format( sub_command_full_name))
class LifecycleCommand(Command): """Implementation of gsutil lifecycle command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'lifecycle', command_name_aliases=['lifecycleconfig'], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='', file_url_ok=True, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'set': [ CommandArgument.MakeNFileURLsArgument(1), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument() ], 'get': [ CommandArgument.MakeNCloudBucketURLsArgument(1) ] } ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='lifecycle', help_name_aliases=['getlifecycle', 'setlifecycle'], help_type='command_help', help_one_line_summary=( 'Get or set lifecycle configuration for a bucket'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={'get': _get_help_text, 'set': _set_help_text}, ) def _SetLifecycleConfig(self): """Sets lifecycle configuration for a Google Cloud Storage bucket.""" lifecycle_arg = self.args[0] url_args = self.args[1:] # Disallow multi-provider 'lifecycle set' requests. if not UrlsAreForSingleProvider(url_args): raise CommandException('"%s" command spanning providers not allowed.' % self.command_name) # Open, read and parse file containing JSON document. lifecycle_file = open(lifecycle_arg, 'r') lifecycle_txt = lifecycle_file.read() lifecycle_file.close() # Iterate over URLs, expanding wildcards and setting the lifecycle on each. some_matched = False for url_str in url_args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['lifecycle']) for blr in bucket_iter: url = blr.storage_url some_matched = True self.logger.info('Setting lifecycle configuration on %s...', blr) if url.scheme == 's3': self.gsutil_api.XmlPassThroughSetLifecycle( lifecycle_txt, url, provider=url.scheme) else: lifecycle = LifecycleTranslation.JsonLifecycleToMessage(lifecycle_txt) bucket_metadata = apitools_messages.Bucket(lifecycle=lifecycle) self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, provider=url.scheme, fields=['id']) if not some_matched: raise CommandException('No URLs matched') return 0 def _GetLifecycleConfig(self): """Gets lifecycle configuration for a Google Cloud Storage bucket.""" bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg( self.args[0], bucket_fields=['lifecycle']) if bucket_url.scheme == 's3': sys.stdout.write(self.gsutil_api.XmlPassThroughGetLifecycle( bucket_url, provider=bucket_url.scheme)) else: if bucket_metadata.lifecycle and bucket_metadata.lifecycle.rule: sys.stdout.write(LifecycleTranslation.JsonLifecycleFromMessage( bucket_metadata.lifecycle)) else: sys.stdout.write('%s has no lifecycle configuration.\n' % bucket_url) return 0 def RunCommand(self): """Command entry point for the lifecycle command.""" subcommand = self.args.pop(0) if subcommand == 'get': return self._GetLifecycleConfig() elif subcommand == 'set': return self._SetLifecycleConfig() else: raise CommandException('Invalid subcommand "%s" for the %s command.' % (subcommand, self.command_name))
class DefAclCommand(Command): """Implementation of gsutil defacl command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'defacl', command_name_aliases=['setdefacl', 'getdefacl', 'chdefacl'], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='fg:u:d:p:', file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'set': [ CommandArgument.MakeFileURLOrCannedACLArgument(), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument() ], 'get': [CommandArgument.MakeNCloudBucketURLsArgument(1)], 'ch': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()], }) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='defacl', help_name_aliases=[ 'default acl', 'setdefacl', 'getdefacl', 'chdefacl' ], help_type='command_help', help_one_line_summary='Get, set, or change default ACL on buckets', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, 'ch': _ch_help_text }, ) def _CalculateUrlsStartArg(self): if not self.args: self.RaiseWrongNumberOfArgumentsException() if (self.args[0].lower() == 'set' or self.command_alias_used == 'setdefacl'): return 1 else: return 0 def _SetDefAcl(self): if not StorageUrlFromString(self.args[-1]).IsBucket(): raise CommandException( 'URL must name a bucket for the %s command' % self.command_name) try: self.SetAclCommandHelper(SetAclFuncWrapper, SetAclExceptionHandler) except AccessDeniedException: self._WarnServiceAccounts() raise def _GetDefAcl(self): if not StorageUrlFromString(self.args[0]).IsBucket(): raise CommandException( 'URL must name a bucket for the %s command' % self.command_name) self.GetAndPrintAcl(self.args[0]) def _ChDefAcl(self): """Parses options and changes default object ACLs on specified buckets.""" self.parse_versions = True self.changes = [] if self.sub_opts: for o, a in self.sub_opts: if o == '-g': self.changes.append( aclhelpers.AclChange( a, scope_type=aclhelpers.ChangeType.GROUP)) if o == '-u': self.changes.append( aclhelpers.AclChange( a, scope_type=aclhelpers.ChangeType.USER)) if o == '-p': self.changes.append( aclhelpers.AclChange( a, scope_type=aclhelpers.ChangeType.PROJECT)) if o == '-d': self.changes.append(aclhelpers.AclDel(a)) if not self.changes: raise CommandException('Please specify at least one access change ' 'with the -g, -u, or -d flags') if (not UrlsAreForSingleProvider(self.args) or StorageUrlFromString(self.args[0]).scheme != 'gs'): raise CommandException( 'The "{0}" command can only be used with gs:// URLs'.format( self.command_name)) bucket_urls = set() for url_arg in self.args: for result in self.WildcardIterator(url_arg): if not result.storage_url.IsBucket(): raise CommandException( 'The defacl ch command can only be applied to buckets.' ) bucket_urls.add(result.storage_url) for storage_url in bucket_urls: self.ApplyAclChanges(storage_url) @Retry(ServiceException, tries=3, timeout_secs=1) def ApplyAclChanges(self, url): """Applies the changes in self.changes to the provided URL.""" bucket = self.gsutil_api.GetBucket( url.bucket_name, provider=url.scheme, fields=['defaultObjectAcl', 'metageneration']) # Default object ACLs can be blank if the ACL was set to private, or # if the user doesn't have permission. We warn about this with defacl get, # so just try the modification here and if the user doesn't have # permission they'll get an AccessDeniedException. current_acl = bucket.defaultObjectAcl modification_count = 0 for change in self.changes: modification_count += change.Execute(url, current_acl, 'defacl', self.logger) if modification_count == 0: self.logger.info('No changes to %s', url) return if not current_acl: # Use a sentinel value to indicate a private (no entries) default # object ACL. current_acl.append(PRIVATE_DEFAULT_OBJ_ACL) try: preconditions = Preconditions(meta_gen_match=bucket.metageneration) bucket_metadata = apitools_messages.Bucket( defaultObjectAcl=current_acl) self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, preconditions=preconditions, provider=url.scheme, fields=['id']) except BadRequestException as e: # Don't retry on bad requests, e.g. invalid email address. raise CommandException('Received bad request from server: %s' % str(e)) except AccessDeniedException: self._WarnServiceAccounts() raise CommandException( 'Failed to set acl for %s. Please ensure you have ' 'OWNER-role access to this resource.' % url) self.logger.info('Updated default ACL on %s', url) def RunCommand(self): """Command entry point for the defacl command.""" action_subcommand = self.args.pop(0) self.ParseSubOpts(check_args=True) self.def_acl = True self.continue_on_error = False if action_subcommand == 'get': func = self._GetDefAcl elif action_subcommand == 'set': func = self._SetDefAcl elif action_subcommand in ('ch', 'change'): func = self._ChDefAcl else: raise CommandException( ('Invalid subcommand "%s" for the %s command.\n' 'See "gsutil help defacl".') % (action_subcommand, self.command_name)) func() return 0