def __init__(self, url_string): self.scheme = None self.bucket_name = None self.object_name = None self.generation = None self.delim = '/' provider_match = PROVIDER_REGEX.match(url_string) bucket_match = BUCKET_REGEX.match(url_string) if provider_match: self.scheme = provider_match.group('provider') elif bucket_match: self.scheme = bucket_match.group('provider') self.bucket_name = bucket_match.group('bucket') else: object_match = OBJECT_REGEX.match(url_string) if object_match: self.scheme = object_match.group('provider') self.bucket_name = object_match.group('bucket') self.object_name = object_match.group('object') if self.scheme == 'gs': generation_match = GS_GENERATION_REGEX.match( self.object_name) if generation_match: self.object_name = generation_match.group('object') self.generation = generation_match.group('generation') elif self.scheme == 's3': version_match = S3_VERSION_REGEX.match(self.object_name) if version_match: self.object_name = version_match.group('object') self.generation = version_match.group('version_id') else: raise InvalidUrlError( 'CloudUrl: URL string %s did not match URL regex' % url_string)
def __init__(self, url_string): self.scheme = None self.delim = '/' self.bucket_name = None self.object_name = None self.generation = None provider_match = PROVIDER_REGEX.match(url_string) bucket_match = BUCKET_REGEX.match(url_string) if provider_match: self.scheme = provider_match.group('provider') elif bucket_match: self.scheme = bucket_match.group('provider') self.bucket_name = bucket_match.group('bucket') else: object_match = OBJECT_REGEX.match(url_string) if object_match: self.scheme = object_match.group('provider') self.bucket_name = object_match.group('bucket') self.object_name = object_match.group('object') if self.object_name == '.' or self.object_name == '..': raise InvalidUrlError( '%s is an invalid root-level object name' % self.object_name) if self.scheme == 'gs': generation_match = GS_GENERATION_REGEX.match( self.object_name) if generation_match: self.object_name = generation_match.group('object') self.generation = generation_match.group('generation') elif self.scheme == 's3': version_match = S3_VERSION_REGEX.match(self.object_name) if version_match: self.object_name = version_match.group('object') self.generation = version_match.group('version_id') else: raise InvalidUrlError( 'CloudUrl: URL string %s did not match URL regex' % url_string) if url_string[(len(self.scheme) + len('://')):].startswith(self.delim): raise InvalidUrlError( 'Cloud URL scheme should be followed by colon and two slashes: "://".' ' Found: "{}"'.format(url_string)) self._WarnIfUnsupportedDoubleWildcard()
def RunCommand(self): """Command entry point for the mb command.""" location = None storage_class = None if self.sub_opts: for o, a in self.sub_opts: if o == '-l': location = a elif o == '-p': # Project IDs are sent as header values when using gs and s3 XML APIs. InsistAscii( a, 'Invalid non-ASCII character found in project ID') self.project_id = a elif o == '-c' or o == '-s': storage_class = NormalizeStorageClass(a) bucket_metadata = apitools_messages.Bucket(location=location, storageClass=storage_class) for bucket_url_str in self.args: bucket_url = StorageUrlFromString(bucket_url_str) if not bucket_url.IsBucket(): raise CommandException( 'The mb command requires a URL that specifies a ' 'bucket.\n"%s" is not valid.' % bucket_url) if (not BUCKET_NAME_RE.match(bucket_url.bucket_name) or TOO_LONG_DNS_NAME_COMP.search(bucket_url.bucket_name)): raise InvalidUrlError('Invalid bucket name in URL "%s"' % bucket_url.bucket_name) self.logger.info('Creating %s...', bucket_url) # Pass storage_class param only if this is a GCS bucket. (In S3 the # storage class is specified on the key object.) try: self.gsutil_api.CreateBucket(bucket_url.bucket_name, project_id=self.project_id, metadata=bucket_metadata, provider=bucket_url.scheme) except BadRequestException as e: if (e.status == 400 and e.reason == 'DotfulBucketNameNotUnderTld' and bucket_url.scheme == 'gs'): bucket_name = bucket_url.bucket_name final_comp = bucket_name[bucket_name.rfind('.') + 1:] raise CommandException('\n'.join( textwrap.wrap( 'Buckets with "." in the name must be valid DNS names. The bucket' ' you are attempting to create (%s) is not a valid DNS name,' ' because the final component (%s) is not currently a valid part' ' of the top-level DNS tree.' % (bucket_name, final_comp)))) else: raise return 0
def StorageUrlFromString(url_str): """Static factory function for creating a StorageUrl from a string.""" scheme = _GetSchemeFromUrlString(url_str) if scheme not in ('file', 's3', 'gs'): raise InvalidUrlError('Unrecognized scheme "%s"' % scheme) if scheme == 'file': path = _GetPathFromUrlString(url_str) is_stream = (path == '-') return _FileUrl(url_str, is_stream=is_stream) return _CloudUrl(url_str)
def __init__(self, url_string): self.scheme = None self.bucket_name = None self.object_name = None self.generation = None self.delim = '/' provider_match = PROVIDER_REGEX.match(url_string) bucket_match = BUCKET_REGEX.match(url_string) if provider_match: self.scheme = provider_match.group('provider') elif bucket_match: self.scheme = bucket_match.group('provider') self.bucket_name = bucket_match.group('bucket') if (not ContainsWildcard(self.bucket_name) and (not BUCKET_NAME_RE.match(self.bucket_name) or TOO_LONG_DNS_NAME_COMP.search(self.bucket_name))): raise InvalidUrlError('Invalid bucket name in URL "%s"' % url_string) else: object_match = OBJECT_REGEX.match(url_string) if object_match: self.scheme = object_match.group('provider') self.bucket_name = object_match.group('bucket') self.object_name = object_match.group('object') if self.scheme == 'gs': generation_match = GS_GENERATION_REGEX.match( self.object_name) if generation_match: self.object_name = generation_match.group('object') self.generation = generation_match.group('generation') elif self.scheme == 's3': version_match = S3_VERSION_REGEX.match(self.object_name) if version_match: self.object_name = version_match.group('object') self.generation = version_match.group('version_id') else: raise InvalidUrlError( 'CloudUrl: URL string %s did not match URL regex' % url_string)
def StorageUrlFromString(url_str): """Static factory function for creating a StorageUrl from a string.""" scheme = GetSchemeFromUrlString(url_str) if not IsKnownUrlScheme(scheme): raise InvalidUrlError('Unrecognized scheme "%s"' % scheme) if scheme == 'file': path = _GetPathFromUrlString(url_str) is_stream = (path == '-') is_fifo = False try: is_fifo = stat.S_ISFIFO(os.stat(path).st_mode) except OSError: pass return _FileUrl(url_str, is_stream=is_stream, is_fifo=is_fifo) return _CloudUrl(url_str)
def StorageUrlFromString(url_str): """Static factory function for creating a StorageUrl from a string.""" end_scheme_idx = url_str.find('://') if end_scheme_idx == -1: # File is the default scheme. scheme = 'file' path = url_str else: scheme = url_str[0:end_scheme_idx].lower() path = url_str[end_scheme_idx + 3:] if scheme not in ('file', 's3', 'gs'): raise InvalidUrlError('Unrecognized scheme "%s"' % scheme) if scheme == 'file': is_stream = (path == '-') return _FileUrl(url_str, is_stream=is_stream) return _CloudUrl(url_str)
def _SetUploadUrl(self, url): """Saves URL and resets upload state. Called when we start a new resumable upload or get a new tracker URL for the upload. Args: url: URL string for the upload. Raises InvalidUrlError if URL is syntactically invalid. """ parse_result = urllib.parse.urlparse(url) if (parse_result.scheme.lower() not in ['http', 'https'] or not parse_result.netloc): raise InvalidUrlError('Invalid upload URL (%s)' % url) self.upload_url = url self.upload_url_host = parse_result.netloc self.upload_url_path = '%s?%s' % (parse_result.path, parse_result.query) self.service_has_bytes = 0
def RunCommand(self): """Command entry point for the mb command.""" bucket_policy_only = None location = None storage_class = None seconds = None if self.sub_opts: for o, a in self.sub_opts: if o == '-l': location = a elif o == '-p': # Project IDs are sent as header values when using gs and s3 XML APIs. InsistAscii( a, 'Invalid non-ASCII character found in project ID') self.project_id = a elif o == '-c' or o == '-s': storage_class = NormalizeStorageClass(a) elif o == '--retention': seconds = RetentionInSeconds(a) elif o == '-b': if self.gsutil_api.GetApiSelector( 'gs') != ApiSelector.JSON: raise CommandException( 'The -b <on|off> option ' 'can only be used with the JSON API') InsistOnOrOff( a, 'Only on and off values allowed for -b option') bucket_policy_only = (a == 'on') bucket_metadata = apitools_messages.Bucket(location=location, storageClass=storage_class) if bucket_policy_only: bucket_metadata.iamConfiguration = IamConfigurationValue() iam_config = bucket_metadata.iamConfiguration iam_config.bucketPolicyOnly = BucketPolicyOnlyValue() iam_config.bucketPolicyOnly.enabled = bucket_policy_only for bucket_url_str in self.args: bucket_url = StorageUrlFromString(bucket_url_str) if seconds is not None: if bucket_url.scheme != 'gs': raise CommandException( 'Retention policy can only be specified for ' 'GCS buckets.') retention_policy = ( apitools_messages.Bucket.RetentionPolicyValue( retentionPeriod=seconds)) bucket_metadata.retentionPolicy = retention_policy if not bucket_url.IsBucket(): raise CommandException( 'The mb command requires a URL that specifies a ' 'bucket.\n"%s" is not valid.' % bucket_url) if (not BUCKET_NAME_RE.match(bucket_url.bucket_name) or TOO_LONG_DNS_NAME_COMP.search(bucket_url.bucket_name)): raise InvalidUrlError('Invalid bucket name in URL "%s"' % bucket_url.bucket_name) self.logger.info('Creating %s...', bucket_url) # Pass storage_class param only if this is a GCS bucket. (In S3 the # storage class is specified on the key object.) try: self.gsutil_api.CreateBucket(bucket_url.bucket_name, project_id=self.project_id, metadata=bucket_metadata, provider=bucket_url.scheme) except BadRequestException as e: if (e.status == 400 and e.reason == 'DotfulBucketNameNotUnderTld' and bucket_url.scheme == 'gs'): bucket_name = bucket_url.bucket_name final_comp = bucket_name[bucket_name.rfind('.') + 1:] raise CommandException('\n'.join( textwrap.wrap( 'Buckets with "." in the name must be valid DNS names. The bucket' ' you are attempting to create (%s) is not a valid DNS name,' ' because the final component (%s) is not currently a valid part' ' of the top-level DNS tree.' % (bucket_name, final_comp)))) else: raise return 0
def RunCommand(self): """Command entry point for the mb command.""" autoclass = False bucket_policy_only = None kms_key = None location = None storage_class = None seconds = None public_access_prevention = None rpo = None json_only_flags_in_command = [] if self.sub_opts: for o, a in self.sub_opts: if o == '--autoclass': autoclass = True json_only_flags_in_command.append(o) elif o == '-k': kms_key = a ValidateCMEK(kms_key) json_only_flags_in_command.append(o) elif o == '-l': location = a elif o == '-p': # Project IDs are sent as header values when using gs and s3 XML APIs. InsistAscii( a, 'Invalid non-ASCII character found in project ID') self.project_id = a elif o == '-c' or o == '-s': storage_class = NormalizeStorageClass(a) elif o == '--retention': seconds = RetentionInSeconds(a) elif o == '--rpo': rpo = a.strip() if rpo not in VALID_RPO_VALUES: raise CommandException( 'Invalid value for --rpo. Must be one of: {},' ' provided: {}'.format(VALID_RPO_VALUES_STRING, a)) json_only_flags_in_command.append(o) elif o == '-b': InsistOnOrOff( a, 'Only on and off values allowed for -b option') bucket_policy_only = (a == 'on') json_only_flags_in_command.append(o) elif o == '--pap': public_access_prevention = a json_only_flags_in_command.append(o) bucket_metadata = apitools_messages.Bucket(location=location, rpo=rpo, storageClass=storage_class) if autoclass: bucket_metadata.autoclass = apitools_messages.Bucket.AutoclassValue( enabled=autoclass) if bucket_policy_only or public_access_prevention: bucket_metadata.iamConfiguration = IamConfigurationValue() iam_config = bucket_metadata.iamConfiguration if bucket_policy_only: iam_config.bucketPolicyOnly = BucketPolicyOnlyValue() iam_config.bucketPolicyOnly.enabled = bucket_policy_only if public_access_prevention: iam_config.publicAccessPrevention = public_access_prevention if kms_key: encryption = apitools_messages.Bucket.EncryptionValue() encryption.defaultKmsKeyName = kms_key bucket_metadata.encryption = encryption for bucket_url_str in self.args: bucket_url = StorageUrlFromString(bucket_url_str) if seconds is not None: if bucket_url.scheme != 'gs': raise CommandException( 'Retention policy can only be specified for ' 'GCS buckets.') retention_policy = ( apitools_messages.Bucket.RetentionPolicyValue( retentionPeriod=seconds)) bucket_metadata.retentionPolicy = retention_policy if json_only_flags_in_command and self.gsutil_api.GetApiSelector( bucket_url.scheme) != ApiSelector.JSON: raise CommandException( 'The {} option(s) can only be used for GCS' ' Buckets with the JSON API'.format( ', '.join(json_only_flags_in_command))) if not bucket_url.IsBucket(): raise CommandException( 'The mb command requires a URL that specifies a ' 'bucket.\n"%s" is not valid.' % bucket_url) if (not BUCKET_NAME_RE.match(bucket_url.bucket_name) or TOO_LONG_DNS_NAME_COMP.search(bucket_url.bucket_name)): raise InvalidUrlError('Invalid bucket name in URL "%s"' % bucket_url.bucket_name) self.logger.info('Creating %s...', bucket_url) # Pass storage_class param only if this is a GCS bucket. (In S3 the # storage class is specified on the key object.) try: self.gsutil_api.CreateBucket(bucket_url.bucket_name, project_id=self.project_id, metadata=bucket_metadata, provider=bucket_url.scheme) except AccessDeniedException as e: message = e.reason if 'key' in message: # This will print the error reason and append the following as a # suggested next step: # # To authorize, run: # gsutil kms authorize \ # -k <kms_key> \ # -p <project_id> message += ' To authorize, run:\n gsutil kms authorize' message += ' \\\n -k %s' % kms_key if (self.project_id): message += ' \\\n -p %s' % self.project_id raise CommandException(message) else: raise except BadRequestException as e: if (e.status == 400 and e.reason == 'DotfulBucketNameNotUnderTld' and bucket_url.scheme == 'gs'): bucket_name = bucket_url.bucket_name final_comp = bucket_name[bucket_name.rfind('.') + 1:] raise CommandException('\n'.join( textwrap.wrap( 'Buckets with "." in the name must be valid DNS names. The bucket' ' you are attempting to create (%s) is not a valid DNS name,' ' because the final component (%s) is not currently a valid part' ' of the top-level DNS tree.' % (bucket_name, final_comp)))) else: raise return 0