def MakeTempName(self, kind, prefix='', suffix=''): """Creates a temporary name that is most-likely unique. Args: kind (str): A string indicating what kind of test name this is. prefix (str): Prefix prepended to the temporary name. suffix (str): Suffix string appended to end of temporary name. Returns: (str) The temporary name. If `kind` was "bucket", the temporary name may have coerced this string, including the supplied `prefix`, such that it contains only characters that are valid across all supported storage providers (e.g. replacing "_" with "-", converting uppercase letters to lowercase, etc.). """ name = '{prefix}gsutil-test-{method}-{kind}'.format( prefix=prefix, method=self.GetTestMethodName(), kind=kind) name = name[:MAX_BUCKET_LENGTH - 13] name = '{name}-{rand}'.format(name=name, rand=self.MakeRandomTestString()) total_name_len = len(name) + len(suffix) if suffix: if kind == 'bucket' and total_name_len > MAX_BUCKET_LENGTH: self.fail( 'Tried to create a psuedo-random bucket name with a specific ' 'suffix, but the generated name was too long and there was not ' 'enough room for the suffix. Please use shorter strings or perform ' 'name randomization manually.\nRequested name: ' + name + suffix) name += suffix if kind == 'bucket': name = util.MakeBucketNameValid(name) return name
def test_recursive_bucket_rm_with_wildcarding(self): """Tests removing all objects and buckets matching a bucket wildcard.""" buri_base = 'gsutil-test-%s' % self.GetTestMethodName() buri_base = buri_base[:MAX_BUCKET_LENGTH - 20] buri_base = '%s-%s' % (buri_base, self.MakeRandomTestString()) buri_base = 'aaa-' + buri_base buri_base = util.MakeBucketNameValid(buri_base) buri1 = self.CreateBucket(bucket_name='%s-tbuck1' % buri_base) buri2 = self.CreateBucket(bucket_name='%s-tbuck2' % buri_base) buri3 = self.CreateBucket(bucket_name='%s-tb3' % buri_base) ouri1 = self.CreateObject(bucket_uri=buri1, object_name='o1', contents=b'z') ouri2 = self.CreateObject(bucket_uri=buri2, object_name='o2', contents=b'z') self.CreateObject(bucket_uri=buri3, object_name='o3', contents=b'z') if self.multiregional_buckets: self.AssertNObjectsInBucket(buri1, 1) self.AssertNObjectsInBucket(buri2, 1) self.AssertNObjectsInBucket(buri3, 1) self._RunRemoveCommandAndCheck( ['rm', '-r', '%s://%s-tbu*' % (self.default_provider, buri_base)], objects_to_remove=[ '%s#%s' % (suri(ouri1), urigen(ouri1)), '%s#%s' % (suri(ouri2), urigen(ouri2)) ], buckets_to_remove=[suri(buri1), suri(buri2)]) self.AssertNObjectsInBucket(buri3, 1)
def CreateBucketJson(self, bucket_name=None, test_objects=0, storage_class=None, location=None, versioning_enabled=False, retention_policy=None, bucket_policy_only=False): """Creates a test bucket using the JSON API. The bucket and all of its contents will be deleted after the test. Args: bucket_name: Create the bucket with this name. If not provided, a temporary test bucket name is constructed. test_objects: The number of objects that should be placed in the bucket. Defaults to 0. storage_class: Storage class to use. If not provided we use standard. location: Location to use. versioning_enabled: If True, set the bucket's versioning attribute to True. retention_policy: Retention policy to be used on the bucket. bucket_policy_only: If True, set the bucket's iamConfiguration's bucketPolicyOnly attribute to True. Returns: Apitools Bucket for the created bucket. """ bucket_name = util.MakeBucketNameValid( bucket_name or self.MakeTempName('bucket')) bucket_metadata = apitools_messages.Bucket(name=bucket_name.lower()) if storage_class: bucket_metadata.storageClass = storage_class if location: bucket_metadata.location = location if versioning_enabled: bucket_metadata.versioning = ( apitools_messages.Bucket.VersioningValue(enabled=True)) if retention_policy: bucket_metadata.retentionPolicy = retention_policy if bucket_policy_only: iam_config = apitools_messages.Bucket.IamConfigurationValue() iam_config.bucketPolicyOnly = iam_config.BucketPolicyOnlyValue() iam_config.bucketPolicyOnly.enabled = True bucket_metadata.iamConfiguration = iam_config # TODO: Add retry and exponential backoff. bucket = self.json_api.CreateBucket(bucket_name, metadata=bucket_metadata) # Add bucket to list of buckets to be cleaned up. # TODO: Clean up JSON buckets using JSON API. self.bucket_uris.append( boto.storage_uri('gs://%s' % bucket_name, suppress_consec_slashes=False)) for i in range(test_objects): self.CreateObjectJson(bucket_name=bucket_name, object_name=self.MakeTempName('obj'), contents='test %d' % i) return bucket
def MakeTempName(self, kind, prefix=''): """Creates a temporary name that is most-likely unique. Args: kind (str): A string indicating what kind of test name this is. prefix (str): Prefix string to be used in the temporary name. Returns: (str) The temporary name. If `kind` was "bucket", the temporary name may have coerced this string, including the supplied `prefix`, such that it contains only characters that are valid across all supported storage providers (e.g. replacing "_" with "-", converting uppercase letters to lowercase, etc.). """ name = '%sgsutil-test-%s-%s' % (prefix, self.GetTestMethodName(), kind) name = name[:MAX_BUCKET_LENGTH - 9] name = '%s-%s' % (name, self.MakeRandomTestString()) # As of March 2018, S3 no longer accepts underscores or uppercase letters in # bucket names. if kind == 'bucket': name = util.MakeBucketNameValid(name) return name
def CreateBucket(self, bucket_name=None, test_objects=0, storage_class=None, retention_policy=None, provider=None, prefer_json_api=False, versioning_enabled=False, bucket_policy_only=False): """Creates a test bucket. The bucket and all of its contents will be deleted after the test. Args: bucket_name: Create the bucket with this name. If not provided, a temporary test bucket name is constructed. test_objects: The number of objects that should be placed in the bucket. Defaults to 0. storage_class: Storage class to use. If not provided we us standard. retention_policy: Retention policy to be used on the bucket. provider: Provider to use - either "gs" (the default) or "s3". prefer_json_api: If True, use the JSON creation functions where possible. versioning_enabled: If True, set the bucket's versioning attribute to True. bucket_policy_only: If True, set the bucket's iamConfiguration's bucketPolicyOnly attribute to True. Returns: StorageUri for the created bucket. """ if not provider: provider = self.default_provider # Location is controlled by the -b test flag. if self.multiregional_buckets or provider == 's3': location = None else: # We default to the "us-central1" location for regional buckets, but allow # overriding this value in the Boto config. location = boto.config.get( 'GSUtil', 'test_cmd_regional_bucket_location', 'us-central1') if bucket_name: bucket_name = util.MakeBucketNameValid(bucket_name) if prefer_json_api and provider == 'gs': json_bucket = self.CreateBucketJson(bucket_name=bucket_name, test_objects=test_objects, storage_class=storage_class, location=location, versioning_enabled=versioning_enabled, retention_policy=retention_policy, bucket_policy_only=bucket_policy_only) bucket_uri = boto.storage_uri( 'gs://%s' % json_bucket.name.encode(UTF8).lower(), suppress_consec_slashes=False) return bucket_uri bucket_name = bucket_name or self.MakeTempName('bucket') bucket_uri = boto.storage_uri('%s://%s' % (provider, bucket_name.lower()), suppress_consec_slashes=False) if provider == 'gs': # Apply API version and project ID headers if necessary. headers = {'x-goog-api-version': self.api_version} headers[GOOG_PROJ_ID_HDR] = PopulateProjectId() else: headers = {} # Parallel tests can easily run into bucket creation quotas. # Retry with exponential backoff so that we create them as fast as we # reasonably can. @Retry(StorageResponseError, tries=7, timeout_secs=1) def _CreateBucketWithExponentialBackoff(): try: bucket_uri.create_bucket(storage_class=storage_class, location=location or '', headers=headers) except StorageResponseError, e: # If the service returns a transient error or a connection breaks, # it's possible the request succeeded. If that happens, the service # will return 409s for all future calls even though our intent # succeeded. If the error message says we already own the bucket, # assume success to reduce test flakiness. This depends on # randomness of test naming buckets to prevent name collisions for # test buckets created concurrently in the same project, which is # acceptable because this is far less likely than service errors. if e.status == 409 and e.body and 'already own' in e.body: pass else: raise