def MakeTempName(self, kind, prefix='', suffix=''):
"""Creates a temporary name that is most-likely unique.
Args:
kind (str): A string indicating what kind of test name this is.
prefix (str): Prefix prepended to the temporary name.
suffix (str): Suffix string appended to end of temporary name.
Returns:
(str) The temporary name. If `kind` was "bucket", the temporary name may
have coerced this string, including the supplied `prefix`, such that it
contains only characters that are valid across all supported storage
providers (e.g. replacing "_" with "-", converting uppercase letters to
lowercase, etc.).
"""
name = '{prefix}gsutil-test-{method}-{kind}'.format(
prefix=prefix, method=self.GetTestMethodName(), kind=kind)
name = name[:MAX_BUCKET_LENGTH - 13]
name = '{name}-{rand}'.format(name=name,
rand=self.MakeRandomTestString())
total_name_len = len(name) + len(suffix)
if suffix:
if kind == 'bucket' and total_name_len > MAX_BUCKET_LENGTH:
self.fail(
'Tried to create a psuedo-random bucket name with a specific '
'suffix, but the generated name was too long and there was not '
'enough room for the suffix. Please use shorter strings or perform '
'name randomization manually.\nRequested name: ' + name +
suffix)
name += suffix
if kind == 'bucket':
name = util.MakeBucketNameValid(name)
return name
def test_recursive_bucket_rm_with_wildcarding(self):
"""Tests removing all objects and buckets matching a bucket wildcard."""
buri_base = 'gsutil-test-%s' % self.GetTestMethodName()
buri_base = buri_base[:MAX_BUCKET_LENGTH - 20]
buri_base = '%s-%s' % (buri_base, self.MakeRandomTestString())
buri_base = 'aaa-' + buri_base
buri_base = util.MakeBucketNameValid(buri_base)
buri1 = self.CreateBucket(bucket_name='%s-tbuck1' % buri_base)
buri2 = self.CreateBucket(bucket_name='%s-tbuck2' % buri_base)
buri3 = self.CreateBucket(bucket_name='%s-tb3' % buri_base)
ouri1 = self.CreateObject(bucket_uri=buri1,
object_name='o1',
contents=b'z')
ouri2 = self.CreateObject(bucket_uri=buri2,
object_name='o2',
contents=b'z')
self.CreateObject(bucket_uri=buri3, object_name='o3', contents=b'z')
if self.multiregional_buckets:
self.AssertNObjectsInBucket(buri1, 1)
self.AssertNObjectsInBucket(buri2, 1)
self.AssertNObjectsInBucket(buri3, 1)
self._RunRemoveCommandAndCheck(
['rm', '-r',
'%s://%s-tbu*' % (self.default_provider, buri_base)],
objects_to_remove=[
'%s#%s' % (suri(ouri1), urigen(ouri1)),
'%s#%s' % (suri(ouri2), urigen(ouri2))
],
buckets_to_remove=[suri(buri1), suri(buri2)])
self.AssertNObjectsInBucket(buri3, 1)
def CreateBucketJson(self, bucket_name=None, test_objects=0,
storage_class=None, location=None,
versioning_enabled=False,
retention_policy=None,
bucket_policy_only=False):
"""Creates a test bucket using the JSON API.
The bucket and all of its contents will be deleted after the test.
Args:
bucket_name: Create the bucket with this name. If not provided, a
temporary test bucket name is constructed.
test_objects: The number of objects that should be placed in the bucket.
Defaults to 0.
storage_class: Storage class to use. If not provided we use standard.
location: Location to use.
versioning_enabled: If True, set the bucket's versioning attribute to
True.
retention_policy: Retention policy to be used on the bucket.
bucket_policy_only: If True, set the bucket's iamConfiguration's
bucketPolicyOnly attribute to True.
Returns:
Apitools Bucket for the created bucket.
"""
bucket_name = util.MakeBucketNameValid(
bucket_name or self.MakeTempName('bucket'))
bucket_metadata = apitools_messages.Bucket(name=bucket_name.lower())
if storage_class:
bucket_metadata.storageClass = storage_class
if location:
bucket_metadata.location = location
if versioning_enabled:
bucket_metadata.versioning = (
apitools_messages.Bucket.VersioningValue(enabled=True))
if retention_policy:
bucket_metadata.retentionPolicy = retention_policy
if bucket_policy_only:
iam_config = apitools_messages.Bucket.IamConfigurationValue()
iam_config.bucketPolicyOnly = iam_config.BucketPolicyOnlyValue()
iam_config.bucketPolicyOnly.enabled = True
bucket_metadata.iamConfiguration = iam_config
# TODO: Add retry and exponential backoff.
bucket = self.json_api.CreateBucket(bucket_name,
metadata=bucket_metadata)
# Add bucket to list of buckets to be cleaned up.
# TODO: Clean up JSON buckets using JSON API.
self.bucket_uris.append(
boto.storage_uri('gs://%s' % bucket_name,
suppress_consec_slashes=False))
for i in range(test_objects):
self.CreateObjectJson(bucket_name=bucket_name,
object_name=self.MakeTempName('obj'),
contents='test %d' % i)
return bucket
def MakeTempName(self, kind, prefix=''):
"""Creates a temporary name that is most-likely unique.
Args:
kind (str): A string indicating what kind of test name this is.
prefix (str): Prefix string to be used in the temporary name.
Returns:
(str) The temporary name. If `kind` was "bucket", the temporary name may
have coerced this string, including the supplied `prefix`, such that it
contains only characters that are valid across all supported storage
providers (e.g. replacing "_" with "-", converting uppercase letters to
lowercase, etc.).
"""
name = '%sgsutil-test-%s-%s' % (prefix, self.GetTestMethodName(), kind)
name = name[:MAX_BUCKET_LENGTH - 9]
name = '%s-%s' % (name, self.MakeRandomTestString())
# As of March 2018, S3 no longer accepts underscores or uppercase letters in
# bucket names.
if kind == 'bucket':
name = util.MakeBucketNameValid(name)
return name
def CreateBucket(self,
bucket_name=None,
test_objects=0,
storage_class=None,
retention_policy=None,
provider=None,
prefer_json_api=False,
versioning_enabled=False,
bucket_policy_only=False):
"""Creates a test bucket.
The bucket and all of its contents will be deleted after the test.
Args:
bucket_name: Create the bucket with this name. If not provided, a
temporary test bucket name is constructed.
test_objects: The number of objects that should be placed in the bucket.
Defaults to 0.
storage_class: Storage class to use. If not provided we us standard.
retention_policy: Retention policy to be used on the bucket.
provider: Provider to use - either "gs" (the default) or "s3".
prefer_json_api: If True, use the JSON creation functions where possible.
versioning_enabled: If True, set the bucket's versioning attribute to
True.
bucket_policy_only: If True, set the bucket's iamConfiguration's
bucketPolicyOnly attribute to True.
Returns:
StorageUri for the created bucket.
"""
if not provider:
provider = self.default_provider
# Location is controlled by the -b test flag.
if self.multiregional_buckets or provider == 's3':
location = None
else:
# We default to the "us-central1" location for regional buckets, but allow
# overriding this value in the Boto config.
location = boto.config.get(
'GSUtil', 'test_cmd_regional_bucket_location', 'us-central1')
if bucket_name:
bucket_name = util.MakeBucketNameValid(bucket_name)
if prefer_json_api and provider == 'gs':
json_bucket = self.CreateBucketJson(bucket_name=bucket_name,
test_objects=test_objects,
storage_class=storage_class,
location=location,
versioning_enabled=versioning_enabled,
retention_policy=retention_policy,
bucket_policy_only=bucket_policy_only)
bucket_uri = boto.storage_uri(
'gs://%s' % json_bucket.name.encode(UTF8).lower(),
suppress_consec_slashes=False)
return bucket_uri
bucket_name = bucket_name or self.MakeTempName('bucket')
bucket_uri = boto.storage_uri('%s://%s' % (provider, bucket_name.lower()),
suppress_consec_slashes=False)
if provider == 'gs':
# Apply API version and project ID headers if necessary.
headers = {'x-goog-api-version': self.api_version}
headers[GOOG_PROJ_ID_HDR] = PopulateProjectId()
else:
headers = {}
# Parallel tests can easily run into bucket creation quotas.
# Retry with exponential backoff so that we create them as fast as we
# reasonably can.
@Retry(StorageResponseError, tries=7, timeout_secs=1)
def _CreateBucketWithExponentialBackoff():
try:
bucket_uri.create_bucket(storage_class=storage_class,
location=location or '',
headers=headers)
except StorageResponseError, e:
# If the service returns a transient error or a connection breaks,
# it's possible the request succeeded. If that happens, the service
# will return 409s for all future calls even though our intent
# succeeded. If the error message says we already own the bucket,
# assume success to reduce test flakiness. This depends on
# randomness of test naming buckets to prevent name collisions for
# test buckets created concurrently in the same project, which is
# acceptable because this is far less likely than service errors.
if e.status == 409 and e.body and 'already own' in e.body:
pass
else:
raise