def test_gcs_bucket_wildcard_and_object_wildcard(self, fields_scope,
client):
"""Test multiple object matches in multiple buckets."""
self.bucket1_object = test_resources.get_object_resource(
storage_url.ProviderPrefix.GCS, 'bucket1', 'a.txt')
self.bucket2_object = test_resources.get_object_resource(
storage_url.ProviderPrefix.GCS, 'bucket2', 'a.txt')
client.list_buckets.side_effect = [self.buckets_response]
client.list_objects.side_effect = [[self.bucket1_object],
[self.bucket2_object]]
resource_iterator = wildcard_iterator.get_wildcard_iterator(
'gs://bucket*/*', fields_scope=fields_scope)
self.assertEqual(list(resource_iterator),
[self.bucket1_object, self.bucket2_object])
client.list_buckets.assert_called_once_with(
cloud_api.FieldsScope(fields_scope))
client.list_objects.mock_calls = [
mock.call(all_versions=False,
bucket_name='bucket1',
delimiter='/',
fields_scope=fields_scope,
prefix=None),
mock.call(all_versions=False,
bucket_name='bucket2',
delimiter='/',
fields_scope=fields_scope,
prefix=None)
]
def test_gcs_list_all_object_versions(self, client):
"""Test with generation."""
client.list_objects.return_value = self._object_resources_with_generation
resource_list = list(
wildcard_iterator.get_wildcard_iterator('gs://bucket1/a.txt',
all_versions=True))
expected_resources = [
test_resources.get_object_resource(storage_url.ProviderPrefix.GCS,
'bucket1', 'a.txt', '1'),
test_resources.get_object_resource(storage_url.ProviderPrefix.GCS,
'bucket1', 'a.txt', '2'),
]
self.assertEqual(resource_list, expected_resources)
client.list_objects.assert_called_once_with(
all_versions=True,
bucket_name='bucket1',
delimiter='/',
fields_scope=cloud_api.FieldsScope.NO_ACL,
prefix='a.txt',
)
def test_object_with_generation_without_wildcard(self, client):
"""Test with generation."""
resource = test_resources.get_object_resource(
storage_url.ProviderPrefix.GCS, 'bucket1', 'a.txt', '1')
client.get_object_metadata.return_value = resource
resource_list = list(
wildcard_iterator.get_wildcard_iterator('gs://bucket1/a.txt#1'))
self.assertEqual(resource_list, [resource])
client.get_object_metadata.assert_called_once_with(
'bucket1', 'a.txt', '1', cloud_api.FieldsScope.NO_ACL)
self.assertFalse(client.list_objects.called)
def test_list_objects_without_wildcard(self, mock_client):
resource = test_resources.get_object_resource(
storage_url.ProviderPrefix.GCS, 'bucket', 'a/b.txt')
mock_client.get_object_metadata.side_effect = [resource]
resources = list(
wildcard_iterator.get_wildcard_iterator('gs://bucket/a/b.txt'))
self.assertEqual(resources, [resource])
mock_client.get_object_metadata.assert_called_once_with(
'bucket', 'a/b.txt', None, cloud_api.FieldsScope.NO_ACL)
self.assertFalse(mock_client.list_objects.called)
def SetUp(self):
self.project = 'fake-project'
properties.VALUES.core.project.Set(self.project)
self.messages = apis.GetMessagesModule('storage', 'v1')
self.bucket1 = self.messages.Bucket(name='bucket1')
self.bucket2 = self.messages.Bucket(name='bucket2')
self.buckets = [self.bucket1, self.bucket2]
self.buckets_response = [
gcs_api._bucket_resource_from_metadata(bucket)
for bucket in self.buckets
]
self._object_resources_with_generation = [
test_resources.get_object_resource(storage_url.ProviderPrefix.GCS,
'bucket1', 'a.txt', '1'),
test_resources.get_object_resource(storage_url.ProviderPrefix.GCS,
'bucket1', 'a.txt', '2'),
test_resources.get_object_resource(storage_url.ProviderPrefix.GCS,
'bucket1', 'b.txt', '3'),
test_resources.get_object_resource(storage_url.ProviderPrefix.S3,
'bucket1', 'c.txt', 'a1')
]
def test_gcs_list_object_with_fields_scope(self, fields_scope, client):
"""Test if list_objects gets correct fields_scope."""
test_resource = test_resources.get_object_resource(
storage_url.ProviderPrefix.GCS, 'b', 'o.txt')
expected_resources = [test_resource]
client.list_objects.side_effect = [expected_resources]
resource_list = list(
wildcard_iterator.get_wildcard_iterator('gs://b/o*',
fields_scope=fields_scope))
self.assertEqual(resource_list, expected_resources)
client.list_objects.assert_called_once_with(
all_versions=False,
bucket_name='b',
delimiter='/',
fields_scope=fields_scope,
prefix='o',
)
def test_gcs_get_object_metadata_with_fields_scope(self, fields_scope,
client):
"""Test if get_object_metadata gets correct fields_scope."""
test_resource = test_resources.get_object_resource(
storage_url.ProviderPrefix.GCS, 'b', 'o.txt')
expected_resources = [test_resource]
client.get_object_metadata.side_effect = [test_resource]
resource_list = list(
wildcard_iterator.get_wildcard_iterator('gs://b/o.txt',
fields_scope=fields_scope))
self.assertEqual(resource_list, expected_resources)
client.get_object_metadata.assert_called_once_with(
bucket_name='b',
fields_scope=fields_scope,
generation=None,
object_name='o.txt',
)
class CloudWildcardIteratorTest(cloud_storage_util.WithGCSCalls,
parameterized.TestCase):
def SetUp(self):
self.project = 'fake-project'
properties.VALUES.core.project.Set(self.project)
self.messages = apis.GetMessagesModule('storage', 'v1')
self.bucket1 = self.messages.Bucket(name='bucket1')
self.bucket2 = self.messages.Bucket(name='bucket2')
self.buckets = [self.bucket1, self.bucket2]
self.buckets_response = [
gcs_api._bucket_resource_from_metadata(bucket)
for bucket in self.buckets
]
self._object_resources_with_generation = [
test_resources.get_object_resource(storage_url.ProviderPrefix.GCS,
'bucket1', 'a.txt', '1'),
test_resources.get_object_resource(storage_url.ProviderPrefix.GCS,
'bucket1', 'a.txt', '2'),
test_resources.get_object_resource(storage_url.ProviderPrefix.GCS,
'bucket1', 'b.txt', '3'),
test_resources.get_object_resource(storage_url.ProviderPrefix.S3,
'bucket1', 'c.txt', 'a1')
]
@parameterized.parameters(cloud_api.FieldsScope)
@mock_cloud_api.patch
def test_gcs_root_listing(self, fields_scope, client):
"""Test retrieving provider URL with no specified resource."""
client.list_buckets.side_effect = [self.buckets_response]
resource_iterator = wildcard_iterator.get_wildcard_iterator(
'gs://', fields_scope=fields_scope)
actual = [resource.metadata.name for resource in resource_iterator]
expected = [b.name for b in self.buckets]
self.assertEqual(actual, expected)
client.list_buckets.assert_called_once_with(
cloud_api.FieldsScope(fields_scope))
def test_invalid_scheme_raises_error(self):
"""Test wildcard iterator refuses invalid URL scheme."""
with self.assertRaises(command_errors.InvalidUrlError):
wildcard_iterator.get_wildcard_iterator(
'invalid://', fields_scope=cloud_api.FieldsScope.SHORT)
@parameterized.parameters(cloud_api.FieldsScope)
@mock_cloud_api.patch
def test_gcs_bucket_url_with_wildcard_gets_all_buckets(
self, fields_scope, client):
"""Test multiple bucket with bucket-level expansion."""
client.list_buckets.side_effect = [self.buckets_response]
resource_iterator = wildcard_iterator.get_wildcard_iterator(
'gs://bucket*', fields_scope=fields_scope)
self.assertEqual(list(resource_iterator), self.buckets_response)
client.list_buckets.assert_called_once_with(
cloud_api.FieldsScope(fields_scope))
@parameterized.parameters(cloud_api.FieldsScope)
@mock_cloud_api.patch
def test_gcs_bucket_url_with_wildcard_gets_single_bucket(
self, fields_scope, client):
"""Test single bucket with bucket-level expansion."""
client.list_buckets.side_effect = [self.buckets_response]
resource_iterator = wildcard_iterator.get_wildcard_iterator(
'gs://buck*1', fields_scope=fields_scope)
self.assertEqual(
list(resource_iterator),
[gcs_api._bucket_resource_from_metadata(self.bucket1)])
client.list_buckets.assert_called_once_with(
cloud_api.FieldsScope(fields_scope))
@parameterized.parameters(cloud_api.FieldsScope)
@mock_cloud_api.patch
def test_gcs_bucket_url_without_wildcard(self, fields_scope, client):
"""Test bucket with no bucket-level expansion."""
client.get_bucket.side_effect = [self.buckets_response[0]]
resource_iterator = wildcard_iterator.get_wildcard_iterator(
'gs://bucket1', fields_scope=fields_scope)
expected = self.buckets_response[:1]
self.assertEqual(list(resource_iterator), expected)
client.get_bucket.assert_called_once_with(
self.bucket1.name, cloud_api.FieldsScope(fields_scope))
@parameterized.parameters(cloud_api.FieldsScope)
@mock_cloud_api.patch
def test_gcs_bucket_wildcard_and_object_wildcard(self, fields_scope,
client):
"""Test multiple object matches in multiple buckets."""
self.bucket1_object = test_resources.get_object_resource(
storage_url.ProviderPrefix.GCS, 'bucket1', 'a.txt')
self.bucket2_object = test_resources.get_object_resource(
storage_url.ProviderPrefix.GCS, 'bucket2', 'a.txt')
client.list_buckets.side_effect = [self.buckets_response]
client.list_objects.side_effect = [[self.bucket1_object],
[self.bucket2_object]]
resource_iterator = wildcard_iterator.get_wildcard_iterator(
'gs://bucket*/*', fields_scope=fields_scope)
self.assertEqual(list(resource_iterator),
[self.bucket1_object, self.bucket2_object])
client.list_buckets.assert_called_once_with(
cloud_api.FieldsScope(fields_scope))
client.list_objects.mock_calls = [
mock.call(all_versions=False,
bucket_name='bucket1',
delimiter='/',
fields_scope=fields_scope,
prefix=None),
mock.call(all_versions=False,
bucket_name='bucket2',
delimiter='/',
fields_scope=fields_scope,
prefix=None)
]
@parameterized.named_parameters([{
'testcase_name':
'_list_with_generation_and_wildcard',
'wildcard_url':
'gs://bucket1/*.txt#3',
'request_prefix':
None,
'expected_resources': [
test_resources.get_object_resource(storage_url.ProviderPrefix.GCS,
'bucket1', 'b.txt', '3')
],
}, {
'testcase_name':
'_list_with_alphanumeric_generation_and_wildcard',
'wildcard_url':
's3://bucket1/*.txt#a1',
'request_prefix':
None,
'expected_resources': [
test_resources.get_object_resource(storage_url.ProviderPrefix.S3,
'bucket1', 'c.txt', 'a1')
],
}])
@mock_cloud_api.patch
def test_object_with_generation(self, client, wildcard_url, request_prefix,
expected_resources):
"""Test with generation."""
client.list_objects.return_value = self._object_resources_with_generation
resource_list = list(
wildcard_iterator.get_wildcard_iterator(wildcard_url))
self.assertEqual(resource_list, expected_resources)
self.assertFalse(client.get_object_metadata.called)
client.list_objects.assert_called_once_with(
all_versions=True,
bucket_name='bucket1',
delimiter='/',
fields_scope=cloud_api.FieldsScope.NO_ACL,
prefix=request_prefix,
)
@mock_cloud_api.patch
def test_object_with_incorrect_generation(self, client):
"""Test with generation."""
client.get_object_metadata.side_effect = api_errors.NotFoundError
client.list_objects.return_value = self._object_resources_with_generation
resources = list(
wildcard_iterator.get_wildcard_iterator('gs://bucket1/b.txt#2'))
self.assertEqual(resources, [])
client.get_object_metadata.assert_called_once_with(
'bucket1', 'b.txt', '2', cloud_api.FieldsScope.NO_ACL)
client.list_objects.assert_called_once_with(
'bucket1', 'b.txt', '/', True, cloud_api.FieldsScope.NO_ACL)
@mock_cloud_api.patch
def test_object_with_generation_without_wildcard(self, client):
"""Test with generation."""
resource = test_resources.get_object_resource(
storage_url.ProviderPrefix.GCS, 'bucket1', 'a.txt', '1')
client.get_object_metadata.return_value = resource
resource_list = list(
wildcard_iterator.get_wildcard_iterator('gs://bucket1/a.txt#1'))
self.assertEqual(resource_list, [resource])
client.get_object_metadata.assert_called_once_with(
'bucket1', 'a.txt', '1', cloud_api.FieldsScope.NO_ACL)
self.assertFalse(client.list_objects.called)
@mock_cloud_api.patch
def test_gcs_list_all_object_versions(self, client):
"""Test with generation."""
client.list_objects.return_value = self._object_resources_with_generation
resource_list = list(
wildcard_iterator.get_wildcard_iterator('gs://bucket1/a.txt',
all_versions=True))
expected_resources = [
test_resources.get_object_resource(storage_url.ProviderPrefix.GCS,
'bucket1', 'a.txt', '1'),
test_resources.get_object_resource(storage_url.ProviderPrefix.GCS,
'bucket1', 'a.txt', '2'),
]
self.assertEqual(resource_list, expected_resources)
client.list_objects.assert_called_once_with(
all_versions=True,
bucket_name='bucket1',
delimiter='/',
fields_scope=cloud_api.FieldsScope.NO_ACL,
prefix='a.txt',
)
@parameterized.parameters(cloud_api.FieldsScope)
@mock_cloud_api.patch
def test_gcs_get_object_metadata_with_fields_scope(self, fields_scope,
client):
"""Test if get_object_metadata gets correct fields_scope."""
test_resource = test_resources.get_object_resource(
storage_url.ProviderPrefix.GCS, 'b', 'o.txt')
expected_resources = [test_resource]
client.get_object_metadata.side_effect = [test_resource]
resource_list = list(
wildcard_iterator.get_wildcard_iterator('gs://b/o.txt',
fields_scope=fields_scope))
self.assertEqual(resource_list, expected_resources)
client.get_object_metadata.assert_called_once_with(
bucket_name='b',
fields_scope=fields_scope,
generation=None,
object_name='o.txt',
)
@parameterized.parameters(cloud_api.FieldsScope)
@mock_cloud_api.patch
def test_gcs_list_object_with_fields_scope(self, fields_scope, client):
"""Test if list_objects gets correct fields_scope."""
test_resource = test_resources.get_object_resource(
storage_url.ProviderPrefix.GCS, 'b', 'o.txt')
expected_resources = [test_resource]
client.list_objects.side_effect = [expected_resources]
resource_list = list(
wildcard_iterator.get_wildcard_iterator('gs://b/o*',
fields_scope=fields_scope))
self.assertEqual(resource_list, expected_resources)
client.list_objects.assert_called_once_with(
all_versions=False,
bucket_name='b',
delimiter='/',
fields_scope=fields_scope,
prefix='o',
)
@parameterized.parameters([
('gs://*', 'gs://*'),
('gs://**', 'gs://**'),
('gs://***', 'gs://*'),
('gs://*******', 'gs://*'),
('gs://b/***', 'gs://b/*'),
('gs://b/o#***', 'gs://b/o#***'),
])
def test_compresses_url_wildcards(self, observed_url_string,
expected_url_string):
"""Test if CloudWildcardIterator compresses URL wildcards correctly."""
iterator = wildcard_iterator.get_wildcard_iterator(observed_url_string)
self.assertEqual(iterator._url.url_string, expected_url_string)