def SetUp(self):
properties.VALUES.core.account.Set(TEST_ACCOUNT)
properties.VALUES.core.project.Set(TEST_PROJECT)
self.messages = core_apis.GetMessagesModule('storage', 'v1')
self.default_projection = (self.messages.StorageObjectsPatchRequest.
ProjectionValueValuesEnum.noAcl)
self.gcs_client = gcs_api.GcsApi()
def get_api(provider):
"""Returns thread local API instance for cloud provider.
Uses thread local storage to make sure only one instance of an API exists
per thread per provider.
Args:
provider (storage_url.ProviderPrefix): Cloud provider prefix.
Returns:
CloudApi client object for provider argument.
Raises:
ValueError: Invalid API provider.
"""
if properties.VALUES.storage.use_threading_local.GetBool():
api_client = getattr(_cloud_api_thread_local_storage, provider.value,
None)
if api_client:
return api_client
if provider == storage_url.ProviderPrefix.GCS:
api_client = gcs_api.GcsApi()
elif provider == storage_url.ProviderPrefix.S3:
api_client = s3_api.S3Api()
else:
raise ValueError(
'Provider must be a valid storage_url.ProviderPrefix.')
if properties.VALUES.storage.use_threading_local.GetBool():
setattr(_cloud_api_thread_local_storage, provider.value, api_client)
return api_client
def test_list_objects_valid_fields_scope(self, fields_scope, projection,
fields):
list_object_response = self.messages.Objects(items=[
self.messages.Object(name='object1'),
self.messages.Object(name='object2'),
])
api_client = mock.MagicMock(spec=['objects'])
api_client.objects.List.return_value = list_object_response
expected_request = self.messages.StorageObjectsListRequest(
bucket=TEST_BUCKET,
projection=getattr(
self.messages.StorageObjectsListRequest.
ProjectionValueValuesEnum, projection),
maxResults=cloud_api.NUM_ITEMS_PER_LIST_PAGE)
global_params = None
if fields:
global_params = self.messages.StandardQueryParameters()
global_params.fields = fields
with mock.patch.object(core_apis,
'GetClientInstance',
autospec=True,
return_value=api_client) as mock_get_instance:
gcs_client = gcs_api.GcsApi()
list(
gcs_client.list_objects(TEST_BUCKET,
fields_scope=fields_scope))
mock_get_instance.assert_called_once_with('storage', 'v1')
api_client.objects.List.assert_called_once_with(
expected_request, global_params=global_params)
def test_list_objects_paging(self, mock_get_instance):
filenames_batch1 = ['obj1', 'obj2', 'obj3']
filenames_batch2 = ['obj4', 'obj5', 'obj6']
filenames_batch3 = ['obj7']
list_objects_response = [
self.messages.Objects(
items=[self.messages.Object(name=c) for c in filenames_batch1],
nextPageToken='page2'),
self.messages.Objects(
items=[self.messages.Object(name=c) for c in filenames_batch2],
nextPageToken='page3'),
self.messages.Objects(
items=[self.messages.Object(name=c) for c in filenames_batch3])
]
default_projection = (self.messages.StorageObjectsListRequest.
ProjectionValueValuesEnum.noAcl)
mock_api_client = mock.MagicMock(spec=['objects'])
# "If side_effect is an iterable then each call to the mock will return
# the next value from the iterable."
# https://docs.python.org/3/library/unittest.mock.html#the-mock-class
mock_api_client.objects.List.side_effect = list_objects_response
mock_get_instance.return_value = mock_api_client
gcs_client = gcs_api.GcsApi()
mock_get_instance.assert_called_once_with('storage', 'v1')
objects = list(gcs_client.list_objects(TEST_BUCKET))
# Check that we looped over all the objects.
self.assertCountEqual([o.name for o in objects], filenames_batch1 +
filenames_batch2 + filenames_batch3)
# Check if calls to List are made as expected. We expected 3 calls.
expected_list_object_calls = []
for page_token in (None, 'page2', 'page3'):
expected_call = mock.call(self.messages.StorageObjectsListRequest(
bucket=TEST_BUCKET,
projection=default_projection,
pageToken=page_token,
maxResults=3),
global_params=None)
expected_list_object_calls.append(expected_call)
self.assertCountEqual(mock_api_client.objects.List.mock_calls,
expected_list_object_calls)
def get_api(provider):
"""Returns API instance for cloud provider.
Args:
provider (ProviderPrefix): Cloud provider prefix.
Returns:
CloudApi instance for specific cloud provider.
Raises:
ValueError: Invalid API provider.
"""
# TODO(b/167685797): Use thread-local.
if provider == storage_url.ProviderPrefix.GCS:
return gcs_api.GcsApi()
elif provider == storage_url.ProviderPrefix.S3:
return s3_api.S3Api()
raise ValueError('Provider API value must be "gs" or "s3".')
def SetUp(self):
properties.VALUES.core.account.Set(TEST_ACCOUNT)
properties.VALUES.core.project.Set(TEST_PROJECT)
self.messages = core_apis.GetMessagesModule('storage', 'v1')
self.gcs_client = gcs_api.GcsApi()
def SetUp(self):
self.messages = core_apis.GetMessagesModule('storage', 'v1')
self.gcs_client = gcs_api.GcsApi()
self.download_stream = mock.mock_open()