def test_service_action_params_reuse(self):
request_model = Request({
'operation':
'GetFrobs',
'params': [{
'target': 'Delete.Objects[].Key',
'source': 'data',
'path': 'Key',
}],
})
item1 = mock.Mock()
item1.meta = ResourceMeta('test', data={'Key': 'item1'})
item2 = mock.Mock()
item2.meta = ResourceMeta('test', data={'Key': 'item2'})
# Here we create params and then re-use it to build up a more
# complex structure over multiple calls.
params = create_request_parameters(item1, request_model)
create_request_parameters(item2, request_model, params=params)
assert params == {
'Delete': {
'Objects': [{
'Key': 'item1'
}, {
'Key': 'item2'
}]
}
}
def test_service_action_params_reuse(self):
request_model = Request({
'operation': 'GetFrobs',
'params': [
{
'target': 'Delete.Objects[].Key',
'source': 'data',
'path': 'Key'
}
]
})
item1 = mock.Mock()
item1.meta = ResourceMeta('test', data={
'Key': 'item1'
})
item2 = mock.Mock()
item2.meta = ResourceMeta('test', data={
'Key': 'item2'
})
# Here we create params and then re-use it to build up a more
# complex structure over multiple calls.
params = create_request_parameters(item1, request_model)
create_request_parameters(item2, request_model, params=params)
self.assertEqual(params, {
'Delete': {
'Objects': [
{'Key': 'item1'},
{'Key': 'item2'}
]
}
})
def test_service_action_params_invalid(self):
request_model = Request({
'operation':
'GetFrobs',
'params': [{
'target': 'Param1',
'source': 'invalid'
}],
})
with pytest.raises(NotImplementedError):
create_request_parameters(None, request_model)
async def __call__(self, parent, *args, **kwargs):
"""
Perform the batch action's operation on every page of results
from the collection.
:type parent:
:py:class:`~boto3.resources.collection.ResourceCollection`
:param parent: The collection iterator to which this action
is attached.
:rtype: list(dict)
:return: A list of low-level response dicts from each call.
"""
service_name = None
client = None
responses = []
operation_name = xform_name(self._action_model.request.operation)
# Unlike the simple action above, a batch action must operate
# on batches (or pages) of items. So we get each page, construct
# the necessary parameters and call the batch operation.
async for page in parent.pages():
params = {}
for index, resource in enumerate(page):
# There is no public interface to get a service name
# or low-level client from a collection, so we get
# these from the first resource in the collection.
if service_name is None:
service_name = resource.meta.service_name
if client is None:
client = resource.meta.client
create_request_parameters(resource,
self._action_model.request,
params=params,
index=index)
if not params:
# There are no items, no need to make a call.
break
params.update(kwargs)
logger.debug('Calling %s:%s with %r', service_name, operation_name,
params)
response = await (getattr(client, operation_name)(**params))
logger.debug('Response: %r', response)
responses.append(self._response_handler(parent, params, response))
return responses
def test_service_action_params_invalid(self):
request_model = Request({
'operation': 'GetFrobs',
'params': [
{
'target': 'Param1',
'source': 'invalid'
}
]
})
with self.assertRaises(NotImplementedError):
create_request_parameters(None, request_model)
def test_service_action_params_input(self):
request_model = Request({
'operation': 'GetFrobs',
'params': [
{'target': 'Param1', 'source': 'input'}
]
})
params = create_request_parameters(None, request_model)
self.assertEqual(params, {})
params['param1'] = 'myinput'
params = create_request_parameters(None, request_model, params=params)
self.assertEqual(params, {'param1': 'myinput'})
def test_service_action_params_invalid(self):
action_def = {
'request': {
'operation':
'GetFrobs',
'params': [{
'target': 'Param1',
'sourceType': 'invalid',
'source': 'param1'
}]
}
}
with self.assertRaises(NotImplementedError):
create_request_parameters(None, action_def['request'])
def test_service_action_params_constants(self):
request_model = Request({
'operation': 'GetFrobs',
'params': [
{
'target': 'Param1',
'source': 'string',
'value': 'param1'
},
{
'target': 'Param2',
'source': 'integer',
'value': 123
},
{
'target': 'Param3',
'source': 'boolean',
'value': True
}
]
})
params = create_request_parameters(None, request_model)
self.assertEqual(params['Param1'], 'param1',
'Parameter not set from string constant')
self.assertEqual(params['Param2'], 123,
'Parameter not set from integer constant')
self.assertEqual(params['Param3'], True,
'Parameter not set from boolean constant')
async def async_call(self, parent, *args, **kwargs):
"""
Perform the action's request operation after building operation
parameters and build any defined resources from the response.
:type parent: :py:class:`~boto3.resources.base.ServiceResource`
:param parent: The resource instance to which this action is attached.
:rtype: dict or ServiceResource or list(ServiceResource)
:return: The response, either as a raw dict or resource instance(s).
"""
operation_name = xform_name(self._action_model.request.operation)
# First, build predefined params and then update with the
# user-supplied kwargs, which allows overriding the pre-built
# params if needed.
params = create_request_parameters(parent, self._action_model.request)
params.update(kwargs)
logger.debug('Calling %s:%s with %r', parent.meta.service_name,
operation_name, params)
response = await getattr(parent.meta.client, operation_name)(**params)
logger.debug('Response: %r', response)
return self._response_handler(parent, params, response)
def test_service_action_params_data_member_missing(self):
request_model = Request({
'operation': 'GetFrobs',
'params': [
{
'target': 'WarehouseUrl',
'source': 'data',
'path': 'SomeMember'
}
]
})
parent = mock.Mock()
def load_data():
parent.meta.data = {
'SomeMember': 'w-url'
}
parent.load.side_effect = load_data
parent.meta = ResourceMeta('test')
params = create_request_parameters(parent, request_model)
parent.load.assert_called_with()
self.assertEqual(params['WarehouseUrl'], 'w-url',
'Parameter not set from resource property')
def _get_paginated_results(self, limit, marker, collection):
"""
If a Boto Paginator is available, use it. The results
are converted back into BotoResources by directly accessing
protected members of ResourceCollection. This logic can be removed
depending on issue: https://github.com/boto/boto3/issues/1268.
"""
# pylint:disable=protected-access
cleaned_params = collection._params.copy()
cleaned_params.pop('limit', None)
cleaned_params.pop('page_size', None)
# pylint:disable=protected-access
params = create_request_parameters(collection._parent,
collection._model.request)
merge_dicts(params, cleaned_params, append_lists=True)
client = self.boto_conn.meta.client
list_op = self._get_list_operation()
paginator = client.get_paginator(list_op)
PaginationConfig = {}
if limit:
PaginationConfig = {'MaxItems': limit, 'PageSize': limit}
if marker:
PaginationConfig.update({'StartingToken': marker})
params.update({'PaginationConfig': PaginationConfig})
args = trim_empty_params(params)
pages = paginator.paginate(**args)
# resume_token is not populated unless the iterator is used
items = pages.build_full_result()
boto_objs = self._to_boto_resource(collection, args, items)
resume_token = pages.resume_token
return (resume_token, boto_objs)
def test_service_action_params_constants(self):
request_model = Request({
'operation':
'GetFrobs',
'params': [
{
'target': 'Param1',
'source': 'string',
'value': 'param1',
},
{
'target': 'Param2',
'source': 'integer',
'value': 123
},
{
'target': 'Param3',
'source': 'boolean',
'value': True
},
],
})
params = create_request_parameters(None, request_model)
assert params['Param1'] == 'param1'
assert params['Param2'] == 123
assert params['Param3'] is True
def test_service_action_params_constants(self):
request_model = Request({
'operation':
'GetFrobs',
'params': [{
'target': 'Param1',
'source': 'string',
'value': 'param1'
}, {
'target': 'Param2',
'source': 'integer',
'value': 123
}, {
'target': 'Param3',
'source': 'boolean',
'value': True
}]
})
params = create_request_parameters(None, request_model)
self.assertEqual(params['Param1'], 'param1',
'Parameter not set from string constant')
self.assertEqual(params['Param2'], 123,
'Parameter not set from integer constant')
self.assertEqual(params['Param3'], True,
'Parameter not set from boolean constant')
def test_service_action_params_input(self):
request_model = Request({
'operation':
'GetFrobs',
'params': [{
'target': 'Param1',
'source': 'input'
}],
})
params = create_request_parameters(None, request_model)
assert params == {}
params['param1'] = 'myinput'
params = create_request_parameters(None, request_model, params=params)
assert params == {'param1': 'myinput'}
def test_service_action_params_data_member_missing_no_load(self):
request_model = Request({
'operation':
'GetFrobs',
'params': [{
'target': 'WarehouseUrl',
'source': 'data',
'path': 'SomeMember',
}],
})
# This mock has no ``load`` method.
parent = mock.Mock(spec=ServiceResource)
parent.meta = ResourceMeta('test', data=None)
with pytest.raises(ResourceLoadException):
create_request_parameters(parent, request_model)
async def pages(self):
client = self._parent.meta.client
cleaned_params = self._params.copy()
limit = cleaned_params.pop('limit', None)
page_size = cleaned_params.pop('page_size', None)
params = create_request_parameters(
self._parent, self._model.request)
merge_dicts(params, cleaned_params, append_lists=True)
# Is this a paginated operation? If so, we need to get an
# iterator for the various pages. If not, then we simply
# call the operation and return the result as a single
# page in a list. For non-paginated results, we just ignore
# the page size parameter.
if client.can_paginate(self._py_operation_name):
logger.debug('Calling paginated %s:%s with %r',
self._parent.meta.service_name,
self._py_operation_name, params)
paginator = client.get_paginator(self._py_operation_name)
pages = paginator.paginate(
PaginationConfig={
'MaxItems': limit, 'PageSize': page_size}, **params)
else:
@async_generator
async def _aiopaginatordummy():
res = await getattr(client, self._py_operation_name)(**params)
await yield_(res)
logger.debug('Calling %s:%s with %r',
self._parent.meta.service_name,
self._py_operation_name, params)
pages = _aiopaginatordummy()
# Now that we have a page iterator or single page of results
# we start processing and yielding individual items.
count = 0
async for page in pages:
page_items = []
for item in self._handler(self._parent, params, page):
page_items.append(item)
# If the limit is set and has been reached, then
# we stop processing items here.
count += 1
if limit is not None and count >= limit:
break
await yield_(page_items)
# Stop reading pages if we've reached out limit
if limit is not None and count >= limit:
break
def test_service_action_params_list(self):
request_model = Request({
'operation':
'GetFrobs',
'params': [{
'target': 'WarehouseUrls[0]',
'source': 'string',
'value': 'w-url',
}],
})
params = create_request_parameters(None, request_model)
assert isinstance(params['WarehouseUrls'], list)
assert len(params['WarehouseUrls']) == 1
assert 'w-url' in params['WarehouseUrls']
def test_service_action_params_data_member(self):
request_model = Request({
'operation':
'GetFrobs',
'params': [{
'target': 'WarehouseUrl',
'source': 'data',
'path': 'SomeMember',
}],
})
parent = mock.Mock()
parent.meta = ResourceMeta('test', data={'SomeMember': 'w-url'})
params = create_request_parameters(parent, request_model)
assert params['WarehouseUrl'] == 'w-url'
async def __call__(self, parent, *args, **kwargs):
operation_name = xform_name(self._action_model.request.operation)
# First, build predefined params and then update with the
# user-supplied kwargs, which allows overriding the pre-built
# params if needed.
params = create_request_parameters(parent, self._action_model.request)
params.update(kwargs)
logger.debug('Calling %s:%s with %r', parent.meta.service_name,
operation_name, params)
response = await getattr(parent.meta.client, operation_name)(**params)
logger.debug('Response: %r', response)
return await self._response_handler(parent, params, response)
def test_service_action_params_identifier(self):
request_model = Request({
'operation':
'GetFrobs',
'params': [{
'target': 'WarehouseUrl',
'source': 'identifier',
'name': 'Url',
}],
})
parent = mock.Mock()
parent.url = 'w-url'
params = create_request_parameters(parent, request_model)
assert params['WarehouseUrl'] == 'w-url'
def test_service_action_params_data_member(self):
request_model = Request({
'operation':
'GetFrobs',
'params': [{
'target': 'WarehouseUrl',
'source': 'data',
'path': 'SomeMember'
}]
})
parent = mock.Mock()
parent.meta = ResourceMeta('test', data={'SomeMember': 'w-url'})
params = create_request_parameters(parent, request_model)
self.assertEqual(params['WarehouseUrl'], 'w-url',
'Parameter not set from resource property')
def test_service_action_params_identifier(self):
request_model = Request({
'operation':
'GetFrobs',
'params': [{
'target': 'WarehouseUrl',
'source': 'identifier',
'name': 'Url'
}]
})
parent = mock.Mock()
parent.url = 'w-url'
params = create_request_parameters(parent, request_model)
self.assertEqual(params['WarehouseUrl'], 'w-url',
'Parameter not set from resource identifier')
def test_service_action_params_data_member_missing_no_load(self):
request_model = Request({
'operation': 'GetFrobs',
'params': [
{
'target': 'WarehouseUrl',
'source': 'data',
'path': 'SomeMember'
}
]
})
# This mock has no ``load`` method.
parent = mock.Mock(spec=ServiceResource)
parent.meta = ResourceMeta('test', data=None)
with self.assertRaises(ResourceLoadException):
params = create_request_parameters(parent, request_model)
def test_service_action_params_data_member(self):
request_model = Request({
'operation': 'GetFrobs',
'params': [
{
'target': 'WarehouseUrl',
'sourceType': 'dataMember',
'source': 'some_member'
}
]
})
parent = mock.Mock()
parent.some_member = 'w-url'
params = create_request_parameters(parent, request_model)
self.assertEqual(params['WarehouseUrl'], 'w-url',
'Parameter not set from resource property')
def test_service_action_params_list(self):
request_model = Request({
'operation':
'GetFrobs',
'params': [{
'target': 'WarehouseUrls[0]',
'source': 'string',
'value': 'w-url'
}]
})
params = create_request_parameters(None, request_model)
self.assertIsInstance(params['WarehouseUrls'], list,
'Parameter did not create a list')
self.assertEqual(len(params['WarehouseUrls']), 1,
'Parameter list should only have a single item')
self.assertIn('w-url', params['WarehouseUrls'],
'Parameter not in expected list')
def test_service_action_params_identifier(self):
request_model = Request({
'operation': 'GetFrobs',
'params': [
{
'target': 'WarehouseUrl',
'source': 'identifier',
'name': 'Url'
}
]
})
parent = mock.Mock()
parent.url = 'w-url'
params = create_request_parameters(parent, request_model)
self.assertEqual(params['WarehouseUrl'], 'w-url',
'Parameter not set from resource identifier')
def test_service_action_params_identifier(self):
action_def = {
'request': {
'operation':
'GetFrobs',
'params': [{
'target': 'WarehouseUrl',
'sourceType': 'identifier',
'source': 'Url'
}]
}
}
parent = mock.Mock()
parent.url = 'w-url'
params = create_request_parameters(parent, action_def['request'])
self.assertEqual(params['WarehouseUrl'], 'w-url',
'Parameter not set from resource identifier')
def test_service_action_params_data_member(self):
action_def = {
'request': {
'operation':
'GetFrobs',
'params': [{
'target': 'WarehouseUrl',
'sourceType': 'dataMember',
'source': 'some_member'
}]
}
}
parent = mock.Mock()
parent.some_member = 'w-url'
params = create_request_parameters(parent, action_def['request'])
self.assertEqual(params['WarehouseUrl'], 'w-url',
'Parameter not set from resource property')
def test_service_action_params_list(self):
request_model = Request({
'operation': 'GetFrobs',
'params': [
{
'target': 'WarehouseUrls[0]',
'source': 'string',
'value': 'w-url'
}
]
})
params = create_request_parameters(None, request_model)
self.assertIsInstance(params['WarehouseUrls'], list,
'Parameter did not create a list')
self.assertEqual(len(params['WarehouseUrls']), 1,
'Parameter list should only have a single item')
self.assertIn('w-url', params['WarehouseUrls'],
'Parameter not in expected list')
def load(self, *args) -> None:
if self.meta.resource_model.load:
identifiers = create_request_parameters(
self, self.meta.resource_model.load.request)
elif self.service_name in "s3" and self.resource_type in "bucket":
# This is hardcoded in boto3 believe it or not!
# https://github.com/boto/boto3/blob/master/boto3/s3/inject.py#L57
identifiers = {"BucketName": self.name}
else:
raise UnsupportedResourceTypeError(
f"{self.service_name} {self.resource_type} does not have a load definition in its resources-1.json"
)
has_non_empty_values = any(x for x in identifiers.values())
if not has_non_empty_values:
logger.debug(
"Load is a noop on this %s %s because we are an empty_resource=True resource",
self.service_name,
self.resource_type,
)
return
original_class_load(self)
def test_service_action_params_data_member_missing(self):
request_model = Request({
'operation':
'GetFrobs',
'params': [{
'target': 'WarehouseUrl',
'source': 'data',
'path': 'SomeMember',
}],
})
parent = mock.Mock()
def load_data():
parent.meta.data = {'SomeMember': 'w-url'}
parent.load.side_effect = load_data
parent.meta = ResourceMeta('test')
params = create_request_parameters(parent, request_model)
parent.load.assert_called_with()
assert params['WarehouseUrl'] == 'w-url'
async def __call__(self, parent, *args, **kwargs):
"""
Perform the wait operation after building operation
parameters.
:type parent: :py:class:`~boto3.resources.base.ServiceResource`
:param parent: The resource instance to which this action is attached.
"""
client_waiter_name = xform_name(self._waiter_model.waiter_name)
# First, build predefined params and then update with the
# user-supplied kwargs, which allows overriding the pre-built
# params if needed.
params = create_request_parameters(parent, self._waiter_model)
params.update(kwargs)
logger.debug('Calling %s:%s with %r', parent.meta.service_name,
self._waiter_resource_name, params)
client = parent.meta.client
waiter = client.get_waiter(client_waiter_name)
response = await waiter.wait(**params)
logger.debug('Response: %r', response)