def test_service_action_params_reuse(self): request_model = Request({ 'operation': 'GetFrobs', 'params': [{ 'target': 'Delete.Objects[].Key', 'source': 'data', 'path': 'Key', }], }) item1 = mock.Mock() item1.meta = ResourceMeta('test', data={'Key': 'item1'}) item2 = mock.Mock() item2.meta = ResourceMeta('test', data={'Key': 'item2'}) # Here we create params and then re-use it to build up a more # complex structure over multiple calls. params = create_request_parameters(item1, request_model) create_request_parameters(item2, request_model, params=params) assert params == { 'Delete': { 'Objects': [{ 'Key': 'item1' }, { 'Key': 'item2' }] } }
def test_service_action_params_reuse(self): request_model = Request({ 'operation': 'GetFrobs', 'params': [ { 'target': 'Delete.Objects[].Key', 'source': 'data', 'path': 'Key' } ] }) item1 = mock.Mock() item1.meta = ResourceMeta('test', data={ 'Key': 'item1' }) item2 = mock.Mock() item2.meta = ResourceMeta('test', data={ 'Key': 'item2' }) # Here we create params and then re-use it to build up a more # complex structure over multiple calls. params = create_request_parameters(item1, request_model) create_request_parameters(item2, request_model, params=params) self.assertEqual(params, { 'Delete': { 'Objects': [ {'Key': 'item1'}, {'Key': 'item2'} ] } })
def test_service_action_params_invalid(self): request_model = Request({ 'operation': 'GetFrobs', 'params': [{ 'target': 'Param1', 'source': 'invalid' }], }) with pytest.raises(NotImplementedError): create_request_parameters(None, request_model)
async def __call__(self, parent, *args, **kwargs): """ Perform the batch action's operation on every page of results from the collection. :type parent: :py:class:`~boto3.resources.collection.ResourceCollection` :param parent: The collection iterator to which this action is attached. :rtype: list(dict) :return: A list of low-level response dicts from each call. """ service_name = None client = None responses = [] operation_name = xform_name(self._action_model.request.operation) # Unlike the simple action above, a batch action must operate # on batches (or pages) of items. So we get each page, construct # the necessary parameters and call the batch operation. async for page in parent.pages(): params = {} for index, resource in enumerate(page): # There is no public interface to get a service name # or low-level client from a collection, so we get # these from the first resource in the collection. if service_name is None: service_name = resource.meta.service_name if client is None: client = resource.meta.client create_request_parameters(resource, self._action_model.request, params=params, index=index) if not params: # There are no items, no need to make a call. break params.update(kwargs) logger.debug('Calling %s:%s with %r', service_name, operation_name, params) response = await (getattr(client, operation_name)(**params)) logger.debug('Response: %r', response) responses.append(self._response_handler(parent, params, response)) return responses
def test_service_action_params_invalid(self): request_model = Request({ 'operation': 'GetFrobs', 'params': [ { 'target': 'Param1', 'source': 'invalid' } ] }) with self.assertRaises(NotImplementedError): create_request_parameters(None, request_model)
def test_service_action_params_input(self): request_model = Request({ 'operation': 'GetFrobs', 'params': [ {'target': 'Param1', 'source': 'input'} ] }) params = create_request_parameters(None, request_model) self.assertEqual(params, {}) params['param1'] = 'myinput' params = create_request_parameters(None, request_model, params=params) self.assertEqual(params, {'param1': 'myinput'})
def test_service_action_params_invalid(self): action_def = { 'request': { 'operation': 'GetFrobs', 'params': [{ 'target': 'Param1', 'sourceType': 'invalid', 'source': 'param1' }] } } with self.assertRaises(NotImplementedError): create_request_parameters(None, action_def['request'])
def test_service_action_params_constants(self): request_model = Request({ 'operation': 'GetFrobs', 'params': [ { 'target': 'Param1', 'source': 'string', 'value': 'param1' }, { 'target': 'Param2', 'source': 'integer', 'value': 123 }, { 'target': 'Param3', 'source': 'boolean', 'value': True } ] }) params = create_request_parameters(None, request_model) self.assertEqual(params['Param1'], 'param1', 'Parameter not set from string constant') self.assertEqual(params['Param2'], 123, 'Parameter not set from integer constant') self.assertEqual(params['Param3'], True, 'Parameter not set from boolean constant')
async def async_call(self, parent, *args, **kwargs): """ Perform the action's request operation after building operation parameters and build any defined resources from the response. :type parent: :py:class:`~boto3.resources.base.ServiceResource` :param parent: The resource instance to which this action is attached. :rtype: dict or ServiceResource or list(ServiceResource) :return: The response, either as a raw dict or resource instance(s). """ operation_name = xform_name(self._action_model.request.operation) # First, build predefined params and then update with the # user-supplied kwargs, which allows overriding the pre-built # params if needed. params = create_request_parameters(parent, self._action_model.request) params.update(kwargs) logger.debug('Calling %s:%s with %r', parent.meta.service_name, operation_name, params) response = await getattr(parent.meta.client, operation_name)(**params) logger.debug('Response: %r', response) return self._response_handler(parent, params, response)
def test_service_action_params_data_member_missing(self): request_model = Request({ 'operation': 'GetFrobs', 'params': [ { 'target': 'WarehouseUrl', 'source': 'data', 'path': 'SomeMember' } ] }) parent = mock.Mock() def load_data(): parent.meta.data = { 'SomeMember': 'w-url' } parent.load.side_effect = load_data parent.meta = ResourceMeta('test') params = create_request_parameters(parent, request_model) parent.load.assert_called_with() self.assertEqual(params['WarehouseUrl'], 'w-url', 'Parameter not set from resource property')
def _get_paginated_results(self, limit, marker, collection): """ If a Boto Paginator is available, use it. The results are converted back into BotoResources by directly accessing protected members of ResourceCollection. This logic can be removed depending on issue: https://github.com/boto/boto3/issues/1268. """ # pylint:disable=protected-access cleaned_params = collection._params.copy() cleaned_params.pop('limit', None) cleaned_params.pop('page_size', None) # pylint:disable=protected-access params = create_request_parameters(collection._parent, collection._model.request) merge_dicts(params, cleaned_params, append_lists=True) client = self.boto_conn.meta.client list_op = self._get_list_operation() paginator = client.get_paginator(list_op) PaginationConfig = {} if limit: PaginationConfig = {'MaxItems': limit, 'PageSize': limit} if marker: PaginationConfig.update({'StartingToken': marker}) params.update({'PaginationConfig': PaginationConfig}) args = trim_empty_params(params) pages = paginator.paginate(**args) # resume_token is not populated unless the iterator is used items = pages.build_full_result() boto_objs = self._to_boto_resource(collection, args, items) resume_token = pages.resume_token return (resume_token, boto_objs)
def test_service_action_params_constants(self): request_model = Request({ 'operation': 'GetFrobs', 'params': [ { 'target': 'Param1', 'source': 'string', 'value': 'param1', }, { 'target': 'Param2', 'source': 'integer', 'value': 123 }, { 'target': 'Param3', 'source': 'boolean', 'value': True }, ], }) params = create_request_parameters(None, request_model) assert params['Param1'] == 'param1' assert params['Param2'] == 123 assert params['Param3'] is True
def test_service_action_params_constants(self): request_model = Request({ 'operation': 'GetFrobs', 'params': [{ 'target': 'Param1', 'source': 'string', 'value': 'param1' }, { 'target': 'Param2', 'source': 'integer', 'value': 123 }, { 'target': 'Param3', 'source': 'boolean', 'value': True }] }) params = create_request_parameters(None, request_model) self.assertEqual(params['Param1'], 'param1', 'Parameter not set from string constant') self.assertEqual(params['Param2'], 123, 'Parameter not set from integer constant') self.assertEqual(params['Param3'], True, 'Parameter not set from boolean constant')
def test_service_action_params_input(self): request_model = Request({ 'operation': 'GetFrobs', 'params': [{ 'target': 'Param1', 'source': 'input' }], }) params = create_request_parameters(None, request_model) assert params == {} params['param1'] = 'myinput' params = create_request_parameters(None, request_model, params=params) assert params == {'param1': 'myinput'}
def test_service_action_params_data_member_missing_no_load(self): request_model = Request({ 'operation': 'GetFrobs', 'params': [{ 'target': 'WarehouseUrl', 'source': 'data', 'path': 'SomeMember', }], }) # This mock has no ``load`` method. parent = mock.Mock(spec=ServiceResource) parent.meta = ResourceMeta('test', data=None) with pytest.raises(ResourceLoadException): create_request_parameters(parent, request_model)
async def pages(self): client = self._parent.meta.client cleaned_params = self._params.copy() limit = cleaned_params.pop('limit', None) page_size = cleaned_params.pop('page_size', None) params = create_request_parameters( self._parent, self._model.request) merge_dicts(params, cleaned_params, append_lists=True) # Is this a paginated operation? If so, we need to get an # iterator for the various pages. If not, then we simply # call the operation and return the result as a single # page in a list. For non-paginated results, we just ignore # the page size parameter. if client.can_paginate(self._py_operation_name): logger.debug('Calling paginated %s:%s with %r', self._parent.meta.service_name, self._py_operation_name, params) paginator = client.get_paginator(self._py_operation_name) pages = paginator.paginate( PaginationConfig={ 'MaxItems': limit, 'PageSize': page_size}, **params) else: @async_generator async def _aiopaginatordummy(): res = await getattr(client, self._py_operation_name)(**params) await yield_(res) logger.debug('Calling %s:%s with %r', self._parent.meta.service_name, self._py_operation_name, params) pages = _aiopaginatordummy() # Now that we have a page iterator or single page of results # we start processing and yielding individual items. count = 0 async for page in pages: page_items = [] for item in self._handler(self._parent, params, page): page_items.append(item) # If the limit is set and has been reached, then # we stop processing items here. count += 1 if limit is not None and count >= limit: break await yield_(page_items) # Stop reading pages if we've reached out limit if limit is not None and count >= limit: break
def test_service_action_params_list(self): request_model = Request({ 'operation': 'GetFrobs', 'params': [{ 'target': 'WarehouseUrls[0]', 'source': 'string', 'value': 'w-url', }], }) params = create_request_parameters(None, request_model) assert isinstance(params['WarehouseUrls'], list) assert len(params['WarehouseUrls']) == 1 assert 'w-url' in params['WarehouseUrls']
def test_service_action_params_data_member(self): request_model = Request({ 'operation': 'GetFrobs', 'params': [{ 'target': 'WarehouseUrl', 'source': 'data', 'path': 'SomeMember', }], }) parent = mock.Mock() parent.meta = ResourceMeta('test', data={'SomeMember': 'w-url'}) params = create_request_parameters(parent, request_model) assert params['WarehouseUrl'] == 'w-url'
async def __call__(self, parent, *args, **kwargs): operation_name = xform_name(self._action_model.request.operation) # First, build predefined params and then update with the # user-supplied kwargs, which allows overriding the pre-built # params if needed. params = create_request_parameters(parent, self._action_model.request) params.update(kwargs) logger.debug('Calling %s:%s with %r', parent.meta.service_name, operation_name, params) response = await getattr(parent.meta.client, operation_name)(**params) logger.debug('Response: %r', response) return await self._response_handler(parent, params, response)
def test_service_action_params_identifier(self): request_model = Request({ 'operation': 'GetFrobs', 'params': [{ 'target': 'WarehouseUrl', 'source': 'identifier', 'name': 'Url', }], }) parent = mock.Mock() parent.url = 'w-url' params = create_request_parameters(parent, request_model) assert params['WarehouseUrl'] == 'w-url'
def test_service_action_params_data_member(self): request_model = Request({ 'operation': 'GetFrobs', 'params': [{ 'target': 'WarehouseUrl', 'source': 'data', 'path': 'SomeMember' }] }) parent = mock.Mock() parent.meta = ResourceMeta('test', data={'SomeMember': 'w-url'}) params = create_request_parameters(parent, request_model) self.assertEqual(params['WarehouseUrl'], 'w-url', 'Parameter not set from resource property')
def test_service_action_params_identifier(self): request_model = Request({ 'operation': 'GetFrobs', 'params': [{ 'target': 'WarehouseUrl', 'source': 'identifier', 'name': 'Url' }] }) parent = mock.Mock() parent.url = 'w-url' params = create_request_parameters(parent, request_model) self.assertEqual(params['WarehouseUrl'], 'w-url', 'Parameter not set from resource identifier')
def test_service_action_params_data_member_missing_no_load(self): request_model = Request({ 'operation': 'GetFrobs', 'params': [ { 'target': 'WarehouseUrl', 'source': 'data', 'path': 'SomeMember' } ] }) # This mock has no ``load`` method. parent = mock.Mock(spec=ServiceResource) parent.meta = ResourceMeta('test', data=None) with self.assertRaises(ResourceLoadException): params = create_request_parameters(parent, request_model)
def test_service_action_params_data_member(self): request_model = Request({ 'operation': 'GetFrobs', 'params': [ { 'target': 'WarehouseUrl', 'sourceType': 'dataMember', 'source': 'some_member' } ] }) parent = mock.Mock() parent.some_member = 'w-url' params = create_request_parameters(parent, request_model) self.assertEqual(params['WarehouseUrl'], 'w-url', 'Parameter not set from resource property')
def test_service_action_params_list(self): request_model = Request({ 'operation': 'GetFrobs', 'params': [{ 'target': 'WarehouseUrls[0]', 'source': 'string', 'value': 'w-url' }] }) params = create_request_parameters(None, request_model) self.assertIsInstance(params['WarehouseUrls'], list, 'Parameter did not create a list') self.assertEqual(len(params['WarehouseUrls']), 1, 'Parameter list should only have a single item') self.assertIn('w-url', params['WarehouseUrls'], 'Parameter not in expected list')
def test_service_action_params_identifier(self): request_model = Request({ 'operation': 'GetFrobs', 'params': [ { 'target': 'WarehouseUrl', 'source': 'identifier', 'name': 'Url' } ] }) parent = mock.Mock() parent.url = 'w-url' params = create_request_parameters(parent, request_model) self.assertEqual(params['WarehouseUrl'], 'w-url', 'Parameter not set from resource identifier')
def test_service_action_params_identifier(self): action_def = { 'request': { 'operation': 'GetFrobs', 'params': [{ 'target': 'WarehouseUrl', 'sourceType': 'identifier', 'source': 'Url' }] } } parent = mock.Mock() parent.url = 'w-url' params = create_request_parameters(parent, action_def['request']) self.assertEqual(params['WarehouseUrl'], 'w-url', 'Parameter not set from resource identifier')
def test_service_action_params_data_member(self): action_def = { 'request': { 'operation': 'GetFrobs', 'params': [{ 'target': 'WarehouseUrl', 'sourceType': 'dataMember', 'source': 'some_member' }] } } parent = mock.Mock() parent.some_member = 'w-url' params = create_request_parameters(parent, action_def['request']) self.assertEqual(params['WarehouseUrl'], 'w-url', 'Parameter not set from resource property')
def test_service_action_params_list(self): request_model = Request({ 'operation': 'GetFrobs', 'params': [ { 'target': 'WarehouseUrls[0]', 'source': 'string', 'value': 'w-url' } ] }) params = create_request_parameters(None, request_model) self.assertIsInstance(params['WarehouseUrls'], list, 'Parameter did not create a list') self.assertEqual(len(params['WarehouseUrls']), 1, 'Parameter list should only have a single item') self.assertIn('w-url', params['WarehouseUrls'], 'Parameter not in expected list')
def load(self, *args) -> None: if self.meta.resource_model.load: identifiers = create_request_parameters( self, self.meta.resource_model.load.request) elif self.service_name in "s3" and self.resource_type in "bucket": # This is hardcoded in boto3 believe it or not! # https://github.com/boto/boto3/blob/master/boto3/s3/inject.py#L57 identifiers = {"BucketName": self.name} else: raise UnsupportedResourceTypeError( f"{self.service_name} {self.resource_type} does not have a load definition in its resources-1.json" ) has_non_empty_values = any(x for x in identifiers.values()) if not has_non_empty_values: logger.debug( "Load is a noop on this %s %s because we are an empty_resource=True resource", self.service_name, self.resource_type, ) return original_class_load(self)
def test_service_action_params_data_member_missing(self): request_model = Request({ 'operation': 'GetFrobs', 'params': [{ 'target': 'WarehouseUrl', 'source': 'data', 'path': 'SomeMember', }], }) parent = mock.Mock() def load_data(): parent.meta.data = {'SomeMember': 'w-url'} parent.load.side_effect = load_data parent.meta = ResourceMeta('test') params = create_request_parameters(parent, request_model) parent.load.assert_called_with() assert params['WarehouseUrl'] == 'w-url'
async def __call__(self, parent, *args, **kwargs): """ Perform the wait operation after building operation parameters. :type parent: :py:class:`~boto3.resources.base.ServiceResource` :param parent: The resource instance to which this action is attached. """ client_waiter_name = xform_name(self._waiter_model.waiter_name) # First, build predefined params and then update with the # user-supplied kwargs, which allows overriding the pre-built # params if needed. params = create_request_parameters(parent, self._waiter_model) params.update(kwargs) logger.debug('Calling %s:%s with %r', parent.meta.service_name, self._waiter_resource_name, params) client = parent.meta.client waiter = client.get_waiter(client_waiter_name) response = await waiter.wait(**params) logger.debug('Response: %r', response)