Example #1
0
    def test_service_action_params_reuse(self):
        request_model = Request({
            'operation':
            'GetFrobs',
            'params': [{
                'target': 'Delete.Objects[].Key',
                'source': 'data',
                'path': 'Key',
            }],
        })

        item1 = mock.Mock()
        item1.meta = ResourceMeta('test', data={'Key': 'item1'})

        item2 = mock.Mock()
        item2.meta = ResourceMeta('test', data={'Key': 'item2'})

        # Here we create params and then re-use it to build up a more
        # complex structure over multiple calls.
        params = create_request_parameters(item1, request_model)
        create_request_parameters(item2, request_model, params=params)

        assert params == {
            'Delete': {
                'Objects': [{
                    'Key': 'item1'
                }, {
                    'Key': 'item2'
                }]
            }
        }
Example #2
0
    def test_service_action_params_reuse(self):
        request_model = Request({
            'operation': 'GetFrobs',
            'params': [
                {
                    'target': 'Delete.Objects[].Key',
                    'source': 'data',
                    'path': 'Key'
                }
            ]
        })

        item1 = mock.Mock()
        item1.meta = ResourceMeta('test', data={
            'Key': 'item1'
        })

        item2 = mock.Mock()
        item2.meta = ResourceMeta('test', data={
            'Key': 'item2'
        })

        # Here we create params and then re-use it to build up a more
        # complex structure over multiple calls.
        params = create_request_parameters(item1, request_model)
        create_request_parameters(item2, request_model, params=params)

        self.assertEqual(params, {
            'Delete': {
                'Objects': [
                    {'Key': 'item1'},
                    {'Key': 'item2'}
                ]
            }
        })
Example #3
0
    def test_service_action_params_invalid(self):
        request_model = Request({
            'operation':
            'GetFrobs',
            'params': [{
                'target': 'Param1',
                'source': 'invalid'
            }],
        })

        with pytest.raises(NotImplementedError):
            create_request_parameters(None, request_model)
Example #4
0
    async def __call__(self, parent, *args, **kwargs):
        """
        Perform the batch action's operation on every page of results
        from the collection.

        :type parent:
            :py:class:`~boto3.resources.collection.ResourceCollection`
        :param parent: The collection iterator to which this action
                       is attached.
        :rtype: list(dict)
        :return: A list of low-level response dicts from each call.
        """
        service_name = None
        client = None
        responses = []
        operation_name = xform_name(self._action_model.request.operation)

        # Unlike the simple action above, a batch action must operate
        # on batches (or pages) of items. So we get each page, construct
        # the necessary parameters and call the batch operation.
        async for page in parent.pages():
            params = {}
            for index, resource in enumerate(page):
                # There is no public interface to get a service name
                # or low-level client from a collection, so we get
                # these from the first resource in the collection.
                if service_name is None:
                    service_name = resource.meta.service_name
                if client is None:
                    client = resource.meta.client

                create_request_parameters(resource,
                                          self._action_model.request,
                                          params=params,
                                          index=index)

            if not params:
                # There are no items, no need to make a call.
                break

            params.update(kwargs)

            logger.debug('Calling %s:%s with %r', service_name, operation_name,
                         params)

            response = await (getattr(client, operation_name)(**params))

            logger.debug('Response: %r', response)

            responses.append(self._response_handler(parent, params, response))

        return responses
Example #5
0
    def test_service_action_params_invalid(self):
        request_model = Request({
            'operation': 'GetFrobs',
            'params': [
                {
                    'target': 'Param1',
                    'source': 'invalid'
                }
            ]
        })

        with self.assertRaises(NotImplementedError):
            create_request_parameters(None, request_model)
Example #6
0
    def test_service_action_params_input(self):
        request_model = Request({
            'operation': 'GetFrobs',
            'params': [
                {'target': 'Param1', 'source': 'input'}
            ]
        })

        params = create_request_parameters(None, request_model)
        self.assertEqual(params, {})

        params['param1'] = 'myinput'
        params = create_request_parameters(None, request_model, params=params)
        self.assertEqual(params, {'param1': 'myinput'})
Example #7
0
    def test_service_action_params_invalid(self):
        action_def = {
            'request': {
                'operation':
                'GetFrobs',
                'params': [{
                    'target': 'Param1',
                    'sourceType': 'invalid',
                    'source': 'param1'
                }]
            }
        }

        with self.assertRaises(NotImplementedError):
            create_request_parameters(None, action_def['request'])
Example #8
0
    def test_service_action_params_constants(self):
        request_model = Request({
            'operation': 'GetFrobs',
            'params': [
                {
                    'target': 'Param1',
                    'source': 'string',
                    'value': 'param1'
                },
                {
                    'target': 'Param2',
                    'source': 'integer',
                    'value': 123
                },
                {
                    'target': 'Param3',
                    'source': 'boolean',
                    'value': True
                }
            ]
        })

        params = create_request_parameters(None, request_model)

        self.assertEqual(params['Param1'], 'param1',
            'Parameter not set from string constant')
        self.assertEqual(params['Param2'], 123,
            'Parameter not set from integer constant')
        self.assertEqual(params['Param3'], True,
            'Parameter not set from boolean constant')
Example #9
0
    async def async_call(self, parent, *args, **kwargs):
        """
        Perform the action's request operation after building operation
        parameters and build any defined resources from the response.

        :type parent: :py:class:`~boto3.resources.base.ServiceResource`
        :param parent: The resource instance to which this action is attached.
        :rtype: dict or ServiceResource or list(ServiceResource)
        :return: The response, either as a raw dict or resource instance(s).
        """
        operation_name = xform_name(self._action_model.request.operation)

        # First, build predefined params and then update with the
        # user-supplied kwargs, which allows overriding the pre-built
        # params if needed.
        params = create_request_parameters(parent, self._action_model.request)
        params.update(kwargs)

        logger.debug('Calling %s:%s with %r', parent.meta.service_name,
                     operation_name, params)

        response = await getattr(parent.meta.client, operation_name)(**params)

        logger.debug('Response: %r', response)

        return self._response_handler(parent, params, response)
Example #10
0
    def test_service_action_params_data_member_missing(self):
        request_model = Request({
            'operation': 'GetFrobs',
            'params': [
                {
                    'target': 'WarehouseUrl',
                    'source': 'data',
                    'path': 'SomeMember'
                }
            ]
        })

        parent = mock.Mock()

        def load_data():
            parent.meta.data = {
                'SomeMember': 'w-url'
            }

        parent.load.side_effect = load_data
        parent.meta = ResourceMeta('test')

        params = create_request_parameters(parent, request_model)

        parent.load.assert_called_with()
        self.assertEqual(params['WarehouseUrl'], 'w-url',
            'Parameter not set from resource property')
Example #11
0
    def _get_paginated_results(self, limit, marker, collection):
        """
        If a Boto Paginator is available, use it. The results
        are converted back into BotoResources by directly accessing
        protected members of ResourceCollection. This logic can be removed
        depending on issue: https://github.com/boto/boto3/issues/1268.
        """
        # pylint:disable=protected-access
        cleaned_params = collection._params.copy()
        cleaned_params.pop('limit', None)
        cleaned_params.pop('page_size', None)
        # pylint:disable=protected-access
        params = create_request_parameters(collection._parent,
                                           collection._model.request)
        merge_dicts(params, cleaned_params, append_lists=True)

        client = self.boto_conn.meta.client
        list_op = self._get_list_operation()
        paginator = client.get_paginator(list_op)
        PaginationConfig = {}
        if limit:
            PaginationConfig = {'MaxItems': limit, 'PageSize': limit}

        if marker:
            PaginationConfig.update({'StartingToken': marker})

        params.update({'PaginationConfig': PaginationConfig})
        args = trim_empty_params(params)
        pages = paginator.paginate(**args)
        # resume_token is not populated unless the iterator is used
        items = pages.build_full_result()

        boto_objs = self._to_boto_resource(collection, args, items)
        resume_token = pages.resume_token
        return (resume_token, boto_objs)
Example #12
0
    def test_service_action_params_constants(self):
        request_model = Request({
            'operation':
            'GetFrobs',
            'params': [
                {
                    'target': 'Param1',
                    'source': 'string',
                    'value': 'param1',
                },
                {
                    'target': 'Param2',
                    'source': 'integer',
                    'value': 123
                },
                {
                    'target': 'Param3',
                    'source': 'boolean',
                    'value': True
                },
            ],
        })

        params = create_request_parameters(None, request_model)

        assert params['Param1'] == 'param1'
        assert params['Param2'] == 123
        assert params['Param3'] is True
Example #13
0
    def test_service_action_params_constants(self):
        request_model = Request({
            'operation':
            'GetFrobs',
            'params': [{
                'target': 'Param1',
                'source': 'string',
                'value': 'param1'
            }, {
                'target': 'Param2',
                'source': 'integer',
                'value': 123
            }, {
                'target': 'Param3',
                'source': 'boolean',
                'value': True
            }]
        })

        params = create_request_parameters(None, request_model)

        self.assertEqual(params['Param1'], 'param1',
                         'Parameter not set from string constant')
        self.assertEqual(params['Param2'], 123,
                         'Parameter not set from integer constant')
        self.assertEqual(params['Param3'], True,
                         'Parameter not set from boolean constant')
Example #14
0
    def test_service_action_params_input(self):
        request_model = Request({
            'operation':
            'GetFrobs',
            'params': [{
                'target': 'Param1',
                'source': 'input'
            }],
        })

        params = create_request_parameters(None, request_model)
        assert params == {}

        params['param1'] = 'myinput'
        params = create_request_parameters(None, request_model, params=params)
        assert params == {'param1': 'myinput'}
Example #15
0
    def test_service_action_params_data_member_missing_no_load(self):
        request_model = Request({
            'operation':
            'GetFrobs',
            'params': [{
                'target': 'WarehouseUrl',
                'source': 'data',
                'path': 'SomeMember',
            }],
        })

        # This mock has no ``load`` method.
        parent = mock.Mock(spec=ServiceResource)
        parent.meta = ResourceMeta('test', data=None)

        with pytest.raises(ResourceLoadException):
            create_request_parameters(parent, request_model)
Example #16
0
    async def pages(self):
        client = self._parent.meta.client
        cleaned_params = self._params.copy()
        limit = cleaned_params.pop('limit', None)
        page_size = cleaned_params.pop('page_size', None)
        params = create_request_parameters(
            self._parent, self._model.request)
        merge_dicts(params, cleaned_params, append_lists=True)

        # Is this a paginated operation? If so, we need to get an
        # iterator for the various pages. If not, then we simply
        # call the operation and return the result as a single
        # page in a list. For non-paginated results, we just ignore
        # the page size parameter.
        if client.can_paginate(self._py_operation_name):
            logger.debug('Calling paginated %s:%s with %r',
                         self._parent.meta.service_name,
                         self._py_operation_name, params)
            paginator = client.get_paginator(self._py_operation_name)
            pages = paginator.paginate(
                PaginationConfig={
                    'MaxItems': limit, 'PageSize': page_size}, **params)
        else:
            @async_generator
            async def _aiopaginatordummy():
                res = await getattr(client, self._py_operation_name)(**params)
                await yield_(res)

            logger.debug('Calling %s:%s with %r',
                         self._parent.meta.service_name,
                         self._py_operation_name, params)
            pages = _aiopaginatordummy()

        # Now that we have a page iterator or single page of results
        # we start processing and yielding individual items.
        count = 0
        async for page in pages:
            page_items = []
            for item in self._handler(self._parent, params, page):
                page_items.append(item)

                # If the limit is set and has been reached, then
                # we stop processing items here.
                count += 1
                if limit is not None and count >= limit:
                    break

            await yield_(page_items)

            # Stop reading pages if we've reached out limit
            if limit is not None and count >= limit:
                break
Example #17
0
    def test_service_action_params_list(self):
        request_model = Request({
            'operation':
            'GetFrobs',
            'params': [{
                'target': 'WarehouseUrls[0]',
                'source': 'string',
                'value': 'w-url',
            }],
        })

        params = create_request_parameters(None, request_model)

        assert isinstance(params['WarehouseUrls'], list)
        assert len(params['WarehouseUrls']) == 1
        assert 'w-url' in params['WarehouseUrls']
Example #18
0
    def test_service_action_params_data_member(self):
        request_model = Request({
            'operation':
            'GetFrobs',
            'params': [{
                'target': 'WarehouseUrl',
                'source': 'data',
                'path': 'SomeMember',
            }],
        })

        parent = mock.Mock()
        parent.meta = ResourceMeta('test', data={'SomeMember': 'w-url'})

        params = create_request_parameters(parent, request_model)

        assert params['WarehouseUrl'] == 'w-url'
Example #19
0
    async def __call__(self, parent, *args, **kwargs):
        operation_name = xform_name(self._action_model.request.operation)

        # First, build predefined params and then update with the
        # user-supplied kwargs, which allows overriding the pre-built
        # params if needed.
        params = create_request_parameters(parent, self._action_model.request)
        params.update(kwargs)

        logger.debug('Calling %s:%s with %r', parent.meta.service_name,
                     operation_name, params)

        response = await getattr(parent.meta.client, operation_name)(**params)

        logger.debug('Response: %r', response)

        return await self._response_handler(parent, params, response)
Example #20
0
    def test_service_action_params_identifier(self):
        request_model = Request({
            'operation':
            'GetFrobs',
            'params': [{
                'target': 'WarehouseUrl',
                'source': 'identifier',
                'name': 'Url',
            }],
        })

        parent = mock.Mock()
        parent.url = 'w-url'

        params = create_request_parameters(parent, request_model)

        assert params['WarehouseUrl'] == 'w-url'
Example #21
0
    def test_service_action_params_data_member(self):
        request_model = Request({
            'operation':
            'GetFrobs',
            'params': [{
                'target': 'WarehouseUrl',
                'source': 'data',
                'path': 'SomeMember'
            }]
        })

        parent = mock.Mock()
        parent.meta = ResourceMeta('test', data={'SomeMember': 'w-url'})

        params = create_request_parameters(parent, request_model)

        self.assertEqual(params['WarehouseUrl'], 'w-url',
                         'Parameter not set from resource property')
Example #22
0
    def test_service_action_params_identifier(self):
        request_model = Request({
            'operation':
            'GetFrobs',
            'params': [{
                'target': 'WarehouseUrl',
                'source': 'identifier',
                'name': 'Url'
            }]
        })

        parent = mock.Mock()
        parent.url = 'w-url'

        params = create_request_parameters(parent, request_model)

        self.assertEqual(params['WarehouseUrl'], 'w-url',
                         'Parameter not set from resource identifier')
Example #23
0
    def test_service_action_params_data_member_missing_no_load(self):
        request_model = Request({
            'operation': 'GetFrobs',
            'params': [
                {
                    'target': 'WarehouseUrl',
                    'source': 'data',
                    'path': 'SomeMember'
                }
            ]
        })

        # This mock has no ``load`` method.
        parent = mock.Mock(spec=ServiceResource)
        parent.meta = ResourceMeta('test', data=None)

        with self.assertRaises(ResourceLoadException):
            params = create_request_parameters(parent, request_model)
Example #24
0
    def test_service_action_params_data_member(self):
        request_model = Request({
            'operation': 'GetFrobs',
            'params': [
                {
                    'target': 'WarehouseUrl',
                    'sourceType': 'dataMember',
                    'source': 'some_member'
                }
            ]
        })

        parent = mock.Mock()
        parent.some_member = 'w-url'

        params = create_request_parameters(parent, request_model)

        self.assertEqual(params['WarehouseUrl'], 'w-url',
            'Parameter not set from resource property')
Example #25
0
    def test_service_action_params_list(self):
        request_model = Request({
            'operation':
            'GetFrobs',
            'params': [{
                'target': 'WarehouseUrls[0]',
                'source': 'string',
                'value': 'w-url'
            }]
        })

        params = create_request_parameters(None, request_model)

        self.assertIsInstance(params['WarehouseUrls'], list,
                              'Parameter did not create a list')
        self.assertEqual(len(params['WarehouseUrls']), 1,
                         'Parameter list should only have a single item')
        self.assertIn('w-url', params['WarehouseUrls'],
                      'Parameter not in expected list')
Example #26
0
    def test_service_action_params_identifier(self):
        request_model = Request({
            'operation': 'GetFrobs',
            'params': [
                {
                    'target': 'WarehouseUrl',
                    'source': 'identifier',
                    'name': 'Url'
                }
            ]
        })

        parent = mock.Mock()
        parent.url = 'w-url'

        params = create_request_parameters(parent, request_model)

        self.assertEqual(params['WarehouseUrl'], 'w-url',
            'Parameter not set from resource identifier')
Example #27
0
    def test_service_action_params_identifier(self):
        action_def = {
            'request': {
                'operation':
                'GetFrobs',
                'params': [{
                    'target': 'WarehouseUrl',
                    'sourceType': 'identifier',
                    'source': 'Url'
                }]
            }
        }

        parent = mock.Mock()
        parent.url = 'w-url'

        params = create_request_parameters(parent, action_def['request'])

        self.assertEqual(params['WarehouseUrl'], 'w-url',
                         'Parameter not set from resource identifier')
Example #28
0
    def test_service_action_params_data_member(self):
        action_def = {
            'request': {
                'operation':
                'GetFrobs',
                'params': [{
                    'target': 'WarehouseUrl',
                    'sourceType': 'dataMember',
                    'source': 'some_member'
                }]
            }
        }

        parent = mock.Mock()
        parent.some_member = 'w-url'

        params = create_request_parameters(parent, action_def['request'])

        self.assertEqual(params['WarehouseUrl'], 'w-url',
                         'Parameter not set from resource property')
Example #29
0
    def test_service_action_params_list(self):
        request_model = Request({
            'operation': 'GetFrobs',
            'params': [
                {
                    'target': 'WarehouseUrls[0]',
                    'source': 'string',
                    'value': 'w-url'
                }
            ]
        })

        params = create_request_parameters(None, request_model)

        self.assertIsInstance(params['WarehouseUrls'], list,
            'Parameter did not create a list')
        self.assertEqual(len(params['WarehouseUrls']), 1,
            'Parameter list should only have a single item')
        self.assertIn('w-url', params['WarehouseUrls'],
            'Parameter not in expected list')
Example #30
0
        def load(self, *args) -> None:
            if self.meta.resource_model.load:
                identifiers = create_request_parameters(
                    self, self.meta.resource_model.load.request)
            elif self.service_name in "s3" and self.resource_type in "bucket":
                # This is hardcoded in boto3 believe it or not!
                # https://github.com/boto/boto3/blob/master/boto3/s3/inject.py#L57
                identifiers = {"BucketName": self.name}
            else:
                raise UnsupportedResourceTypeError(
                    f"{self.service_name} {self.resource_type} does not have a load definition in its resources-1.json"
                )

            has_non_empty_values = any(x for x in identifiers.values())
            if not has_non_empty_values:
                logger.debug(
                    "Load is a noop on this %s %s because we are an empty_resource=True resource",
                    self.service_name,
                    self.resource_type,
                )
                return
            original_class_load(self)
Example #31
0
    def test_service_action_params_data_member_missing(self):
        request_model = Request({
            'operation':
            'GetFrobs',
            'params': [{
                'target': 'WarehouseUrl',
                'source': 'data',
                'path': 'SomeMember',
            }],
        })

        parent = mock.Mock()

        def load_data():
            parent.meta.data = {'SomeMember': 'w-url'}

        parent.load.side_effect = load_data
        parent.meta = ResourceMeta('test')

        params = create_request_parameters(parent, request_model)

        parent.load.assert_called_with()
        assert params['WarehouseUrl'] == 'w-url'
Example #32
0
    async def __call__(self, parent, *args, **kwargs):
        """
        Perform the wait operation after building operation
        parameters.

        :type parent: :py:class:`~boto3.resources.base.ServiceResource`
        :param parent: The resource instance to which this action is attached.
        """
        client_waiter_name = xform_name(self._waiter_model.waiter_name)

        # First, build predefined params and then update with the
        # user-supplied kwargs, which allows overriding the pre-built
        # params if needed.
        params = create_request_parameters(parent, self._waiter_model)
        params.update(kwargs)

        logger.debug('Calling %s:%s with %r', parent.meta.service_name,
                     self._waiter_resource_name, params)

        client = parent.meta.client
        waiter = client.get_waiter(client_waiter_name)
        response = await waiter.wait(**params)

        logger.debug('Response: %r', response)