Example #1
0
    def parse_batch_write_request_items(cls, request_items_json):
        request_map = {}
        for table_name, request_list_json in request_items_json.iteritems():
            validation.validate_table_name(table_name)
            validation.validate_list_of_objects(request_list_json, table_name)

            request_list_for_table = []
            for request_json in request_list_json:
                for request_type, request_body in request_json.iteritems():
                    validation.validate_string(request_type, "request_type")
                    if request_type == Props.REQUEST_PUT:
                        validation.validate_object(request_body, request_type)
                        item = request_body.pop(Props.ITEM, None)
                        validation.validate_object(item, Props.ITEM)
                        validation.validate_unexpected_props(
                            request_body, request_type)
                        request_list_for_table.append(
                            WriteItemRequest.put(
                                cls.parse_item_attributes(item)))
                    elif request_type == Props.REQUEST_DELETE:
                        validation.validate_object(request_body, request_type)
                        key = request_body.pop(Props.KEY, None)
                        validation.validate_object(key, Props.KEY)
                        validation.validate_unexpected_props(
                            request_body, request_type)
                        request_list_for_table.append(
                            WriteItemRequest.delete(
                                cls.parse_item_attributes(key)))
                    else:
                        raise ValidationError(_(
                            "Unsupported request type found: "
                            "%(request_type)s"),
                                              request_type=request_type)
            request_map[table_name] = request_list_for_table
        return request_map
    def test_execute_write_batch_put_delete_same_item(
            self, mock_repo_get, mock_validate_table_is_active,
            mock_validate_table_schema):

        table_info = mock.Mock()
        table_info.schema.key_attributes = ['id', 'range']
        mock_repo_get.return_value = table_info

        context = mock.Mock(tenant='fake_tenant')

        table_name = 'fake_table'

        request_map = {
            table_name: [
                WriteItemRequest.put(
                    {
                        'id': models.AttributeValue('N', 1),
                        'range': models.AttributeValue('S', '1'),
                        'str': models.AttributeValue('S', 'str1'),
                    }
                ),
                WriteItemRequest.delete(
                    {
                        'id': models.AttributeValue('N', 1),
                        'range': models.AttributeValue('S', '1')
                    }
                )
            ]
        }

        storage_manager = SimpleStorageManager(StorageDriver(),
                                               TableInfoRepository())

        with self.assertRaises(ValidationError) as raises_cm:
            storage_manager.execute_write_batch(
                context, request_map
            )

        exception = raises_cm.exception
        self.assertIn("More than one", exception._error_string)
Example #3
0
    def parse_batch_write_request_items(cls, request_items_json):
        request_map = {}
        for table_name, request_list_json in request_items_json.iteritems():
            validation.validate_table_name(table_name)
            validation.validate_list_of_objects(request_list_json, table_name)

            request_list_for_table = []
            for request_json in request_list_json:
                for request_type, request_body in request_json.iteritems():
                    validation.validate_string(request_type, "request_type")
                    if request_type == Props.REQUEST_PUT:
                        validation.validate_object(request_body, request_type)
                        item = request_body.pop(Props.ITEM, None)
                        validation.validate_object(item, Props.ITEM)
                        validation.validate_unexpected_props(request_body,
                                                             request_type)
                        request_list_for_table.append(
                            WriteItemRequest.put(
                                cls.parse_item_attributes(item)
                            )
                        )
                    elif request_type == Props.REQUEST_DELETE:
                        validation.validate_object(request_body, request_type)
                        key = request_body.pop(Props.KEY, None)
                        validation.validate_object(key, Props.KEY)
                        validation.validate_unexpected_props(request_body,
                                                             request_type)
                        request_list_for_table.append(
                            WriteItemRequest.delete(
                                cls.parse_item_attributes(key)
                            )
                        )
                    else:
                        raise ValidationError(
                            _("Unsupported request type found: "
                              "%(request_type)s"),
                            request_type=request_type
                        )
            request_map[table_name] = request_list_for_table
        return request_map
    def test_execute_write_batch_put_delete_same_item(
            self, mock_repo_get, mock_validate_table_is_active,
            mock_validate_table_schema):

        table_info = mock.Mock()
        table_info.schema.key_attributes = ['id', 'range']
        mock_repo_get.return_value = table_info

        context = mock.Mock(tenant='fake_tenant')

        table_name = 'fake_table'

        request_map = {
            table_name: [
                WriteItemRequest.put({
                    'id': models.AttributeValue('N', 1),
                    'range': models.AttributeValue('S', '1'),
                    'str': models.AttributeValue('S', 'str1'),
                }),
                WriteItemRequest.delete({
                    'id':
                    models.AttributeValue('N', 1),
                    'range':
                    models.AttributeValue('S', '1')
                })
            ]
        }

        storage_manager = SimpleStorageManager(StorageDriver(),
                                               TableInfoRepository())

        with self.assertRaises(ValidationError) as raises_cm:
            storage_manager.execute_write_batch(context, request_map)

        exception = raises_cm.exception
        self.assertIn("More than one", exception._error_string)
    def test_execute_write_batch(self, mock_put_item, mock_delete_item,
                                 mock_repo_get, mock_validate_table_is_active,
                                 mock_validate_table_schema, mock_batch_write):
        future = Future()
        future.set_result(True)
        mock_put_item.return_value = future
        mock_delete_item.return_value = future
        mock_batch_write.side_effect = NotImplementedError()

        table_info = mock.Mock()
        table_info.schema.key_attributes = ['id', 'range']
        mock_repo_get.return_value = table_info

        context = mock.Mock(tenant='fake_tenant')

        table_name = 'fake_table'

        request_map = {
            table_name: [
                WriteItemRequest.put(
                    {
                        'id': models.AttributeValue('N', 1),
                        'range': models.AttributeValue('S', '1'),
                        'str': models.AttributeValue('S', 'str1'),
                    }
                ),
                WriteItemRequest.put(
                    {
                        'id': models.AttributeValue('N', 2),
                        'range': models.AttributeValue('S', '1'),
                        'str': models.AttributeValue('S', 'str1')
                    }
                ),
                WriteItemRequest.delete(
                    {
                        'id': models.AttributeValue('N', 3),
                        'range': models.AttributeValue('S', '3')
                    }
                )
            ]
        }

        expected_put = [
            mock.call(
                context, table_info,
                {
                    'id': models.AttributeValue('N', 1),
                    'range': models.AttributeValue('S', '1'),
                    'str': models.AttributeValue('S', 'str1')
                }
            ),
            mock.call(
                context, table_info,
                {
                    'id': models.AttributeValue('N', 2),
                    'range': models.AttributeValue('S', '1'),
                    'str': models.AttributeValue('S', 'str1')
                }
            ),
        ]
        expected_delete = [
            mock.call(
                context, table_info,
                {
                    'id': models.AttributeValue('N', 3),
                    'range': models.AttributeValue('S', '3')
                }
            )
        ]

        storage_manager = SimpleStorageManager(StorageDriver(),
                                               TableInfoRepository())

        unprocessed_items = storage_manager.execute_write_batch(
            context, request_map
        )

        self.assertEqual(expected_put, mock_put_item.call_args_list)
        self.assertEqual(expected_delete,
                         mock_delete_item.call_args_list)

        self.assertEqual(unprocessed_items, {})
    def test_execute_write_batch(self, mock_put_item, mock_delete_item,
                                 mock_repo_get, mock_validate_table_is_active,
                                 mock_validate_table_schema, mock_batch_write):
        future = Future()
        future.set_result(True)
        mock_put_item.return_value = future
        mock_delete_item.return_value = future
        mock_batch_write.side_effect = NotImplementedError()

        table_info = mock.Mock()
        table_info.schema.key_attributes = ['id', 'range']
        mock_repo_get.return_value = table_info

        context = mock.Mock(tenant='fake_tenant')

        table_name = 'fake_table'

        request_map = {
            table_name: [
                WriteItemRequest.put({
                    'id': models.AttributeValue('N', 1),
                    'range': models.AttributeValue('S', '1'),
                    'str': models.AttributeValue('S', 'str1'),
                }),
                WriteItemRequest.put({
                    'id': models.AttributeValue('N', 2),
                    'range': models.AttributeValue('S', '1'),
                    'str': models.AttributeValue('S', 'str1')
                }),
                WriteItemRequest.delete({
                    'id':
                    models.AttributeValue('N', 3),
                    'range':
                    models.AttributeValue('S', '3')
                })
            ]
        }

        expected_put = [
            mock.call(
                context, table_info, {
                    'id': models.AttributeValue('N', 1),
                    'range': models.AttributeValue('S', '1'),
                    'str': models.AttributeValue('S', 'str1')
                }),
            mock.call(
                context, table_info, {
                    'id': models.AttributeValue('N', 2),
                    'range': models.AttributeValue('S', '1'),
                    'str': models.AttributeValue('S', 'str1')
                }),
        ]
        expected_delete = [
            mock.call(
                context, table_info, {
                    'id': models.AttributeValue('N', 3),
                    'range': models.AttributeValue('S', '3')
                })
        ]

        storage_manager = SimpleStorageManager(StorageDriver(),
                                               TableInfoRepository())

        unprocessed_items = storage_manager.execute_write_batch(
            context, request_map)

        self.assertEqual(expected_put, mock_put_item.call_args_list)
        self.assertEqual(expected_delete, mock_delete_item.call_args_list)

        self.assertEqual(unprocessed_items, {})
    def test_notify_batch_write(self, mock_put_item, mock_delete_item,
                                mock_repo_get, mock_validate_table_is_active,
                                mock_validate_table_schema, mock_batch_write):
        self.cleanup_test_notifier()

        future = Future()
        future.set_result(True)
        mock_put_item.return_value = future
        mock_delete_item.return_value = future

        table_info = mock.Mock()
        table_info.schema.key_attributes = ['id', 'range']
        mock_repo_get.return_value = table_info

        mock_batch_write.side_effect = NotImplementedError()

        context = mock.Mock(tenant='fake_tenant')

        table_name = 'fake_table'

        request_map = {
            table_name: [
                WriteItemRequest.put(
                    {
                        'id': models.AttributeValue('N', 1),
                        'range': models.AttributeValue('S', '1'),
                        'str': models.AttributeValue('S', 'str1'),
                    }
                ),
                WriteItemRequest.put(
                    {
                        'id': models.AttributeValue('N', 2),
                        'range': models.AttributeValue('S', '1'),
                        'str': models.AttributeValue('S', 'str1')
                    }
                ),
                WriteItemRequest.delete(
                    {
                        'id': models.AttributeValue('N', 3),
                        'range': models.AttributeValue('S', '3')
                    }
                )
            ]
        }

        storage_manager = SimpleStorageManager(
            StorageDriver(), TableInfoRepository()
        )
        storage_manager.execute_write_batch(context, request_map)

        # check notification queue
        self.assertEqual(len(self.get_notifications()), 2)

        start_event = self.get_notifications()[0]
        end_event = self.get_notifications()[1]

        self.assertEqual(start_event['priority'], 'INFO')
        self.assertEqual(start_event['event_type'],
                         notifier.EVENT_TYPE_DATA_BATCHWRITE_START)
        self.assertEqual(len(start_event['payload']), len(request_map))

        self.assertEqual(end_event['priority'], 'INFO')
        self.assertEqual(end_event['event_type'],
                         notifier.EVENT_TYPE_DATA_BATCHWRITE_END)
        self.assertEqual(len(end_event['payload']['write_request_map']),
                         len(request_map))
        self.assertEqual(len(end_event['payload']['unprocessed_items']), 0)

        time_start = datetime.datetime.strptime(
            start_event['timestamp'], DATETIMEFORMAT)
        time_end = datetime.datetime.strptime(
            end_event['timestamp'], DATETIMEFORMAT)
        self.assertTrue(time_start < time_end,
                        "start event is later than end event")
Example #8
0
    def test_notify_batch_write(self, mock_put_item, mock_delete_item,
                                mock_repo_get, mock_validate_table_is_active,
                                mock_validate_table_schema, mock_batch_write):
        self.cleanup_test_notifier()

        future = Future()
        future.set_result(True)
        mock_put_item.return_value = future
        mock_delete_item.return_value = future

        table_info = mock.Mock()
        table_info.schema.key_attributes = ['id', 'range']
        mock_repo_get.return_value = table_info

        mock_batch_write.side_effect = NotImplementedError()

        context = mock.Mock(tenant='fake_tenant')

        table_name = 'fake_table'

        request_map = {
            table_name: [
                WriteItemRequest.put({
                    'id': models.AttributeValue('N', 1),
                    'range': models.AttributeValue('S', '1'),
                    'str': models.AttributeValue('S', 'str1'),
                }),
                WriteItemRequest.put({
                    'id': models.AttributeValue('N', 2),
                    'range': models.AttributeValue('S', '1'),
                    'str': models.AttributeValue('S', 'str1')
                }),
                WriteItemRequest.delete({
                    'id':
                    models.AttributeValue('N', 3),
                    'range':
                    models.AttributeValue('S', '3')
                })
            ]
        }

        storage_manager = SimpleStorageManager(StorageDriver(),
                                               TableInfoRepository())
        storage_manager.execute_write_batch(context, request_map)

        # check notification queue
        self.assertEqual(len(self.get_notifications()), 2)

        start_event = self.get_notifications()[0]
        end_event = self.get_notifications()[1]

        self.assertEqual(start_event['priority'], 'INFO')
        self.assertEqual(start_event['event_type'],
                         notifier.EVENT_TYPE_DATA_BATCHWRITE_START)
        self.assertEqual(len(start_event['payload']), len(request_map))

        self.assertEqual(end_event['priority'], 'INFO')
        self.assertEqual(end_event['event_type'],
                         notifier.EVENT_TYPE_DATA_BATCHWRITE_END)
        self.assertEqual(len(end_event['payload']['write_request_map']),
                         len(request_map))
        self.assertEqual(len(end_event['payload']['unprocessed_items']), 0)

        time_start = datetime.datetime.strptime(start_event['timestamp'],
                                                DATETIMEFORMAT)
        time_end = datetime.datetime.strptime(end_event['timestamp'],
                                              DATETIMEFORMAT)
        self.assertTrue(time_start < time_end,
                        "start event is later than end event")