def parse_batch_write_request_items(cls, request_items_json):
request_map = {}
for table_name, request_list_json in request_items_json.iteritems():
validation.validate_table_name(table_name)
validation.validate_list_of_objects(request_list_json, table_name)
request_list_for_table = []
for request_json in request_list_json:
for request_type, request_body in request_json.iteritems():
validation.validate_string(request_type, "request_type")
if request_type == Props.REQUEST_PUT:
validation.validate_object(request_body, request_type)
item = request_body.pop(Props.ITEM, None)
validation.validate_object(item, Props.ITEM)
validation.validate_unexpected_props(
request_body, request_type)
request_list_for_table.append(
WriteItemRequest.put(
cls.parse_item_attributes(item)))
elif request_type == Props.REQUEST_DELETE:
validation.validate_object(request_body, request_type)
key = request_body.pop(Props.KEY, None)
validation.validate_object(key, Props.KEY)
validation.validate_unexpected_props(
request_body, request_type)
request_list_for_table.append(
WriteItemRequest.delete(
cls.parse_item_attributes(key)))
else:
raise ValidationError(_(
"Unsupported request type found: "
"%(request_type)s"),
request_type=request_type)
request_map[table_name] = request_list_for_table
return request_map
def test_execute_write_batch_put_delete_same_item(
self, mock_repo_get, mock_validate_table_is_active,
mock_validate_table_schema):
table_info = mock.Mock()
table_info.schema.key_attributes = ['id', 'range']
mock_repo_get.return_value = table_info
context = mock.Mock(tenant='fake_tenant')
table_name = 'fake_table'
request_map = {
table_name: [
WriteItemRequest.put(
{
'id': models.AttributeValue('N', 1),
'range': models.AttributeValue('S', '1'),
'str': models.AttributeValue('S', 'str1'),
}
),
WriteItemRequest.delete(
{
'id': models.AttributeValue('N', 1),
'range': models.AttributeValue('S', '1')
}
)
]
}
storage_manager = SimpleStorageManager(StorageDriver(),
TableInfoRepository())
with self.assertRaises(ValidationError) as raises_cm:
storage_manager.execute_write_batch(
context, request_map
)
exception = raises_cm.exception
self.assertIn("More than one", exception._error_string)
def parse_batch_write_request_items(cls, request_items_json):
request_map = {}
for table_name, request_list_json in request_items_json.iteritems():
validation.validate_table_name(table_name)
validation.validate_list_of_objects(request_list_json, table_name)
request_list_for_table = []
for request_json in request_list_json:
for request_type, request_body in request_json.iteritems():
validation.validate_string(request_type, "request_type")
if request_type == Props.REQUEST_PUT:
validation.validate_object(request_body, request_type)
item = request_body.pop(Props.ITEM, None)
validation.validate_object(item, Props.ITEM)
validation.validate_unexpected_props(request_body,
request_type)
request_list_for_table.append(
WriteItemRequest.put(
cls.parse_item_attributes(item)
)
)
elif request_type == Props.REQUEST_DELETE:
validation.validate_object(request_body, request_type)
key = request_body.pop(Props.KEY, None)
validation.validate_object(key, Props.KEY)
validation.validate_unexpected_props(request_body,
request_type)
request_list_for_table.append(
WriteItemRequest.delete(
cls.parse_item_attributes(key)
)
)
else:
raise ValidationError(
_("Unsupported request type found: "
"%(request_type)s"),
request_type=request_type
)
request_map[table_name] = request_list_for_table
return request_map
def test_execute_write_batch_put_delete_same_item(
self, mock_repo_get, mock_validate_table_is_active,
mock_validate_table_schema):
table_info = mock.Mock()
table_info.schema.key_attributes = ['id', 'range']
mock_repo_get.return_value = table_info
context = mock.Mock(tenant='fake_tenant')
table_name = 'fake_table'
request_map = {
table_name: [
WriteItemRequest.put({
'id': models.AttributeValue('N', 1),
'range': models.AttributeValue('S', '1'),
'str': models.AttributeValue('S', 'str1'),
}),
WriteItemRequest.delete({
'id':
models.AttributeValue('N', 1),
'range':
models.AttributeValue('S', '1')
})
]
}
storage_manager = SimpleStorageManager(StorageDriver(),
TableInfoRepository())
with self.assertRaises(ValidationError) as raises_cm:
storage_manager.execute_write_batch(context, request_map)
exception = raises_cm.exception
self.assertIn("More than one", exception._error_string)
def test_execute_write_batch(self, mock_put_item, mock_delete_item,
mock_repo_get, mock_validate_table_is_active,
mock_validate_table_schema, mock_batch_write):
future = Future()
future.set_result(True)
mock_put_item.return_value = future
mock_delete_item.return_value = future
mock_batch_write.side_effect = NotImplementedError()
table_info = mock.Mock()
table_info.schema.key_attributes = ['id', 'range']
mock_repo_get.return_value = table_info
context = mock.Mock(tenant='fake_tenant')
table_name = 'fake_table'
request_map = {
table_name: [
WriteItemRequest.put(
{
'id': models.AttributeValue('N', 1),
'range': models.AttributeValue('S', '1'),
'str': models.AttributeValue('S', 'str1'),
}
),
WriteItemRequest.put(
{
'id': models.AttributeValue('N', 2),
'range': models.AttributeValue('S', '1'),
'str': models.AttributeValue('S', 'str1')
}
),
WriteItemRequest.delete(
{
'id': models.AttributeValue('N', 3),
'range': models.AttributeValue('S', '3')
}
)
]
}
expected_put = [
mock.call(
context, table_info,
{
'id': models.AttributeValue('N', 1),
'range': models.AttributeValue('S', '1'),
'str': models.AttributeValue('S', 'str1')
}
),
mock.call(
context, table_info,
{
'id': models.AttributeValue('N', 2),
'range': models.AttributeValue('S', '1'),
'str': models.AttributeValue('S', 'str1')
}
),
]
expected_delete = [
mock.call(
context, table_info,
{
'id': models.AttributeValue('N', 3),
'range': models.AttributeValue('S', '3')
}
)
]
storage_manager = SimpleStorageManager(StorageDriver(),
TableInfoRepository())
unprocessed_items = storage_manager.execute_write_batch(
context, request_map
)
self.assertEqual(expected_put, mock_put_item.call_args_list)
self.assertEqual(expected_delete,
mock_delete_item.call_args_list)
self.assertEqual(unprocessed_items, {})
def test_execute_write_batch(self, mock_put_item, mock_delete_item,
mock_repo_get, mock_validate_table_is_active,
mock_validate_table_schema, mock_batch_write):
future = Future()
future.set_result(True)
mock_put_item.return_value = future
mock_delete_item.return_value = future
mock_batch_write.side_effect = NotImplementedError()
table_info = mock.Mock()
table_info.schema.key_attributes = ['id', 'range']
mock_repo_get.return_value = table_info
context = mock.Mock(tenant='fake_tenant')
table_name = 'fake_table'
request_map = {
table_name: [
WriteItemRequest.put({
'id': models.AttributeValue('N', 1),
'range': models.AttributeValue('S', '1'),
'str': models.AttributeValue('S', 'str1'),
}),
WriteItemRequest.put({
'id': models.AttributeValue('N', 2),
'range': models.AttributeValue('S', '1'),
'str': models.AttributeValue('S', 'str1')
}),
WriteItemRequest.delete({
'id':
models.AttributeValue('N', 3),
'range':
models.AttributeValue('S', '3')
})
]
}
expected_put = [
mock.call(
context, table_info, {
'id': models.AttributeValue('N', 1),
'range': models.AttributeValue('S', '1'),
'str': models.AttributeValue('S', 'str1')
}),
mock.call(
context, table_info, {
'id': models.AttributeValue('N', 2),
'range': models.AttributeValue('S', '1'),
'str': models.AttributeValue('S', 'str1')
}),
]
expected_delete = [
mock.call(
context, table_info, {
'id': models.AttributeValue('N', 3),
'range': models.AttributeValue('S', '3')
})
]
storage_manager = SimpleStorageManager(StorageDriver(),
TableInfoRepository())
unprocessed_items = storage_manager.execute_write_batch(
context, request_map)
self.assertEqual(expected_put, mock_put_item.call_args_list)
self.assertEqual(expected_delete, mock_delete_item.call_args_list)
self.assertEqual(unprocessed_items, {})
def test_notify_batch_write(self, mock_put_item, mock_delete_item,
mock_repo_get, mock_validate_table_is_active,
mock_validate_table_schema, mock_batch_write):
self.cleanup_test_notifier()
future = Future()
future.set_result(True)
mock_put_item.return_value = future
mock_delete_item.return_value = future
table_info = mock.Mock()
table_info.schema.key_attributes = ['id', 'range']
mock_repo_get.return_value = table_info
mock_batch_write.side_effect = NotImplementedError()
context = mock.Mock(tenant='fake_tenant')
table_name = 'fake_table'
request_map = {
table_name: [
WriteItemRequest.put(
{
'id': models.AttributeValue('N', 1),
'range': models.AttributeValue('S', '1'),
'str': models.AttributeValue('S', 'str1'),
}
),
WriteItemRequest.put(
{
'id': models.AttributeValue('N', 2),
'range': models.AttributeValue('S', '1'),
'str': models.AttributeValue('S', 'str1')
}
),
WriteItemRequest.delete(
{
'id': models.AttributeValue('N', 3),
'range': models.AttributeValue('S', '3')
}
)
]
}
storage_manager = SimpleStorageManager(
StorageDriver(), TableInfoRepository()
)
storage_manager.execute_write_batch(context, request_map)
# check notification queue
self.assertEqual(len(self.get_notifications()), 2)
start_event = self.get_notifications()[0]
end_event = self.get_notifications()[1]
self.assertEqual(start_event['priority'], 'INFO')
self.assertEqual(start_event['event_type'],
notifier.EVENT_TYPE_DATA_BATCHWRITE_START)
self.assertEqual(len(start_event['payload']), len(request_map))
self.assertEqual(end_event['priority'], 'INFO')
self.assertEqual(end_event['event_type'],
notifier.EVENT_TYPE_DATA_BATCHWRITE_END)
self.assertEqual(len(end_event['payload']['write_request_map']),
len(request_map))
self.assertEqual(len(end_event['payload']['unprocessed_items']), 0)
time_start = datetime.datetime.strptime(
start_event['timestamp'], DATETIMEFORMAT)
time_end = datetime.datetime.strptime(
end_event['timestamp'], DATETIMEFORMAT)
self.assertTrue(time_start < time_end,
"start event is later than end event")
def test_notify_batch_write(self, mock_put_item, mock_delete_item,
mock_repo_get, mock_validate_table_is_active,
mock_validate_table_schema, mock_batch_write):
self.cleanup_test_notifier()
future = Future()
future.set_result(True)
mock_put_item.return_value = future
mock_delete_item.return_value = future
table_info = mock.Mock()
table_info.schema.key_attributes = ['id', 'range']
mock_repo_get.return_value = table_info
mock_batch_write.side_effect = NotImplementedError()
context = mock.Mock(tenant='fake_tenant')
table_name = 'fake_table'
request_map = {
table_name: [
WriteItemRequest.put({
'id': models.AttributeValue('N', 1),
'range': models.AttributeValue('S', '1'),
'str': models.AttributeValue('S', 'str1'),
}),
WriteItemRequest.put({
'id': models.AttributeValue('N', 2),
'range': models.AttributeValue('S', '1'),
'str': models.AttributeValue('S', 'str1')
}),
WriteItemRequest.delete({
'id':
models.AttributeValue('N', 3),
'range':
models.AttributeValue('S', '3')
})
]
}
storage_manager = SimpleStorageManager(StorageDriver(),
TableInfoRepository())
storage_manager.execute_write_batch(context, request_map)
# check notification queue
self.assertEqual(len(self.get_notifications()), 2)
start_event = self.get_notifications()[0]
end_event = self.get_notifications()[1]
self.assertEqual(start_event['priority'], 'INFO')
self.assertEqual(start_event['event_type'],
notifier.EVENT_TYPE_DATA_BATCHWRITE_START)
self.assertEqual(len(start_event['payload']), len(request_map))
self.assertEqual(end_event['priority'], 'INFO')
self.assertEqual(end_event['event_type'],
notifier.EVENT_TYPE_DATA_BATCHWRITE_END)
self.assertEqual(len(end_event['payload']['write_request_map']),
len(request_map))
self.assertEqual(len(end_event['payload']['unprocessed_items']), 0)
time_start = datetime.datetime.strptime(start_event['timestamp'],
DATETIMEFORMAT)
time_end = datetime.datetime.strptime(end_event['timestamp'],
DATETIMEFORMAT)
self.assertTrue(time_start < time_end,
"start event is later than end event")