def __init__(self,
storage_driver,
table_info_repo,
concurrent_tasks=1000,
batch_chunk_size=25,
schema_operation_timeout=300):
SimpleStorageManager.__init__(self, storage_driver, table_info_repo,
concurrent_tasks, batch_chunk_size,
schema_operation_timeout)
def __init__(self, storage_driver, table_info_repo,
concurrent_tasks=1000, batch_chunk_size=25,
schema_operation_timeout=300):
SimpleStorageManager.__init__(
self, storage_driver, table_info_repo,
concurrent_tasks, batch_chunk_size,
schema_operation_timeout)
transport = messaging.get_transport(CONF)
target = messaging.Target(topic='schema')
self._rpc_client = messaging.RPCClient(transport, target)
def __init__(self,
storage_driver,
table_info_repo,
concurrent_tasks=1000,
batch_chunk_size=25,
schema_operation_timeout=300):
SimpleStorageManager.__init__(self, storage_driver, table_info_repo,
concurrent_tasks, batch_chunk_size,
schema_operation_timeout)
transport = messaging.get_transport(CONF)
target = messaging.Target(topic='schema')
self._rpc_client = messaging.RPCClient(transport, target)
def test_update_status_on_describe_for_deleting_table_negative(
self, mock_repo_get, mock_repo_update):
context = mock.Mock(tenant='fake_tenant')
table_name = 'fake_table'
storage_manager = SimpleStorageManager(None, TableInfoRepository())
table_info = TableInfo(table_name, None, None,
TableMeta.TABLE_STATUS_DELETING)
table_info.last_update_date_time = datetime.now()
mock_repo_get.return_value = table_info
table_meta = storage_manager.describe_table(context, table_name)
self.assertEqual(table_meta.status, TableMeta.TABLE_STATUS_DELETING)
def test_update_status_on_describe_for_deleting_table_negative(
self, mock_repo_get, mock_repo_update):
context = mock.Mock(tenant='fake_tenant')
table_name = 'fake_table'
storage_manager = SimpleStorageManager(None, TableInfoRepository())
table_info = TableInfo(
table_name, None, TableMeta.TABLE_STATUS_DELETING)
table_info.last_update_date_time = datetime.now()
mock_repo_get.return_value = table_info
table_meta = storage_manager.describe_table(context, table_name)
self.assertEqual(
table_meta.status, TableMeta.TABLE_STATUS_DELETING)
def test_update_status_on_describe_for_creating_table(
self, mock_repo_get, mock_repo_update, mock_notify):
context = mock.Mock(tenant='fake_tenant')
table_name = 'fake_table'
storage_manager = SimpleStorageManager(None, TableInfoRepository())
table_info = TableInfo(
table_name, None, TableMeta.TABLE_STATUS_CREATING)
table_info.last_updated = datetime.now() - timedelta(0, 1000)
mock_repo_get.return_value = table_info
table_meta = storage_manager.describe_table(context, table_name)
self.assertEqual(
table_meta.status, TableMeta.TABLE_STATUS_CREATE_FAILED)
def test_update_status_on_describe_for_creating_table(
self, mock_repo_get, mock_repo_update, mock_notify):
context = mock.Mock(tenant='fake_tenant')
table_name = 'fake_table'
storage_manager = SimpleStorageManager(None, TableInfoRepository())
table_info = TableInfo(table_name, None,
TableMeta.TABLE_STATUS_CREATING)
table_info.last_updated = datetime.now() - timedelta(0, 1000)
mock_repo_get.return_value = table_info
table_meta = storage_manager.describe_table(context, table_name)
self.assertEqual(table_meta.status,
TableMeta.TABLE_STATUS_CREATE_FAILED)
def test_execute_get_batch(self, mock_get_item, mock_repo_get,
mock_validate_table_is_active,
mock_validate_table_schema):
future = Future()
future.set_result(True)
mock_get_item.return_value = future
context = mock.Mock(tenant='fake_tenant')
table_name = 'fake_table'
request_list = [
models.GetItemRequest(
table_name,
{
'id': models.AttributeValue('N', 1),
'str': models.AttributeValue('S', 'str1'),
},
None,
True
),
models.GetItemRequest(
table_name,
{
'id': models.AttributeValue('N', 1),
'str': models.AttributeValue('S', 'str2'),
},
None,
True
)
]
expected_get = [mock.call(context, req.table_name,
req.key_attribute_map,
req.attributes_to_get, req.consistent)
for req in request_list]
storage_manager = SimpleStorageManager(None, TableInfoRepository())
result, unprocessed_items = storage_manager.execute_get_batch(
context, request_list
)
mock_get_item.has_calls(expected_get)
self.assertEqual(unprocessed_items, [])
def test_execute_write_batch_put_delete_same_item(
self, mock_repo_get, mock_validate_table_is_active,
mock_validate_table_schema):
table_info = mock.Mock()
table_info.schema.key_attributes = ['id', 'range']
mock_repo_get.return_value = table_info
context = mock.Mock(tenant='fake_tenant')
table_name = 'fake_table'
request_map = {
table_name: [
WriteItemRequest.put(
{
'id': models.AttributeValue('N', 1),
'range': models.AttributeValue('S', '1'),
'str': models.AttributeValue('S', 'str1'),
}
),
WriteItemRequest.delete(
{
'id': models.AttributeValue('N', 1),
'range': models.AttributeValue('S', '1')
}
)
]
}
storage_manager = SimpleStorageManager(StorageDriver(),
TableInfoRepository())
with self.assertRaises(ValidationError) as raises_cm:
storage_manager.execute_write_batch(
context, request_map
)
exception = raises_cm.exception
self.assertIn("More than one", exception._error_string)
def test_execute_write_batch_put_delete_same_item(
self, mock_repo_get, mock_validate_table_is_active,
mock_validate_table_schema):
table_info = mock.Mock()
table_info.schema.key_attributes = ['id', 'range']
mock_repo_get.return_value = table_info
context = mock.Mock(tenant='fake_tenant')
table_name = 'fake_table'
request_map = {
table_name: [
WriteItemRequest.put({
'id': models.AttributeValue('N', 1),
'range': models.AttributeValue('S', '1'),
'str': models.AttributeValue('S', 'str1'),
}),
WriteItemRequest.delete({
'id':
models.AttributeValue('N', 1),
'range':
models.AttributeValue('S', '1')
})
]
}
storage_manager = SimpleStorageManager(StorageDriver(),
TableInfoRepository())
with self.assertRaises(ValidationError) as raises_cm:
storage_manager.execute_write_batch(context, request_map)
exception = raises_cm.exception
self.assertIn("More than one", exception._error_string)
def test_execute_get_batch(self, mock_get_item, mock_repo_get,
mock_validate_table_is_active,
mock_validate_table_schema):
future = Future()
future.set_result(True)
mock_get_item.return_value = future
context = mock.Mock(tenant='fake_tenant')
table_name = 'fake_table'
request_list = [
models.GetItemRequest(
table_name, {
'id': models.AttributeValue('N', 1),
'str': models.AttributeValue('S', 'str1'),
}, None, True),
models.GetItemRequest(
table_name, {
'id': models.AttributeValue('N', 1),
'str': models.AttributeValue('S', 'str2'),
}, None, True)
]
expected_get = [
mock.call(context, req.table_name, req.key_attribute_map,
req.attributes_to_get, req.consistent)
for req in request_list
]
storage_manager = SimpleStorageManager(None, TableInfoRepository())
result, unprocessed_items = storage_manager.execute_get_batch(
context, request_list)
mock_get_item.has_calls(expected_get)
self.assertEqual(unprocessed_items, [])
def test_execute_write_batch(self, mock_put_item, mock_delete_item,
mock_repo_get, mock_validate_table_is_active,
mock_validate_table_schema, mock_batch_write):
future = Future()
future.set_result(True)
mock_put_item.return_value = future
mock_delete_item.return_value = future
mock_batch_write.side_effect = NotImplementedError()
table_info = mock.Mock()
table_info.schema.key_attributes = ['id', 'range']
mock_repo_get.return_value = table_info
context = mock.Mock(tenant='fake_tenant')
table_name = 'fake_table'
request_map = {
table_name: [
WriteItemRequest.put(
{
'id': models.AttributeValue('N', 1),
'range': models.AttributeValue('S', '1'),
'str': models.AttributeValue('S', 'str1'),
}
),
WriteItemRequest.put(
{
'id': models.AttributeValue('N', 2),
'range': models.AttributeValue('S', '1'),
'str': models.AttributeValue('S', 'str1')
}
),
WriteItemRequest.delete(
{
'id': models.AttributeValue('N', 3),
'range': models.AttributeValue('S', '3')
}
)
]
}
expected_put = [
mock.call(
context, table_info,
{
'id': models.AttributeValue('N', 1),
'range': models.AttributeValue('S', '1'),
'str': models.AttributeValue('S', 'str1')
}
),
mock.call(
context, table_info,
{
'id': models.AttributeValue('N', 2),
'range': models.AttributeValue('S', '1'),
'str': models.AttributeValue('S', 'str1')
}
),
]
expected_delete = [
mock.call(
context, table_info,
{
'id': models.AttributeValue('N', 3),
'range': models.AttributeValue('S', '3')
}
)
]
storage_manager = SimpleStorageManager(StorageDriver(),
TableInfoRepository())
unprocessed_items = storage_manager.execute_write_batch(
context, request_map
)
self.assertEqual(expected_put, mock_put_item.call_args_list)
self.assertEqual(expected_delete,
mock_delete_item.call_args_list)
self.assertEqual(unprocessed_items, {})
def __init__(self, storage_driver, table_info_repo,
concurrent_tasks=1000, batch_chunk_size=25,
schema_operation_timeout=300):
SimpleStorageManager.__init__(self, storage_driver, table_info_repo,
concurrent_tasks, batch_chunk_size,
schema_operation_timeout)
def test_execute_write_batch(self, mock_put_item, mock_delete_item,
mock_repo_get, mock_validate_table_is_active,
mock_validate_table_schema, mock_batch_write):
future = Future()
future.set_result(True)
mock_put_item.return_value = future
mock_delete_item.return_value = future
mock_batch_write.side_effect = NotImplementedError()
table_info = mock.Mock()
table_info.schema.key_attributes = ['id', 'range']
mock_repo_get.return_value = table_info
context = mock.Mock(tenant='fake_tenant')
table_name = 'fake_table'
request_map = {
table_name: [
WriteItemRequest.put({
'id': models.AttributeValue('N', 1),
'range': models.AttributeValue('S', '1'),
'str': models.AttributeValue('S', 'str1'),
}),
WriteItemRequest.put({
'id': models.AttributeValue('N', 2),
'range': models.AttributeValue('S', '1'),
'str': models.AttributeValue('S', 'str1')
}),
WriteItemRequest.delete({
'id':
models.AttributeValue('N', 3),
'range':
models.AttributeValue('S', '3')
})
]
}
expected_put = [
mock.call(
context, table_info, {
'id': models.AttributeValue('N', 1),
'range': models.AttributeValue('S', '1'),
'str': models.AttributeValue('S', 'str1')
}),
mock.call(
context, table_info, {
'id': models.AttributeValue('N', 2),
'range': models.AttributeValue('S', '1'),
'str': models.AttributeValue('S', 'str1')
}),
]
expected_delete = [
mock.call(
context, table_info, {
'id': models.AttributeValue('N', 3),
'range': models.AttributeValue('S', '3')
})
]
storage_manager = SimpleStorageManager(StorageDriver(),
TableInfoRepository())
unprocessed_items = storage_manager.execute_write_batch(
context, request_map)
self.assertEqual(expected_put, mock_put_item.call_args_list)
self.assertEqual(expected_delete, mock_delete_item.call_args_list)
self.assertEqual(unprocessed_items, {})
def test_notify_batch_write(self, mock_put_item, mock_delete_item,
mock_repo_get, mock_validate_table_is_active,
mock_validate_table_schema, mock_batch_write):
self.cleanup_test_notifier()
future = Future()
future.set_result(True)
mock_put_item.return_value = future
mock_delete_item.return_value = future
table_info = mock.Mock()
table_info.schema.key_attributes = ['id', 'range']
mock_repo_get.return_value = table_info
mock_batch_write.side_effect = NotImplementedError()
context = mock.Mock(tenant='fake_tenant')
table_name = 'fake_table'
request_map = {
table_name: [
WriteItemRequest.put(
{
'id': models.AttributeValue('N', 1),
'range': models.AttributeValue('S', '1'),
'str': models.AttributeValue('S', 'str1'),
}
),
WriteItemRequest.put(
{
'id': models.AttributeValue('N', 2),
'range': models.AttributeValue('S', '1'),
'str': models.AttributeValue('S', 'str1')
}
),
WriteItemRequest.delete(
{
'id': models.AttributeValue('N', 3),
'range': models.AttributeValue('S', '3')
}
)
]
}
storage_manager = SimpleStorageManager(
StorageDriver(), TableInfoRepository()
)
storage_manager.execute_write_batch(context, request_map)
# check notification queue
self.assertEqual(len(self.get_notifications()), 2)
start_event = self.get_notifications()[0]
end_event = self.get_notifications()[1]
self.assertEqual(start_event['priority'], 'INFO')
self.assertEqual(start_event['event_type'],
notifier.EVENT_TYPE_DATA_BATCHWRITE_START)
self.assertEqual(len(start_event['payload']), len(request_map))
self.assertEqual(end_event['priority'], 'INFO')
self.assertEqual(end_event['event_type'],
notifier.EVENT_TYPE_DATA_BATCHWRITE_END)
self.assertEqual(len(end_event['payload']['write_request_map']),
len(request_map))
self.assertEqual(len(end_event['payload']['unprocessed_items']), 0)
time_start = datetime.datetime.strptime(
start_event['timestamp'], DATETIMEFORMAT)
time_end = datetime.datetime.strptime(
end_event['timestamp'], DATETIMEFORMAT)
self.assertTrue(time_start < time_end,
"start event is later than end event")
def test_notify_batch_write(self, mock_put_item, mock_delete_item,
mock_repo_get, mock_validate_table_is_active,
mock_validate_table_schema, mock_batch_write):
self.cleanup_test_notifier()
future = Future()
future.set_result(True)
mock_put_item.return_value = future
mock_delete_item.return_value = future
table_info = mock.Mock()
table_info.schema.key_attributes = ['id', 'range']
mock_repo_get.return_value = table_info
mock_batch_write.side_effect = NotImplementedError()
context = mock.Mock(tenant='fake_tenant')
table_name = 'fake_table'
request_map = {
table_name: [
WriteItemRequest.put({
'id': models.AttributeValue('N', 1),
'range': models.AttributeValue('S', '1'),
'str': models.AttributeValue('S', 'str1'),
}),
WriteItemRequest.put({
'id': models.AttributeValue('N', 2),
'range': models.AttributeValue('S', '1'),
'str': models.AttributeValue('S', 'str1')
}),
WriteItemRequest.delete({
'id':
models.AttributeValue('N', 3),
'range':
models.AttributeValue('S', '3')
})
]
}
storage_manager = SimpleStorageManager(StorageDriver(),
TableInfoRepository())
storage_manager.execute_write_batch(context, request_map)
# check notification queue
self.assertEqual(len(self.get_notifications()), 2)
start_event = self.get_notifications()[0]
end_event = self.get_notifications()[1]
self.assertEqual(start_event['priority'], 'INFO')
self.assertEqual(start_event['event_type'],
notifier.EVENT_TYPE_DATA_BATCHWRITE_START)
self.assertEqual(len(start_event['payload']), len(request_map))
self.assertEqual(end_event['priority'], 'INFO')
self.assertEqual(end_event['event_type'],
notifier.EVENT_TYPE_DATA_BATCHWRITE_END)
self.assertEqual(len(end_event['payload']['write_request_map']),
len(request_map))
self.assertEqual(len(end_event['payload']['unprocessed_items']), 0)
time_start = datetime.datetime.strptime(start_event['timestamp'],
DATETIMEFORMAT)
time_end = datetime.datetime.strptime(end_event['timestamp'],
DATETIMEFORMAT)
self.assertTrue(time_start < time_end,
"start event is later than end event")
