def scan(self,
context,
table_name,
condition_map,
attributes_to_get=None,
limit=None,
exclusive_start_key=None,
consistent=False):
table_info = self._table_info_repo.get(context, table_name)
self._validate_table_is_active(table_info)
if exclusive_start_key is not None:
self._validate_table_schema(table_info, exclusive_start_key)
payload = dict(table_name=table_name,
condition_map=condition_map,
attributes_to_get=attributes_to_get,
limit=limit,
exclusive_start_key=exclusive_start_key,
consistent=consistent)
notifier.notify(context, notifier.EVENT_TYPE_DATA_SCAN_START, payload)
with self.__task_semaphore:
result = self._storage_driver.scan(context, table_info,
condition_map,
attributes_to_get, limit,
exclusive_start_key, consistent)
notifier.notify(context, notifier.EVENT_TYPE_DATA_SCAN_END, payload)
return result
def _delete_item_async(self, context, table_info, key_attribute_map,
expected_condition_map=None):
payload = dict(
table_name=table_info.name,
key_attribute_map=key_attribute_map,
expected_condition_map=expected_condition_map
)
notifier.notify(context, notifier.EVENT_TYPE_DATA_DELETEITEM_START,
payload)
del_future = self._execute_async(
self._storage_driver.delete_item,
context, table_info, key_attribute_map, expected_condition_map
)
def callback(future):
if not future.exception():
notifier.notify(
context, notifier.EVENT_TYPE_DATA_DELETEITEM_END,
payload
)
else:
notifier.notify(
context, notifier.EVENT_TYPE_DATA_DELETEITEM_ERROR,
future.exception(), priority=notifier.PRIORITY_ERROR
)
del_future.add_done_callback(callback)
return del_future
def put_item(self, context, table_name, attribute_map, return_values=None,
if_not_exist=False, expected_condition_map=None):
table_info = self._table_info_repo.get(context, table_name)
self._validate_table_is_active(table_info)
self._validate_table_schema(table_info, attribute_map,
keys_only=False)
with self.__task_semaphore:
result = self._storage_driver.put_item(
context, table_info, attribute_map, return_values,
if_not_exist, expected_condition_map
)
notifier.notify(
context, notifier.EVENT_TYPE_DATA_PUTITEM,
dict(
table_name=table_name,
attribute_map=attribute_map,
return_values=return_values,
if_not_exist=if_not_exist,
expected_condition_map=expected_condition_map
),
priority=notifier.PRIORITY_DEBUG
)
return result
def _get_item_async(self,
context,
table_info,
hash_key,
range_key,
attributes_to_get,
consistent=True):
payload = dict(table_name=table_info.name,
hash_key=hash_key,
range_key=range_key,
attributes_to_get=attributes_to_get,
consistent=consistent)
notifier.notify(context, notifier.EVENT_TYPE_DATA_SELECTITEM_START,
payload)
select_type = (SelectType.all() if attributes_to_get is None else
SelectType.specific_attributes(attributes_to_get))
hash_key_condition_list = [IndexedCondition.eq(hash_key)]
range_key_condition_list = (None if range_key is None else
[IndexedCondition.eq(range_key)])
result = self._execute_async(self._storage_driver.select_item,
context,
table_info,
hash_key_condition_list,
range_key_condition_list,
select_type,
consistent=consistent)
notifier.notify(context, notifier.EVENT_TYPE_DATA_SELECTITEM_END,
payload)
return result
def _delete_item_async(self,
context,
table_info,
key_attribute_map,
expected_condition_map=None):
payload = dict(table_name=table_info.name,
key_attribute_map=key_attribute_map,
expected_condition_map=expected_condition_map)
notifier.notify(context, notifier.EVENT_TYPE_DATA_DELETEITEM_START,
payload)
del_future = self._execute_async(self._storage_driver.delete_item,
context, table_info,
key_attribute_map,
expected_condition_map)
def callback(future):
if not future.exception():
notifier.notify(context,
notifier.EVENT_TYPE_DATA_DELETEITEM_END,
payload)
else:
notifier.notify(context,
notifier.EVENT_TYPE_DATA_DELETEITEM_ERROR,
future.exception(),
priority=notifier.PRIORITY_ERROR)
del_future.add_done_callback(callback)
return del_future
def scan(self, context, table_name, condition_map, attributes_to_get=None,
limit=None, exclusive_start_key=None,
consistent=False):
table_info = self._table_info_repo.get(context, table_name)
self._validate_table_is_active(table_info)
if exclusive_start_key is not None:
self._validate_table_schema(table_info, exclusive_start_key)
payload = dict(table_name=table_name,
condition_map=condition_map,
attributes_to_get=attributes_to_get,
limit=limit,
exclusive_start_key=exclusive_start_key,
consistent=consistent)
notifier.notify(context, notifier.EVENT_TYPE_DATA_SCAN_START,
payload)
with self.__task_semaphore:
result = self._storage_driver.scan(
context, table_info, condition_map, attributes_to_get,
limit, exclusive_start_key, consistent
)
notifier.notify(context, notifier.EVENT_TYPE_DATA_SCAN_END,
payload)
return result
def _put_item_async(self,
context,
table_info,
attribute_map,
return_values=None,
if_not_exist=False,
expected_condition_map=None):
payload = dict(table_name=table_info.name,
attribute_map=attribute_map,
return_values=return_values,
if_not_exist=if_not_exist,
expected_condition_map=expected_condition_map)
notifier.notify(context, notifier.EVENT_TYPE_DATA_PUTITEM_START,
payload)
put_future = self._execute_async(self._storage_driver.put_item,
context, table_info, attribute_map,
return_values, if_not_exist,
expected_condition_map)
def callback(future):
if not future.exception():
notifier.notify(context, notifier.EVENT_TYPE_DATA_PUTITEM_END,
payload)
else:
notifier.notify(context,
notifier.EVENT_TYPE_DATA_DELETEITEM_ERROR,
payload=future.exception(),
priority=notifier.PRIORITY_ERROR)
put_future.add_done_callback(callback)
return put_future
def _do_delete_table(self, context, table_info):
self._storage_driver.delete_table(context, table_info)
self._table_info_repo.delete(context, table_info.name)
notifier.notify(context, notifier.EVENT_TYPE_TABLE_DELETE_END,
table_info.name)
def put_item(self,
context,
table_name,
attribute_map,
return_values=None,
if_not_exist=False,
expected_condition_map=None):
table_info = self._table_info_repo.get(context, table_name)
self._validate_table_is_active(table_info)
self._validate_table_schema(table_info, attribute_map, keys_only=False)
with self.__task_semaphore:
result = self._storage_driver.put_item(context, table_info,
attribute_map,
return_values, if_not_exist,
expected_condition_map)
notifier.notify(context,
notifier.EVENT_TYPE_DATA_PUTITEM,
dict(table_name=table_name,
attribute_map=attribute_map,
return_values=return_values,
if_not_exist=if_not_exist,
expected_condition_map=expected_condition_map),
priority=notifier.PRIORITY_DEBUG)
return result
def _put_item_async(self, context, table_info, attribute_map,
return_values=None, if_not_exist=False,
expected_condition_map=None):
payload = dict(
table_name=table_info.name,
attribute_map=attribute_map,
return_values=return_values,
if_not_exist=if_not_exist,
expected_condition_map=expected_condition_map
)
notifier.notify(context, notifier.EVENT_TYPE_DATA_PUTITEM_START,
payload)
put_future = self._execute_async(
self._storage_driver.put_item,
context, table_info, attribute_map, return_values,
if_not_exist, expected_condition_map
)
def callback(future):
if not future.exception():
notifier.notify(
context, notifier.EVENT_TYPE_DATA_PUTITEM_END,
payload
)
else:
notifier.notify(
context, notifier.EVENT_TYPE_DATA_DELETEITEM_ERROR,
payload=future.exception(),
priority=notifier.PRIORITY_ERROR
)
put_future.add_done_callback(callback)
return put_future
def _do_delete_table(self, context, table_info):
self._storage_driver.delete_table(context, table_info)
self._table_info_repo.delete(context, table_info.name)
notifier.notify(context, notifier.EVENT_TYPE_TABLE_DELETE_END,
table_info.name)
def _get_item_async(self, context, table_info, hash_key, range_key,
attributes_to_get, consistent=True):
payload = dict(table_name=table_info.name,
hash_key=hash_key,
range_key=range_key,
attributes_to_get=attributes_to_get,
consistent=consistent)
notifier.notify(context, notifier.EVENT_TYPE_DATA_SELECTITEM_START,
payload)
select_type = (
SelectType.all() if attributes_to_get is None else
SelectType.specific_attributes(attributes_to_get)
)
hash_key_condition_list = [IndexedCondition.eq(hash_key)]
range_key_condition_list = (
None if range_key is None else [IndexedCondition.eq(range_key)]
)
result = self._execute_async(
self._storage_driver.select_item,
context, table_info, hash_key_condition_list,
range_key_condition_list, select_type, consistent=consistent
)
notifier.notify(context, notifier.EVENT_TYPE_DATA_SELECTITEM_END,
payload)
return result
def callback(future):
if not future.exception():
notifier.notify(context, notifier.EVENT_TYPE_DATA_PUTITEM_END,
payload)
else:
notifier.notify(context,
notifier.EVENT_TYPE_DATA_DELETEITEM_ERROR,
payload=future.exception(),
priority=notifier.PRIORITY_ERROR)
def execute_write_batch(self, context, write_request_map):
notifier.notify(context, notifier.EVENT_TYPE_DATA_BATCHWRITE_START,
write_request_map)
write_request_list_to_send = []
for table_name, write_request_list in write_request_map.iteritems():
table_info = self._table_info_repo.get(context, table_name)
for req in write_request_list:
self._validate_table_is_active(table_info)
if req.is_put:
self._validate_table_schema(table_info, req.attribute_map,
keys_only=False)
else:
self._validate_table_schema(table_info, req.attribute_map)
write_request_list_to_send.append(
(table_info, req)
)
future_result_list = []
for i in xrange(0, len(write_request_list_to_send),
self._batch_chunk_size):
req_list = (
write_request_list_to_send[i:i+self._batch_chunk_size]
)
future_result_list.append(
self._batch_write_async(context, req_list)
)
unprocessed_items = {}
for future_result in future_result_list:
unprocessed_request_list = future_result.result()
for (table_info, write_request) in unprocessed_request_list:
table_name = table_info.name
tables_unprocessed_items = (
unprocessed_items.get(table_name, None)
)
if tables_unprocessed_items is None:
tables_unprocessed_items = []
unprocessed_items[
table_name
] = tables_unprocessed_items
tables_unprocessed_items.append(write_request)
notifier.notify(
context, notifier.EVENT_TYPE_DATA_BATCHWRITE_END,
dict(
write_request_map=write_request_map,
unprocessed_items=unprocessed_items
)
)
return unprocessed_items
def callback(future):
if not future.exception():
notifier.notify(
context, notifier.EVENT_TYPE_DATA_DELETEITEM_END,
payload
)
else:
notifier.notify(
context, notifier.EVENT_TYPE_DATA_DELETEITEM_ERROR,
future.exception(), priority=notifier.PRIORITY_ERROR
)
def callback(future):
if not future.exception():
self._table_info_repo.delete(context, table_info.name)
notifier.notify(context, notifier.EVENT_TYPE_TABLE_DELETE_END,
table_info.name)
else:
table_info.status = models.TableMeta.TABLE_STATUS_DELETE_FAILED
self._table_info_repo.update(context, table_info, ["status"])
notifier.notify(context,
notifier.EVENT_TYPE_TABLE_DELETE_ERROR,
future.exception(),
priority=notifier.PRIORITY_ERROR)
def delete_table(self, context, table_name):
notifier.notify(context, notifier.EVENT_TYPE_TABLE_DELETE_START,
table_name)
try:
table_info = self._table_info_repo.get(context,
table_name,
['status'])
except TableNotExistsException as e:
notifier.notify(context, notifier.EVENT_TYPE_TABLE_DELETE_ERROR,
e.message, priority=notifier.PRIORITY_ERROR)
raise
if table_info.status == TableMeta.TABLE_STATUS_DELETING:
# table is already being deleted, just return immediately
notifier.notify(context, notifier.EVENT_TYPE_TABLE_DELETE_END,
table_name)
return TableMeta(table_info.schema, table_info.status)
elif table_info.status != TableMeta.TABLE_STATUS_ACTIVE:
e = ResourceInUseException()
notifier.notify(context, notifier.EVENT_TYPE_TABLE_DELETE_ERROR,
table_name + ' ' + e.message,
priority=notifier.PRIORITY_ERROR)
raise e
table_info.status = TableMeta.TABLE_STATUS_DELETING
self._table_info_repo.update(context, table_info, ["status"])
self._do_delete_table(context, table_info)
return TableMeta(table_info.schema, table_info.status)
def list_tables(self,
context,
exclusive_start_table_name=None,
limit=None):
tnames = self._table_info_repo.get_tenant_table_names(
context, exclusive_start_table_name, limit)
notifier.notify(
context,
notifier.EVENT_TYPE_TABLE_LIST,
dict(exclusive_start_table_name=exclusive_start_table_name,
limit=limit),
priority=notifier.PRIORITY_DEBUG)
return tnames
def delete_table(self, context, table_name):
notifier.notify(context, notifier.EVENT_TYPE_TABLE_DELETE_START,
table_name)
try:
table_info = self._table_info_repo.get(context, table_name,
['status'])
except TableNotExistsException as e:
notifier.notify(context,
notifier.EVENT_TYPE_TABLE_DELETE_ERROR,
e.message,
priority=notifier.PRIORITY_ERROR)
raise
if table_info.status == TableMeta.TABLE_STATUS_DELETING:
# table is already being deleted, just return immediately
notifier.notify(context, notifier.EVENT_TYPE_TABLE_DELETE_END,
table_name)
return TableMeta(table_info.schema, table_info.status)
elif table_info.status != TableMeta.TABLE_STATUS_ACTIVE:
e = ResourceInUseException()
notifier.notify(context,
notifier.EVENT_TYPE_TABLE_DELETE_ERROR,
table_name + ' ' + e.message,
priority=notifier.PRIORITY_ERROR)
raise e
table_info.status = TableMeta.TABLE_STATUS_DELETING
self._table_info_repo.update(context, table_info, ["status"])
self._do_delete_table(context, table_info)
return TableMeta(table_info.schema, table_info.status)
def callback(future):
if not future.exception():
table_info.status = models.TableMeta.TABLE_STATUS_ACTIVE
table_info.internal_name = future.result()
self._table_info_repo.update(context, table_info,
["status", "internal_name"])
notifier.notify(context, notifier.EVENT_TYPE_TABLE_CREATE_END,
table_info.schema)
else:
table_info.status = models.TableMeta.TABLE_STATUS_CREATE_FAILED
self._table_info_repo.update(context, table_info, ["status"])
notifier.notify(context,
notifier.EVENT_TYPE_TABLE_CREATE_ERROR,
future.exception(),
priority=notifier.PRIORITY_ERROR)
def process_request(self, req):
tenant_id = self._get_tenant_id(req)
now = time.time()
prev = self.last_time.get(tenant_id, 0)
if self.rps_per_tenant and now - prev < 1. / self.rps_per_tenant:
LOG.debug(
'Request rate for tenant {} exceeded preconfigured'
' limit {}. Request rejected.', tenant_id, self.rps_per_tenant)
notifier.notify({}, notifier.EVENT_TYPE_REQUEST_RATE_LIMITED,
tenant_id)
raise exception.RequestQuotaExceeded()
self.last_time[tenant_id] = now
def list_tables(self, context, exclusive_start_table_name=None,
limit=None):
tnames = self._table_info_repo.get_tenant_table_names(
context, exclusive_start_table_name, limit
)
notifier.notify(
context, notifier.EVENT_TYPE_TABLE_LIST,
dict(
exclusive_start_table_name=exclusive_start_table_name,
limit=limit
),
priority=notifier.PRIORITY_DEBUG
)
return tnames
def create_table(self, context, table_name, table_schema):
notifier.notify(context, notifier.EVENT_TYPE_TABLE_CREATE_START,
table_schema)
table_info = TableInfo(table_name, table_schema,
TableMeta.TABLE_STATUS_CREATING)
try:
self._table_info_repo.save(context, table_info)
except TableAlreadyExistsException as e:
notifier.notify(context, notifier.EVENT_TYPE_TABLE_CREATE_ERROR,
e.message, priority=notifier.PRIORITY_ERROR)
raise
self._do_create_table(context, table_info)
return TableMeta(table_info.schema, table_info.status)
def callback(future):
if not future.exception():
self._table_info_repo.delete(
context, table_info.name
)
notifier.notify(context, notifier.EVENT_TYPE_TABLE_DELETE_END,
table_info.name)
else:
table_info.status = models.TableMeta.TABLE_STATUS_DELETE_FAILED
self._table_info_repo.update(
context, table_info, ["status"]
)
notifier.notify(
context, notifier.EVENT_TYPE_TABLE_DELETE_ERROR,
future.exception(), priority=notifier.PRIORITY_ERROR
)
def _do_create_table(self, context, table_info):
try:
table_info.internal_name = self._storage_driver.create_table(
context, table_info)
table_info.status = TableMeta.TABLE_STATUS_ACTIVE
self._table_info_repo.update(context, table_info,
["status", "internal_name"])
except BackendInteractionException as ex:
notifier.notify(context,
notifier.EVENT_TYPE_TABLE_CREATE_ERROR,
ex.message,
priority=notifier.PRIORITY_ERROR)
raise
notifier.notify(context, notifier.EVENT_TYPE_TABLE_CREATE_END,
table_info.schema)
def _do_create_table(self, context, table_info):
try:
table_info.internal_name = self._storage_driver.create_table(
context, table_info
)
table_info.status = TableMeta.TABLE_STATUS_ACTIVE
self._table_info_repo.update(
context, table_info, ["status", "internal_name"]
)
except BackendInteractionException as ex:
notifier.notify(context, notifier.EVENT_TYPE_TABLE_CREATE_ERROR,
ex.message, priority=notifier.PRIORITY_ERROR)
raise
notifier.notify(context, notifier.EVENT_TYPE_TABLE_CREATE_END,
table_info.schema)
def execute_write_batch(self, context, write_request_map):
notifier.notify(context, notifier.EVENT_TYPE_DATA_BATCHWRITE_START,
write_request_map)
write_request_list_to_send = []
for table_name, write_request_list in write_request_map.iteritems():
table_info = self._table_info_repo.get(context, table_name)
for req in write_request_list:
self._validate_table_is_active(table_info)
if req.is_put:
self._validate_table_schema(table_info,
req.attribute_map,
keys_only=False)
else:
self._validate_table_schema(table_info, req.attribute_map)
write_request_list_to_send.append((table_info, req))
future_result_list = []
for i in xrange(0, len(write_request_list_to_send),
self._batch_chunk_size):
req_list = (write_request_list_to_send[i:i +
self._batch_chunk_size])
future_result_list.append(
self._batch_write_async(context, req_list))
unprocessed_items = {}
for future_result in future_result_list:
unprocessed_request_list = future_result.result()
for (table_info, write_request) in unprocessed_request_list:
table_name = table_info.name
tables_unprocessed_items = (unprocessed_items.get(
table_name, None))
if tables_unprocessed_items is None:
tables_unprocessed_items = []
unprocessed_items[table_name] = tables_unprocessed_items
tables_unprocessed_items.append(write_request)
notifier.notify(
context, notifier.EVENT_TYPE_DATA_BATCHWRITE_END,
dict(write_request_map=write_request_map,
unprocessed_items=unprocessed_items))
return unprocessed_items
def callback(future):
if not future.exception():
table_info.status = models.TableMeta.TABLE_STATUS_ACTIVE
table_info.internal_name = future.result()
self._table_info_repo.update(
context, table_info, ["status", "internal_name"]
)
notifier.notify(context, notifier.EVENT_TYPE_TABLE_CREATE_END,
table_info.schema)
else:
table_info.status = models.TableMeta.TABLE_STATUS_CREATE_FAILED
self._table_info_repo.update(
context, table_info, ["status"]
)
notifier.notify(
context, notifier.EVENT_TYPE_TABLE_CREATE_ERROR,
future.exception(), priority=notifier.PRIORITY_ERROR
)
def create_table(self, context, table_name, table_schema):
notifier.notify(context, notifier.EVENT_TYPE_TABLE_CREATE_START,
table_schema)
table_info = TableInfo(table_name, table_schema,
TableMeta.TABLE_STATUS_CREATING)
try:
self._table_info_repo.save(context, table_info)
except TableAlreadyExistsException as e:
notifier.notify(context,
notifier.EVENT_TYPE_TABLE_CREATE_ERROR,
e.message,
priority=notifier.PRIORITY_ERROR)
raise
self._do_create_table(context, table_info)
return TableMeta(table_info.schema, table_info.status)
def delete_item(self,
context,
table_name,
key_attribute_map,
expected_condition_map=None):
table_info = self._table_info_repo.get(context, table_name)
self._validate_table_is_active(table_info)
self._validate_table_schema(table_info, key_attribute_map)
with self.__task_semaphore:
result = self._storage_driver.delete_item(context, table_info,
key_attribute_map,
expected_condition_map)
notifier.notify(context,
notifier.EVENT_TYPE_DATA_DELETEITEM,
dict(table_name=table_name,
key_attribute_map=key_attribute_map,
expected_condition_map=expected_condition_map),
priority=notifier.PRIORITY_DEBUG)
return result
def delete_item(self, context, table_name, key_attribute_map,
expected_condition_map=None):
table_info = self._table_info_repo.get(context, table_name)
self._validate_table_is_active(table_info)
self._validate_table_schema(table_info, key_attribute_map)
with self.__task_semaphore:
result = self._storage_driver.delete_item(
context, table_info, key_attribute_map, expected_condition_map
)
notifier.notify(
context, notifier.EVENT_TYPE_DATA_DELETEITEM,
dict(
table_name=table_name,
key_attribute_map=key_attribute_map,
expected_condition_map=expected_condition_map
),
priority=notifier.PRIORITY_DEBUG
)
return result
def describe_table(self, context, table_name):
table_info = self._table_info_repo.get(context, table_name,
['status', 'last_updated'])
notifier.notify(context,
notifier.EVENT_TYPE_TABLE_DESCRIBE,
table_name,
priority=notifier.PRIORITY_DEBUG)
timedelta = datetime.now() - table_info.last_updated
if timedelta.total_seconds() > self._schema_operation_timeout:
if table_info.status == TableMeta.TABLE_STATUS_CREATING:
table_info.status = TableMeta.TABLE_STATUS_CREATE_FAILED
self._table_info_repo.update(context, table_info, ['status'])
LOG.debug("Table '{}' creation timed out."
" Setting status to {}".format(
table_info.name,
TableMeta.TABLE_STATUS_CREATE_FAILED))
notifier.notify(
context.to_dict(), notifier.EVENT_TYPE_TABLE_CREATE_ERROR,
dict(table_name=table_name, message='Operation timed out'))
if table_info.status == TableMeta.TABLE_STATUS_DELETING:
table_info.status = TableMeta.TABLE_STATUS_DELETE_FAILED
self._table_info_repo.update(context, table_info, ['status'])
LOG.debug("Table '{}' deletion timed out."
" Setting status to {}".format(
table_info.name,
TableMeta.TABLE_STATUS_DELETE_FAILED))
notifier.notify(
context.to_dict(), notifier.EVENT_TYPE_TABLE_DELETE_ERROR,
dict(table_name=table_name, message='Operation timed out'))
return TableMeta(table_info.schema, table_info.status)
def describe_table(self, context, table_name):
table_info = self._table_info_repo.get(
context, table_name, ['status', 'last_updated'])
notifier.notify(context, notifier.EVENT_TYPE_TABLE_DESCRIBE,
table_name, priority=notifier.PRIORITY_DEBUG)
timedelta = datetime.now() - table_info.last_updated
if timedelta.total_seconds() > self._schema_operation_timeout:
if table_info.status == TableMeta.TABLE_STATUS_CREATING:
table_info.status = TableMeta.TABLE_STATUS_CREATE_FAILED
self._table_info_repo.update(context, table_info, ['status'])
LOG.debug(
"Table '{}' creation timed out."
" Setting status to {}".format(
table_info.name, TableMeta.TABLE_STATUS_CREATE_FAILED)
)
notifier.notify(
context.to_dict(),
notifier.EVENT_TYPE_TABLE_CREATE_ERROR,
dict(
table_name=table_name,
message='Operation timed out'
)
)
if table_info.status == TableMeta.TABLE_STATUS_DELETING:
table_info.status = TableMeta.TABLE_STATUS_DELETE_FAILED
self._table_info_repo.update(context, table_info, ['status'])
LOG.debug(
"Table '{}' deletion timed out."
" Setting status to {}".format(
table_info.name, TableMeta.TABLE_STATUS_DELETE_FAILED)
)
notifier.notify(
context.to_dict(),
notifier.EVENT_TYPE_TABLE_DELETE_ERROR,
dict(
table_name=table_name,
message='Operation timed out'
)
)
return TableMeta(table_info.schema, table_info.status)
def bulk_load_app(environ, start_response):
context = environ['webob.adhoc_attrs']['context']
path = environ['PATH_INFO']
LOG.debug('Request received: %s', path)
if not re.match("^/v1/\w+/data/tables/\w+/bulk_load$", path):
start_response('404 Not found', [('Content-Type', 'text/plain')])
yield 'Incorrect url. Please check it and try again\n'
notifier.notify(context,
notifier.EVENT_TYPE_STREAMING_PATH_ERROR,
path,
priority=notifier.PRIORITY_ERROR)
return
url_comp = path.split('/')
project_id = url_comp[2]
table_name = url_comp[5]
LOG.debug('Tenant: %s, table name: %s', project_id, table_name)
utils.check_project_id(context, project_id)
notifier.notify(context, notifier.EVENT_TYPE_STREAMING_DATA_START, path)
read_count = 0
processed_count = 0
unprocessed_count = 0
failed_count = 0
put_count = 0
done_count = [0]
last_read = None
failed_items = {}
dont_process = False
future_ready_event = Event()
future_ready_queue = Queue.Queue()
stream = environ['wsgi.input']
while True:
chunk = stream.readline()
if not chunk:
break
read_count += 1
if dont_process:
LOG.debug('Skipping item #%d', read_count)
unprocessed_count += 1
continue
last_read = chunk
try:
future = storage.put_item_async(context, table_name,
make_put_item(chunk))
put_count += 1
future.add_done_callback(
make_callback(future_ready_queue, future_ready_event,
done_count, chunk))
# try to get result of finished futures
try:
while True:
finished_future, chunk = future_ready_queue.get_nowait()
finished_future.result()
processed_count += 1
except Queue.Empty:
pass
except Exception as e:
failed_items[chunk] = repr(e)
dont_process = True
LOG.debug('Error inserting item: %s, message: %s', chunk, repr(e))
notifier.notify(context, notifier.EVENT_TYPE_STREAMING_DATA_ERROR,
{
'path': path,
'item': chunk,
'error': e.message
})
LOG.debug('Request body has been read completely')
# wait for all futures to be finished
while done_count[0] < put_count:
LOG.debug('Waiting for %d item(s) to be processed...',
put_count - done_count[0])
future_ready_event.wait()
future_ready_event.clear()
LOG.debug('All items are processed. Getting results of item processing...')
# get results of finished futures
while done_count[0] > processed_count + failed_count:
LOG.debug('Waiting for %d result(s)...',
done_count[0] - processed_count - failed_count)
chunk = None
try:
finished_future, chunk = future_ready_queue.get_nowait()
finished_future.result()
processed_count += 1
except Queue.Empty:
break
except Exception as e:
failed_count += 1
failed_items[chunk] = repr(e)
LOG.debug('Error inserting item: %s, message: %s', chunk, repr(e))
notifier.notify(context, notifier.EVENT_TYPE_STREAMING_DATA_ERROR,
{
'path': path,
'item': chunk,
'error': e.message
})
# Update count if error happened before put_item_async was invoked
if dont_process:
failed_count += 1
start_response('200 OK', [('Content-Type', 'application/json')])
resp = {
'read': read_count,
'processed': processed_count,
'unprocessed': unprocessed_count,
'failed': failed_count,
'last_item': last_read,
'failed_items': failed_items
}
notifier.notify(context, notifier.EVENT_TYPE_STREAMING_DATA_END, {
'path': path,
'response': resp
})
yield json.dumps(resp)
def bulk_load_app(environ, start_response):
context = environ['webob.adhoc_attrs']['context']
path = environ['PATH_INFO']
LOG.debug('Request received: %s', path)
if not re.match("^/v1/\w+/data/tables/\w+/bulk_load$", path):
start_response('404 Not found', [('Content-Type', 'text/plain')])
yield 'Incorrect url. Please check it and try again\n'
notifier.notify(context, notifier.EVENT_TYPE_STREAMING_PATH_ERROR,
path, priority=notifier.PRIORITY_ERROR)
return
url_comp = path.split('/')
project_id = url_comp[2]
table_name = url_comp[5]
LOG.debug('Tenant: %s, table name: %s', project_id, table_name)
utils.check_project_id(context, project_id)
notifier.notify(context, notifier.EVENT_TYPE_STREAMING_DATA_START, path)
read_count = 0
processed_count = 0
unprocessed_count = 0
failed_count = 0
put_count = 0
done_count = [0]
last_read = None
failed_items = {}
dont_process = False
future_ready_event = Event()
future_ready_queue = Queue.Queue()
stream = environ['wsgi.input']
while True:
chunk = stream.readline()
if not chunk:
break
read_count += 1
if dont_process:
LOG.debug('Skipping item #%d', read_count)
unprocessed_count += 1
continue
last_read = chunk
try:
future = storage.put_item_async(
context, table_name, make_put_item(chunk)
)
put_count += 1
future.add_done_callback(make_callback(
future_ready_queue,
future_ready_event,
done_count,
chunk
))
# try to get result of finished futures
try:
while True:
finished_future, chunk = future_ready_queue.get_nowait()
finished_future.result()
processed_count += 1
except Queue.Empty:
pass
except Exception as e:
failed_items[chunk] = repr(e)
dont_process = True
LOG.debug('Error inserting item: %s, message: %s',
chunk, repr(e))
notifier.notify(context, notifier.EVENT_TYPE_STREAMING_DATA_ERROR,
{'path': path, 'item': chunk, 'error': e.message})
LOG.debug('Request body has been read completely')
# wait for all futures to be finished
while done_count[0] < put_count:
LOG.debug('Waiting for %d item(s) to be processed...',
put_count - done_count[0])
future_ready_event.wait()
future_ready_event.clear()
LOG.debug('All items are processed. Getting results of item processing...')
# get results of finished futures
while done_count[0] > processed_count + failed_count:
LOG.debug('Waiting for %d result(s)...',
done_count[0] - processed_count - failed_count)
chunk = None
try:
finished_future, chunk = future_ready_queue.get_nowait()
finished_future.result()
processed_count += 1
except Queue.Empty:
break
except Exception as e:
failed_count += 1
failed_items[chunk] = repr(e)
LOG.debug('Error inserting item: %s, message: %s',
chunk, repr(e))
notifier.notify(context, notifier.EVENT_TYPE_STREAMING_DATA_ERROR,
{'path': path, 'item': chunk, 'error': e.message})
# Update count if error happened before put_item_async was invoked
if dont_process:
failed_count += 1
start_response('200 OK', [('Content-Type', 'application/json')])
resp = {
'read': read_count,
'processed': processed_count,
'unprocessed': unprocessed_count,
'failed': failed_count,
'last_item': last_read,
'failed_items': failed_items
}
notifier.notify(context, notifier.EVENT_TYPE_STREAMING_DATA_END,
{'path': path, 'response': resp})
yield json.dumps(resp)
def select_item(self,
context,
table_name,
indexed_condition_map,
select_type,
index_name=None,
limit=None,
exclusive_start_key=None,
consistent=True,
order_type=None):
table_info = self._table_info_repo.get(context, table_name)
self._validate_table_is_active(table_info)
schema_attribute_type_map = table_info.schema.attribute_type_map
hash_key_name = table_info.schema.hash_key_name
range_key_name = table_info.schema.range_key_name
if index_name is not None:
index_def = table_info.schema.index_def_map.get(index_name)
if index_def is None:
raise ValidationError(_(
"Index '%(index_name)s' doesn't exist for table "
"'%(table_name)s'"),
index_name=index_name,
table_name=table_name)
range_key_name_to_query = index_def.alt_range_key_attr
else:
range_key_name_to_query = range_key_name
if exclusive_start_key is not None:
self._validate_table_schema(table_info,
exclusive_start_key,
index_name=index_name)
indexed_condition_map_copy = indexed_condition_map.copy()
hash_key_condition_list = indexed_condition_map_copy.pop(
hash_key_name, None)
range_key_to_query_condition_list = indexed_condition_map_copy.pop(
range_key_name_to_query, None)
indexed_condition_schema_valid = False
if len(indexed_condition_map_copy) == 0 and hash_key_condition_list:
hash_key_type = schema_attribute_type_map[hash_key_name]
for hash_key_condition in hash_key_condition_list:
for hash_key_condition_arg in hash_key_condition.args:
if hash_key_condition_arg.attr_type != hash_key_type:
break
else:
continue
break
else:
if range_key_to_query_condition_list:
range_key_to_query_type = schema_attribute_type_map[
range_key_name_to_query]
for range_key_to_query_condition in (
range_key_to_query_condition_list):
for range_key_to_query_condition_arg in (
range_key_to_query_condition.args):
if (range_key_to_query_condition_arg.attr_type !=
range_key_to_query_type):
break
else:
continue
break
else:
indexed_condition_schema_valid = True
else:
indexed_condition_schema_valid = True
if not indexed_condition_schema_valid:
raise ValidationError(_(
"Specified query conditions %(indexed_condition_map)s "
"doesn't match table schema: %(table_schema)s"),
indexed_condition_map=indexed_condition_map,
table_schema=table_info.schema)
if (len(hash_key_condition_list) != 1
or hash_key_condition_list[0].type !=
IndexedCondition.CONDITION_TYPE_EQUAL):
raise ValidationError(
_("Only equality condition is allowed for HASH key attribute "
"'%(hash_key_name)s'"),
hash_key_name=hash_key_name,
)
with self.__task_semaphore:
result = self._storage_driver.select_item(
context, table_info, hash_key_condition_list,
range_key_to_query_condition_list, select_type, index_name,
limit, exclusive_start_key, consistent, order_type)
notifier.notify(context,
notifier.EVENT_TYPE_DATA_SELECTITEM,
dict(table_name=table_name,
indexed_condition_map=indexed_condition_map,
select_type=select_type,
index_name=index_name,
limit=limit,
exclusive_start_key=exclusive_start_key,
consistent=consistent,
order_type=order_type),
priority=notifier.PRIORITY_DEBUG)
return result
def execute_get_batch(self, context, read_request_list):
assert read_request_list
items = []
unprocessed_items = []
request_count = len(read_request_list)
done_count = [0]
done_event = Event()
prepared_batch = []
for req in read_request_list:
def make_request_executor():
_req = req
_table_name = _req.table_name
_key_attribute_map = _req.key_attribute_map
_table_info = self._table_info_repo.get(context, _table_name)
self._validate_table_is_active(_table_info)
self._validate_table_schema(_table_info, _key_attribute_map)
_attributes_to_get = req.attributes_to_get
def callback(res):
try:
items.append((_table_name, res.result()))
except Exception:
unprocessed_items.append(_req)
LOG.exception("Can't process GetItemRequest")
done_count[0] += 1
if done_count[0] >= request_count:
done_event.set()
def executor():
future_result = self._get_item_async(
context, _table_info,
_key_attribute_map.get(
_table_info.schema.hash_key_name
),
_key_attribute_map.get(
_table_info.schema.range_key_name
),
_attributes_to_get, consistent=_req.consistent
)
future_result.add_done_callback(callback)
return executor
prepared_batch.append(make_request_executor())
notifier.notify(context, notifier.EVENT_TYPE_DATA_BATCHREAD_START,
read_request_list)
for request_executor in prepared_batch:
request_executor()
done_event.wait()
notifier.notify(
context, notifier.EVENT_TYPE_DATA_BATCHREAD_END,
dict(
read_request_list=read_request_list,
unprocessed_items=unprocessed_items
)
)
return items, unprocessed_items
def select_item(self, context, table_name, indexed_condition_map,
select_type, index_name=None, limit=None,
exclusive_start_key=None, consistent=True,
order_type=None):
table_info = self._table_info_repo.get(context, table_name)
self._validate_table_is_active(table_info)
schema_attribute_type_map = table_info.schema.attribute_type_map
hash_key_name = table_info.schema.hash_key_name
range_key_name = table_info.schema.range_key_name
if index_name is not None:
index_def = table_info.schema.index_def_map.get(index_name)
if index_def is None:
raise ValidationError(
_("Index '%(index_name)s' doesn't exist for table "
"'%(table_name)s'"),
index_name=index_name, table_name=table_name)
range_key_name_to_query = index_def.alt_range_key_attr
else:
range_key_name_to_query = range_key_name
if exclusive_start_key is not None:
self._validate_table_schema(
table_info, exclusive_start_key, index_name=index_name
)
indexed_condition_map_copy = indexed_condition_map.copy()
hash_key_condition_list = indexed_condition_map_copy.pop(hash_key_name,
None)
range_key_to_query_condition_list = indexed_condition_map_copy.pop(
range_key_name_to_query, None
)
indexed_condition_schema_valid = False
if len(indexed_condition_map_copy) == 0 and hash_key_condition_list:
hash_key_type = schema_attribute_type_map[hash_key_name]
for hash_key_condition in hash_key_condition_list:
for hash_key_condition_arg in hash_key_condition.args:
if hash_key_condition_arg.attr_type != hash_key_type:
break
else:
continue
break
else:
if range_key_to_query_condition_list:
range_key_to_query_type = schema_attribute_type_map[
range_key_name_to_query
]
for range_key_to_query_condition in (
range_key_to_query_condition_list):
for range_key_to_query_condition_arg in (
range_key_to_query_condition.args):
if (range_key_to_query_condition_arg.attr_type !=
range_key_to_query_type):
break
else:
continue
break
else:
indexed_condition_schema_valid = True
else:
indexed_condition_schema_valid = True
if not indexed_condition_schema_valid:
raise ValidationError(
_("Specified query conditions %(indexed_condition_map)s "
"doesn't match table schema: %(table_schema)s"),
indexed_condition_map=indexed_condition_map,
table_schema=table_info.schema
)
if (len(hash_key_condition_list) != 1 or
hash_key_condition_list[0].type !=
IndexedCondition.CONDITION_TYPE_EQUAL):
raise ValidationError(
_("Only equality condition is allowed for HASH key attribute "
"'%(hash_key_name)s'"),
hash_key_name=hash_key_name,
)
with self.__task_semaphore:
result = self._storage_driver.select_item(
context, table_info, hash_key_condition_list,
range_key_to_query_condition_list, select_type,
index_name, limit, exclusive_start_key, consistent, order_type
)
notifier.notify(
context, notifier.EVENT_TYPE_DATA_SELECTITEM,
dict(
table_name=table_name,
indexed_condition_map=indexed_condition_map,
select_type=select_type,
index_name=index_name,
limit=limit,
exclusive_start_key=exclusive_start_key,
consistent=consistent,
order_type=order_type
),
priority=notifier.PRIORITY_DEBUG
)
return result
def execute_get_batch(self, context, read_request_list):
assert read_request_list
items = []
unprocessed_items = []
request_count = len(read_request_list)
done_count = [0]
done_event = Event()
prepared_batch = []
for req in read_request_list:
def make_request_executor():
_req = req
_table_name = _req.table_name
_key_attribute_map = _req.key_attribute_map
_table_info = self._table_info_repo.get(context, _table_name)
self._validate_table_is_active(_table_info)
self._validate_table_schema(_table_info, _key_attribute_map)
_attributes_to_get = req.attributes_to_get
def callback(res):
try:
items.append((_table_name, res.result()))
except Exception:
unprocessed_items.append(_req)
LOG.exception("Can't process GetItemRequest")
done_count[0] += 1
if done_count[0] >= request_count:
done_event.set()
def executor():
future_result = self._get_item_async(
context,
_table_info,
_key_attribute_map.get(
_table_info.schema.hash_key_name),
_key_attribute_map.get(
_table_info.schema.range_key_name),
_attributes_to_get,
consistent=_req.consistent)
future_result.add_done_callback(callback)
return executor
prepared_batch.append(make_request_executor())
notifier.notify(context, notifier.EVENT_TYPE_DATA_BATCHREAD_START,
read_request_list)
for request_executor in prepared_batch:
request_executor()
done_event.wait()
notifier.notify(
context, notifier.EVENT_TYPE_DATA_BATCHREAD_END,
dict(read_request_list=read_request_list,
unprocessed_items=unprocessed_items))
return items, unprocessed_items
