def process_request(self, req):
tenant_id = self._get_tenant_id_from_request(req)
now = time.time()
prev = self.last_time.get(tenant_id, 0)
if self.rps_per_tenant and now - prev < 1. / self.rps_per_tenant:
LOG.debug('Request rate for tenant {} exceeded preconfigured'
' limit {}. Request rejected.',
tenant_id, self.rps_per_tenant)
notifier.get_notifier().info(
{}, notifier.EVENT_TYPE_REQUEST_RATE_LIMITED, tenant_id)
raise exception.RequestQuotaExceeded()
self.last_time[tenant_id] = now
def process_request(self, req):
tenant_id = self._get_tenant_id(req)
now = time.time()
prev = self.last_time.get(tenant_id, 0)
if self.rps_per_tenant and now - prev < 1. / self.rps_per_tenant:
LOG.debug(
'Request rate for tenant {} exceeded preconfigured'
' limit {}. Request rejected.', tenant_id, self.rps_per_tenant)
notifier.get_notifier().info(
{}, notifier.EVENT_TYPE_REQUEST_RATE_LIMITED, tenant_id)
raise exception.RequestQuotaExceeded()
self.last_time[tenant_id] = now
def __init__(self, storage_driver, table_info_repo, concurrent_tasks=1000,
batch_chunk_size=25, schema_operation_timeout=300):
self._storage_driver = storage_driver
self._table_info_repo = table_info_repo
self._batch_chunk_size = batch_chunk_size
self._schema_operation_timeout = schema_operation_timeout
self.__task_executor = ThreadPoolExecutor(concurrent_tasks)
self.__task_semaphore = BoundedSemaphore(concurrent_tasks)
self._notifier = notifier.get_notifier()
def __init__(self, storage_driver, table_info_repo, concurrent_tasks=1000,
batch_chunk_size=25, schema_operation_timeout=300):
self._storage_driver = storage_driver
self._table_info_repo = table_info_repo
self._batch_chunk_size = batch_chunk_size
self._schema_operation_timeout = schema_operation_timeout
self.__task_executor = futures.ThreadPoolExecutor(concurrent_tasks)
self.__task_semaphore = threading.BoundedSemaphore(concurrent_tasks)
self._notifier = notifier.get_notifier()
def __init__(self, app, options):
self.api_type = options["api_type"]
self._notifier = notifier.get_notifier()
super(RequestNotificationsMiddleware, self).__init__(app)
def __init__(self, app, options):
self.api_type = options["api_type"]
self._notifier = notifier.get_notifier()
super(RequestNotificationsMiddleware, self).__init__(app)
def bulk_load_app(environ, start_response):
context = environ['webob.adhoc_attrs']['context']
path = environ['PATH_INFO']
LOG.debug('Request received: %s', path)
_notifier = notifier.get_notifier()
if not re.match("^/v1/data/\w+/tables/\w+/bulk_load$", path):
start_response('404 Not found', [('Content-Type', 'text/plain')])
yield 'Incorrect url. Please check it and try again\n'
_notifier.error(
context,
notifier.EVENT_TYPE_STREAMING_PATH_ERROR,
{'path': path})
return
url_comp = path.split('/')
project_id = url_comp[3]
table_name = url_comp[5]
LOG.debug('Tenant: %s, table name: %s', project_id, table_name)
utils.check_project_id(context, project_id)
_notifier.info(
context,
notifier.EVENT_TYPE_STREAMING_DATA_START,
{'path': path})
read_count = 0
processed_count = 0
unprocessed_count = 0
failed_count = 0
put_count = 0
done_count = [0]
last_read = None
failed_items = {}
dont_process = False
future_ready_event = threading.Event()
future_ready_queue = Queue.Queue()
stream = environ['wsgi.input']
while True:
chunk = stream.readline()
if not chunk:
break
read_count += 1
if dont_process:
LOG.debug('Skipping item #%d', read_count)
unprocessed_count += 1
continue
last_read = chunk
try:
future = storage.put_item_async(
context, table_name, make_put_item(chunk)
)
put_count += 1
future.add_done_callback(make_callback(
future_ready_queue,
future_ready_event,
done_count,
chunk
))
# try to get result of finished futures
try:
while True:
finished_future, chunk = future_ready_queue.get_nowait()
finished_future.result()
processed_count += 1
except Queue.Empty:
pass
except Exception as e:
failed_items[chunk] = repr(e)
dont_process = True
LOG.debug('Error inserting item: %s, message: %s',
chunk, repr(e))
_notifier.error(
context,
notifier.EVENT_TYPE_STREAMING_DATA_ERROR,
{'path': path, 'item': chunk, 'error': e.message})
LOG.debug('Request body has been read completely')
# wait for all futures to be finished
while done_count[0] < put_count:
LOG.debug('Waiting for %d item(s) to be processed...',
put_count - done_count[0])
future_ready_event.wait()
future_ready_event.clear()
LOG.debug('All items are processed. Getting results of item processing...')
# get results of finished futures
while done_count[0] > processed_count + failed_count:
LOG.debug('Waiting for %d result(s)...',
done_count[0] - processed_count - failed_count)
chunk = None
try:
finished_future, chunk = future_ready_queue.get_nowait()
finished_future.result()
processed_count += 1
except Queue.Empty:
break
except Exception as e:
failed_count += 1
failed_items[chunk] = repr(e)
LOG.debug('Error inserting item: %s, message: %s',
chunk, repr(e))
_notifier.error(
context,
notifier.EVENT_TYPE_STREAMING_DATA_ERROR,
{'path': path, 'item': chunk, 'error': e.message})
# Update count if error happened before put_item_async was invoked
if dont_process:
failed_count += 1
start_response('200 OK', [('Content-Type', 'application/json')])
resp = {
'read': read_count,
'processed': processed_count,
'unprocessed': unprocessed_count,
'failed': failed_count,
'last_item': last_read,
'failed_items': failed_items
}
_notifier.info(
context, notifier.EVENT_TYPE_STREAMING_DATA_END,
{'path': path, 'response': resp})
yield json.dumps(resp)
def bulk_load_app(environ, start_response):
context = environ['webob.adhoc_attrs']['context']
path = environ['PATH_INFO']
LOG.debug('Request received: %s', path)
_notifier = notifier.get_notifier()
if not re.match("^/v1/data/\w+/tables/\w+/bulk_load$", path):
start_response('404 Not found', [('Content-Type', 'text/plain')])
yield 'Incorrect url. Please check it and try again\n'
_notifier.error(context, notifier.EVENT_TYPE_STREAMING_PATH_ERROR,
{'path': path})
return
url_comp = path.split('/')
project_id = url_comp[3]
table_name = url_comp[5]
LOG.debug('Tenant: %s, table name: %s', project_id, table_name)
utils.check_project_id(context, project_id)
_notifier.info(context, notifier.EVENT_TYPE_STREAMING_DATA_START,
{'path': path})
read_count = 0
processed_count = 0
unprocessed_count = 0
failed_count = 0
put_count = 0
done_count = [0]
last_read = None
failed_items = {}
dont_process = False
future_ready_event = threading.Event()
future_ready_queue = Queue.Queue()
stream = environ['wsgi.input']
while True:
chunk = stream.readline()
if not chunk:
break
read_count += 1
if dont_process:
LOG.debug('Skipping item #%d', read_count)
unprocessed_count += 1
continue
last_read = chunk
try:
future = storage.put_item_async(context, table_name,
make_put_item(chunk))
put_count += 1
future.add_done_callback(
make_callback(future_ready_queue, future_ready_event,
done_count, chunk))
# try to get result of finished futures
try:
while True:
finished_future, chunk = future_ready_queue.get_nowait()
finished_future.result()
processed_count += 1
except Queue.Empty:
pass
except Exception as e:
failed_items[chunk] = repr(e)
dont_process = True
LOG.debug('Error inserting item: %s, message: %s', chunk, repr(e))
_notifier.error(context, notifier.EVENT_TYPE_STREAMING_DATA_ERROR,
{
'path': path,
'item': chunk,
'error': e.message
})
LOG.debug('Request body has been read completely')
# wait for all futures to be finished
while done_count[0] < put_count:
LOG.debug('Waiting for %d item(s) to be processed...',
put_count - done_count[0])
future_ready_event.wait()
future_ready_event.clear()
LOG.debug('All items are processed. Getting results of item processing...')
# get results of finished futures
while done_count[0] > processed_count + failed_count:
LOG.debug('Waiting for %d result(s)...',
done_count[0] - processed_count - failed_count)
chunk = None
try:
finished_future, chunk = future_ready_queue.get_nowait()
finished_future.result()
processed_count += 1
except Queue.Empty:
break
except Exception as e:
failed_count += 1
failed_items[chunk] = repr(e)
LOG.debug('Error inserting item: %s, message: %s', chunk, repr(e))
_notifier.error(context, notifier.EVENT_TYPE_STREAMING_DATA_ERROR,
{
'path': path,
'item': chunk,
'error': e.message
})
# Update count if error happened before put_item_async was invoked
if dont_process:
failed_count += 1
start_response('200 OK', [('Content-Type', 'application/json')])
resp = {
'read': read_count,
'processed': processed_count,
'unprocessed': unprocessed_count,
'failed': failed_count,
'last_item': last_read,
'failed_items': failed_items
}
_notifier.info(context, notifier.EVENT_TYPE_STREAMING_DATA_END, {
'path': path,
'response': resp
})
yield json.dumps(resp)