def process_request(self, req): tenant_id = self._get_tenant_id_from_request(req) now = time.time() prev = self.last_time.get(tenant_id, 0) if self.rps_per_tenant and now - prev < 1. / self.rps_per_tenant: LOG.debug('Request rate for tenant {} exceeded preconfigured' ' limit {}. Request rejected.', tenant_id, self.rps_per_tenant) notifier.get_notifier().info( {}, notifier.EVENT_TYPE_REQUEST_RATE_LIMITED, tenant_id) raise exception.RequestQuotaExceeded() self.last_time[tenant_id] = now
def process_request(self, req): tenant_id = self._get_tenant_id(req) now = time.time() prev = self.last_time.get(tenant_id, 0) if self.rps_per_tenant and now - prev < 1. / self.rps_per_tenant: LOG.debug( 'Request rate for tenant {} exceeded preconfigured' ' limit {}. Request rejected.', tenant_id, self.rps_per_tenant) notifier.get_notifier().info( {}, notifier.EVENT_TYPE_REQUEST_RATE_LIMITED, tenant_id) raise exception.RequestQuotaExceeded() self.last_time[tenant_id] = now
def __init__(self, storage_driver, table_info_repo, concurrent_tasks=1000, batch_chunk_size=25, schema_operation_timeout=300): self._storage_driver = storage_driver self._table_info_repo = table_info_repo self._batch_chunk_size = batch_chunk_size self._schema_operation_timeout = schema_operation_timeout self.__task_executor = ThreadPoolExecutor(concurrent_tasks) self.__task_semaphore = BoundedSemaphore(concurrent_tasks) self._notifier = notifier.get_notifier()
def __init__(self, storage_driver, table_info_repo, concurrent_tasks=1000, batch_chunk_size=25, schema_operation_timeout=300): self._storage_driver = storage_driver self._table_info_repo = table_info_repo self._batch_chunk_size = batch_chunk_size self._schema_operation_timeout = schema_operation_timeout self.__task_executor = futures.ThreadPoolExecutor(concurrent_tasks) self.__task_semaphore = threading.BoundedSemaphore(concurrent_tasks) self._notifier = notifier.get_notifier()
def __init__(self, app, options): self.api_type = options["api_type"] self._notifier = notifier.get_notifier() super(RequestNotificationsMiddleware, self).__init__(app)
def __init__(self, app, options): self.api_type = options["api_type"] self._notifier = notifier.get_notifier() super(RequestNotificationsMiddleware, self).__init__(app)
def bulk_load_app(environ, start_response): context = environ['webob.adhoc_attrs']['context'] path = environ['PATH_INFO'] LOG.debug('Request received: %s', path) _notifier = notifier.get_notifier() if not re.match("^/v1/data/\w+/tables/\w+/bulk_load$", path): start_response('404 Not found', [('Content-Type', 'text/plain')]) yield 'Incorrect url. Please check it and try again\n' _notifier.error( context, notifier.EVENT_TYPE_STREAMING_PATH_ERROR, {'path': path}) return url_comp = path.split('/') project_id = url_comp[3] table_name = url_comp[5] LOG.debug('Tenant: %s, table name: %s', project_id, table_name) utils.check_project_id(context, project_id) _notifier.info( context, notifier.EVENT_TYPE_STREAMING_DATA_START, {'path': path}) read_count = 0 processed_count = 0 unprocessed_count = 0 failed_count = 0 put_count = 0 done_count = [0] last_read = None failed_items = {} dont_process = False future_ready_event = threading.Event() future_ready_queue = Queue.Queue() stream = environ['wsgi.input'] while True: chunk = stream.readline() if not chunk: break read_count += 1 if dont_process: LOG.debug('Skipping item #%d', read_count) unprocessed_count += 1 continue last_read = chunk try: future = storage.put_item_async( context, table_name, make_put_item(chunk) ) put_count += 1 future.add_done_callback(make_callback( future_ready_queue, future_ready_event, done_count, chunk )) # try to get result of finished futures try: while True: finished_future, chunk = future_ready_queue.get_nowait() finished_future.result() processed_count += 1 except Queue.Empty: pass except Exception as e: failed_items[chunk] = repr(e) dont_process = True LOG.debug('Error inserting item: %s, message: %s', chunk, repr(e)) _notifier.error( context, notifier.EVENT_TYPE_STREAMING_DATA_ERROR, {'path': path, 'item': chunk, 'error': e.message}) LOG.debug('Request body has been read completely') # wait for all futures to be finished while done_count[0] < put_count: LOG.debug('Waiting for %d item(s) to be processed...', put_count - done_count[0]) future_ready_event.wait() future_ready_event.clear() LOG.debug('All items are processed. Getting results of item processing...') # get results of finished futures while done_count[0] > processed_count + failed_count: LOG.debug('Waiting for %d result(s)...', done_count[0] - processed_count - failed_count) chunk = None try: finished_future, chunk = future_ready_queue.get_nowait() finished_future.result() processed_count += 1 except Queue.Empty: break except Exception as e: failed_count += 1 failed_items[chunk] = repr(e) LOG.debug('Error inserting item: %s, message: %s', chunk, repr(e)) _notifier.error( context, notifier.EVENT_TYPE_STREAMING_DATA_ERROR, {'path': path, 'item': chunk, 'error': e.message}) # Update count if error happened before put_item_async was invoked if dont_process: failed_count += 1 start_response('200 OK', [('Content-Type', 'application/json')]) resp = { 'read': read_count, 'processed': processed_count, 'unprocessed': unprocessed_count, 'failed': failed_count, 'last_item': last_read, 'failed_items': failed_items } _notifier.info( context, notifier.EVENT_TYPE_STREAMING_DATA_END, {'path': path, 'response': resp}) yield json.dumps(resp)
def bulk_load_app(environ, start_response): context = environ['webob.adhoc_attrs']['context'] path = environ['PATH_INFO'] LOG.debug('Request received: %s', path) _notifier = notifier.get_notifier() if not re.match("^/v1/data/\w+/tables/\w+/bulk_load$", path): start_response('404 Not found', [('Content-Type', 'text/plain')]) yield 'Incorrect url. Please check it and try again\n' _notifier.error(context, notifier.EVENT_TYPE_STREAMING_PATH_ERROR, {'path': path}) return url_comp = path.split('/') project_id = url_comp[3] table_name = url_comp[5] LOG.debug('Tenant: %s, table name: %s', project_id, table_name) utils.check_project_id(context, project_id) _notifier.info(context, notifier.EVENT_TYPE_STREAMING_DATA_START, {'path': path}) read_count = 0 processed_count = 0 unprocessed_count = 0 failed_count = 0 put_count = 0 done_count = [0] last_read = None failed_items = {} dont_process = False future_ready_event = threading.Event() future_ready_queue = Queue.Queue() stream = environ['wsgi.input'] while True: chunk = stream.readline() if not chunk: break read_count += 1 if dont_process: LOG.debug('Skipping item #%d', read_count) unprocessed_count += 1 continue last_read = chunk try: future = storage.put_item_async(context, table_name, make_put_item(chunk)) put_count += 1 future.add_done_callback( make_callback(future_ready_queue, future_ready_event, done_count, chunk)) # try to get result of finished futures try: while True: finished_future, chunk = future_ready_queue.get_nowait() finished_future.result() processed_count += 1 except Queue.Empty: pass except Exception as e: failed_items[chunk] = repr(e) dont_process = True LOG.debug('Error inserting item: %s, message: %s', chunk, repr(e)) _notifier.error(context, notifier.EVENT_TYPE_STREAMING_DATA_ERROR, { 'path': path, 'item': chunk, 'error': e.message }) LOG.debug('Request body has been read completely') # wait for all futures to be finished while done_count[0] < put_count: LOG.debug('Waiting for %d item(s) to be processed...', put_count - done_count[0]) future_ready_event.wait() future_ready_event.clear() LOG.debug('All items are processed. Getting results of item processing...') # get results of finished futures while done_count[0] > processed_count + failed_count: LOG.debug('Waiting for %d result(s)...', done_count[0] - processed_count - failed_count) chunk = None try: finished_future, chunk = future_ready_queue.get_nowait() finished_future.result() processed_count += 1 except Queue.Empty: break except Exception as e: failed_count += 1 failed_items[chunk] = repr(e) LOG.debug('Error inserting item: %s, message: %s', chunk, repr(e)) _notifier.error(context, notifier.EVENT_TYPE_STREAMING_DATA_ERROR, { 'path': path, 'item': chunk, 'error': e.message }) # Update count if error happened before put_item_async was invoked if dont_process: failed_count += 1 start_response('200 OK', [('Content-Type', 'application/json')]) resp = { 'read': read_count, 'processed': processed_count, 'unprocessed': unprocessed_count, 'failed': failed_count, 'last_item': last_read, 'failed_items': failed_items } _notifier.info(context, notifier.EVENT_TYPE_STREAMING_DATA_END, { 'path': path, 'response': resp }) yield json.dumps(resp)