예제 #1
0
    def process_request(self, req):
        tenant_id = self._get_tenant_id_from_request(req)

        now = time.time()
        prev = self.last_time.get(tenant_id, 0)

        if self.rps_per_tenant and now - prev < 1. / self.rps_per_tenant:
            LOG.debug('Request rate for tenant {} exceeded preconfigured'
                      ' limit {}. Request rejected.',
                      tenant_id, self.rps_per_tenant)
            notifier.get_notifier().info(
                {}, notifier.EVENT_TYPE_REQUEST_RATE_LIMITED, tenant_id)
            raise exception.RequestQuotaExceeded()

        self.last_time[tenant_id] = now
예제 #2
0
    def process_request(self, req):
        tenant_id = self._get_tenant_id(req)

        now = time.time()
        prev = self.last_time.get(tenant_id, 0)

        if self.rps_per_tenant and now - prev < 1. / self.rps_per_tenant:
            LOG.debug(
                'Request rate for tenant {} exceeded preconfigured'
                ' limit {}. Request rejected.', tenant_id, self.rps_per_tenant)
            notifier.get_notifier().info(
                {}, notifier.EVENT_TYPE_REQUEST_RATE_LIMITED, tenant_id)
            raise exception.RequestQuotaExceeded()

        self.last_time[tenant_id] = now
예제 #3
0
 def __init__(self, storage_driver, table_info_repo, concurrent_tasks=1000,
              batch_chunk_size=25, schema_operation_timeout=300):
     self._storage_driver = storage_driver
     self._table_info_repo = table_info_repo
     self._batch_chunk_size = batch_chunk_size
     self._schema_operation_timeout = schema_operation_timeout
     self.__task_executor = ThreadPoolExecutor(concurrent_tasks)
     self.__task_semaphore = BoundedSemaphore(concurrent_tasks)
     self._notifier = notifier.get_notifier()
예제 #4
0
 def __init__(self, storage_driver, table_info_repo, concurrent_tasks=1000,
              batch_chunk_size=25, schema_operation_timeout=300):
     self._storage_driver = storage_driver
     self._table_info_repo = table_info_repo
     self._batch_chunk_size = batch_chunk_size
     self._schema_operation_timeout = schema_operation_timeout
     self.__task_executor = futures.ThreadPoolExecutor(concurrent_tasks)
     self.__task_semaphore = threading.BoundedSemaphore(concurrent_tasks)
     self._notifier = notifier.get_notifier()
예제 #5
0
    def __init__(self, app, options):
        self.api_type = options["api_type"]
        self._notifier = notifier.get_notifier()

        super(RequestNotificationsMiddleware, self).__init__(app)
예제 #6
0
    def __init__(self, app, options):
        self.api_type = options["api_type"]
        self._notifier = notifier.get_notifier()

        super(RequestNotificationsMiddleware, self).__init__(app)
예제 #7
0
def bulk_load_app(environ, start_response):
    context = environ['webob.adhoc_attrs']['context']

    path = environ['PATH_INFO']

    LOG.debug('Request received: %s', path)

    _notifier = notifier.get_notifier()

    if not re.match("^/v1/data/\w+/tables/\w+/bulk_load$", path):
        start_response('404 Not found', [('Content-Type', 'text/plain')])
        yield 'Incorrect url. Please check it and try again\n'
        _notifier.error(
            context,
            notifier.EVENT_TYPE_STREAMING_PATH_ERROR,
            {'path': path})
        return

    url_comp = path.split('/')
    project_id = url_comp[3]
    table_name = url_comp[5]

    LOG.debug('Tenant: %s, table name: %s', project_id, table_name)

    utils.check_project_id(context, project_id)

    _notifier.info(
        context,
        notifier.EVENT_TYPE_STREAMING_DATA_START,
        {'path': path})

    read_count = 0
    processed_count = 0
    unprocessed_count = 0
    failed_count = 0
    put_count = 0
    done_count = [0]
    last_read = None
    failed_items = {}

    dont_process = False

    future_ready_event = threading.Event()
    future_ready_queue = Queue.Queue()

    stream = environ['wsgi.input']
    while True:
        chunk = stream.readline()

        if not chunk:
            break

        read_count += 1

        if dont_process:
            LOG.debug('Skipping item #%d', read_count)
            unprocessed_count += 1
            continue

        last_read = chunk

        try:
            future = storage.put_item_async(
                context, table_name, make_put_item(chunk)
            )

            put_count += 1

            future.add_done_callback(make_callback(
                future_ready_queue,
                future_ready_event,
                done_count,
                chunk
            ))

            # try to get result of finished futures
            try:
                while True:
                    finished_future, chunk = future_ready_queue.get_nowait()
                    finished_future.result()
                    processed_count += 1
            except Queue.Empty:
                pass

        except Exception as e:
            failed_items[chunk] = repr(e)
            dont_process = True
            LOG.debug('Error inserting item: %s, message: %s',
                      chunk, repr(e))

            _notifier.error(
                context,
                notifier.EVENT_TYPE_STREAMING_DATA_ERROR,
                {'path': path, 'item': chunk, 'error': e.message})

    LOG.debug('Request body has been read completely')

    # wait for all futures to be finished
    while done_count[0] < put_count:
        LOG.debug('Waiting for %d item(s) to be processed...',
                  put_count - done_count[0])
        future_ready_event.wait()
        future_ready_event.clear()

    LOG.debug('All items are processed. Getting results of item processing...')

    # get results of finished futures
    while done_count[0] > processed_count + failed_count:
        LOG.debug('Waiting for %d result(s)...',
                  done_count[0] - processed_count - failed_count)
        chunk = None
        try:
            finished_future, chunk = future_ready_queue.get_nowait()
            finished_future.result()
            processed_count += 1
        except Queue.Empty:
            break
        except Exception as e:
            failed_count += 1
            failed_items[chunk] = repr(e)
            LOG.debug('Error inserting item: %s, message: %s',
                      chunk, repr(e))

            _notifier.error(
                context,
                notifier.EVENT_TYPE_STREAMING_DATA_ERROR,
                {'path': path, 'item': chunk, 'error': e.message})

    # Update count if error happened before put_item_async was invoked
    if dont_process:
        failed_count += 1

    start_response('200 OK', [('Content-Type', 'application/json')])

    resp = {
        'read': read_count,
        'processed': processed_count,
        'unprocessed': unprocessed_count,
        'failed': failed_count,
        'last_item': last_read,
        'failed_items': failed_items
    }

    _notifier.info(
        context, notifier.EVENT_TYPE_STREAMING_DATA_END,
        {'path': path, 'response': resp})

    yield json.dumps(resp)
예제 #8
0
def bulk_load_app(environ, start_response):
    context = environ['webob.adhoc_attrs']['context']

    path = environ['PATH_INFO']

    LOG.debug('Request received: %s', path)

    _notifier = notifier.get_notifier()

    if not re.match("^/v1/data/\w+/tables/\w+/bulk_load$", path):
        start_response('404 Not found', [('Content-Type', 'text/plain')])
        yield 'Incorrect url. Please check it and try again\n'
        _notifier.error(context, notifier.EVENT_TYPE_STREAMING_PATH_ERROR,
                        {'path': path})
        return

    url_comp = path.split('/')
    project_id = url_comp[3]
    table_name = url_comp[5]

    LOG.debug('Tenant: %s, table name: %s', project_id, table_name)

    utils.check_project_id(context, project_id)

    _notifier.info(context, notifier.EVENT_TYPE_STREAMING_DATA_START,
                   {'path': path})

    read_count = 0
    processed_count = 0
    unprocessed_count = 0
    failed_count = 0
    put_count = 0
    done_count = [0]
    last_read = None
    failed_items = {}

    dont_process = False

    future_ready_event = threading.Event()
    future_ready_queue = Queue.Queue()

    stream = environ['wsgi.input']
    while True:
        chunk = stream.readline()

        if not chunk:
            break

        read_count += 1

        if dont_process:
            LOG.debug('Skipping item #%d', read_count)
            unprocessed_count += 1
            continue

        last_read = chunk

        try:
            future = storage.put_item_async(context, table_name,
                                            make_put_item(chunk))

            put_count += 1

            future.add_done_callback(
                make_callback(future_ready_queue, future_ready_event,
                              done_count, chunk))

            # try to get result of finished futures
            try:
                while True:
                    finished_future, chunk = future_ready_queue.get_nowait()
                    finished_future.result()
                    processed_count += 1
            except Queue.Empty:
                pass

        except Exception as e:
            failed_items[chunk] = repr(e)
            dont_process = True
            LOG.debug('Error inserting item: %s, message: %s', chunk, repr(e))

            _notifier.error(context, notifier.EVENT_TYPE_STREAMING_DATA_ERROR,
                            {
                                'path': path,
                                'item': chunk,
                                'error': e.message
                            })

    LOG.debug('Request body has been read completely')

    # wait for all futures to be finished
    while done_count[0] < put_count:
        LOG.debug('Waiting for %d item(s) to be processed...',
                  put_count - done_count[0])
        future_ready_event.wait()
        future_ready_event.clear()

    LOG.debug('All items are processed. Getting results of item processing...')

    # get results of finished futures
    while done_count[0] > processed_count + failed_count:
        LOG.debug('Waiting for %d result(s)...',
                  done_count[0] - processed_count - failed_count)
        chunk = None
        try:
            finished_future, chunk = future_ready_queue.get_nowait()
            finished_future.result()
            processed_count += 1
        except Queue.Empty:
            break
        except Exception as e:
            failed_count += 1
            failed_items[chunk] = repr(e)
            LOG.debug('Error inserting item: %s, message: %s', chunk, repr(e))

            _notifier.error(context, notifier.EVENT_TYPE_STREAMING_DATA_ERROR,
                            {
                                'path': path,
                                'item': chunk,
                                'error': e.message
                            })

    # Update count if error happened before put_item_async was invoked
    if dont_process:
        failed_count += 1

    start_response('200 OK', [('Content-Type', 'application/json')])

    resp = {
        'read': read_count,
        'processed': processed_count,
        'unprocessed': unprocessed_count,
        'failed': failed_count,
        'last_item': last_read,
        'failed_items': failed_items
    }

    _notifier.info(context, notifier.EVENT_TYPE_STREAMING_DATA_END, {
        'path': path,
        'response': resp
    })

    yield json.dumps(resp)