Пример #1
0
    def _next(self):
        runner = self.runner
        cfg = self.monitor.cfg

        if self.tests:
            run = True
            tag, testcls = self.tests.pop()
            testcls.tag = tag
            testcls.cfg = cfg
            testcls.async = AsyncAssert(testcls)
            try:
                all_tests = runner.loadTestsFromTestCase(testcls)
            except Exception:
                self.logger.exception('Could not load tests', exc_info=True)
                run = False
            else:
                run = all_tests.countTestCases()

            if run:
                self.logger.info('Running Tests from %s', testcls)
                runner.startTestClass(testcls)
                ensure_future(self._run_testcls(testcls, all_tests),
                              loop=self._loop)
            else:
                self._loop.call_soon(self._next)
        else:
            time_taken = self._loop.time() - self._time_start
            runner = self.runner
            runner.on_end()
            runner.printSummary(time_taken)
            if runner.result.errors or runner.result.failures:
                exit_code = 2
            else:
                exit_code = 0
            self._loop.call_soon(self._exit, exit_code)
Пример #2
0
 def on_open(self, websocket):
     qid = websocket.handshake.qid
     channel = self.channel % qid
     self.pubsub.add_client(self.client(websocket, channel))
     ensure_future(self.pubsub.subscribe(channel))
     logging.info('New websocket opened. Add client to %s on "%s" channel',
                  self.pubsub, self.channel)
Пример #3
0
    def emit(self, record):
        """Emit record to slack channel using pycurl to avoid recurrence
        event logging (log logged record)
        """
        if record.message.startswith(MESSAGE):   # avoid cyrcular emit
            return

        cfg = self.app.config
        managers = cfg['SLACK_LINK_NAMES']
        text = ''
        data = {}
        if managers:
            text = ' '.join(('@%s' % m for m in managers))
            text = '%s\n\n' % text
            data['link_names'] = 1
        context_factory = cfg['LOG_CONTEXT_FACTORY']
        data['text'] = text
        if context_factory:
            ctx = context_factory(self)
            data['text'] += "\n" + context_text_formatter(ctx)
        data['text'] += "```\n%s\n```" % self.format(record)
        sessions = self.app.http()
        response = sessions.post(self.webhook_url,
                                 data=json.dumps(data))
        if isawaitable(response):
            ensure_future(self._emit(response), loop=sessions._loop)
        else:
            sessions._loop.call_soon(self._raise_error, response)
Пример #4
0
    def _next_class(self, tag, test_classes):
        try:
            test_cls, test_fun = next(test_classes)
        except StopIteration:
            return self._loop.call_soon(self._next_file)

        test_cls.tag = tag
        test_cls.cfg = self.cfg
        test_cls.wait = AsyncAssert(test_cls)
        try:
            all_tests = self.runner.loadTestsFromTestCase(test_cls)
        except Exception:
            self.logger.exception('Could not load tests')
            all_tests = False
        else:
            all_tests = list(all_tests)
            if test_fun:
                all_tests, funcs = False, all_tests
                for fun in funcs:
                    if fun._testMethodName == test_fun:
                        all_tests = [fun]
                        break

        if all_tests:
            self.logger.info('Running %d Tests from %s.%s',
                             len(all_tests), tag, test_cls.__name__)
            self.runner.startTestClass(test_cls)
            coro = self._run_test_cls(test_cls, test_classes, all_tests)
            ensure_future(coro, loop=self._loop)
        else:
            self._loop.call_soon(self._next_class, tag, test_classes)
Пример #5
0
 def __call__(self, environ, start_response):
     uri = environ['RAW_URI']
     logger.debug('new request for %r' % uri)
     if not uri or uri.startswith('/'):  # No proper uri, raise 404
         raise HttpException(status=404)
     response = TunnelResponse(self, environ, start_response)
     ensure_future(response.request())
     return response.future
Пример #6
0
 def _poll_tasks(self, worker, next_time=None):
     if self.closing() and not self._concurrent_tasks:
         self.do_close()
     elif worker.is_running() and not next_time:
         ensure_future(self._may_poll_task(worker), loop=worker._loop)
     else:
         next_time = next_time or 0
         worker._loop.call_later(next_time, self._poll_tasks, worker)
Пример #7
0
 def __call__(self, environ, start_response):
     uri = environ['RAW_URI']
     logger.debug('new request for %r' % uri)
     if not uri or uri.startswith('/'):  # No proper uri, raise 404
         raise HttpException(status=404)
     response = TunnelResponse(self, environ, start_response)
     ensure_future(response.request())
     return response.future
Пример #8
0
 def _poll_tasks(self, worker, next_time=None):
     if self.closing() and not self._concurrent_tasks:
         self.do_close()
     elif worker.is_running() and not next_time:
         ensure_future(self._may_poll_task(worker), loop=worker._loop)
     else:
         next_time = next_time or 0
         worker._loop.call_later(next_time, self._poll_tasks, worker)
Пример #9
0
 def tick(self):
     if self.closing():
         if self.accounts:
             accounts = self.accounts
             self.accounts = None
             self.logger.info('Closing twitter accounts')
             ensure_future(close(self, accounts), loop=self._loop)
         else:
             self.do_close()
Пример #10
0
    async def _may_poll_task(self, worker):
        # Called in the ``worker`` event loop.
        #
        # It pools a new task if possible, and add it to the queue of
        # tasks consumed by the ``worker`` CPU-bound thread.'''
        task = None
        next_time = None
        lag = 0
        if worker.is_running():
            loop = worker._loop

            if self.num_concurrent_tasks < self.max_concurrent_tasks:
                max_tasks = self.cfg.max_requests
                if max_tasks and self._processed >= max_tasks:
                    self.backend.close(
                        'Processed %s tasks. Stop polling tasks.' %
                        self._processed)

                if not self.closing():
                    try:
                        t0 = loop.time()
                        task = await self.broker.get_message(*self.queues())
                        lag = loop.time() - t0
                    except ConnectionError:
                        if self.broker.connection_error:
                            next_time = backoff(self._next_time)
                        else:
                            next_time = RECONNECT_LAG
                            self.broker.connection_error = True
                        if worker.is_running():
                            self.logger.critical(
                                '%s cannot pool messages - '
                                'connection error - try again in %s seconds',
                                self.broker, next_time)
                    except CANCELLED_ERRORS:
                        self.logger.debug('stopped polling messages')
                        raise
                    except Exception:
                        if worker.is_running():
                            self.logger.exception('server exception')
                    else:
                        self.broker.connection_ok()
                    if task:  # Got a new task
                        self._processed += 1
                        self._concurrent_tasks[task.id] = TaskExecutor(task)
                        ensure_future(self._execute_task(task, worker))
            else:
                self.logger.debug('%s concurrent messages. Cannot poll.',
                                  self.max_concurrent_tasks)

            if next_time is None:
                next_time = poll_time(
                    self.cfg.task_pool_timeout, self.cfg.task_pool_timeout_max,
                    self.num_concurrent_tasks / self.max_concurrent_tasks, lag)
            self._next_time = next_time

        self._poll_tasks(worker, next_time)
Пример #11
0
 def _check_queue(self):
     # Run in the main greenlet of the event-loop thread
     if not self._adjust_greenlet_count():
         self.logger.debug("No greenlet available")
         return self._loop.call_soon(self._check_queue)
     try:
         task = self._queue.pop()
     except IndexError:
         return
     ensure_future(self._green_task(self._available.pop(), task), loop=self._loop)
Пример #12
0
 def write_body(self, transport):
     assert not self._write_done, 'Body already sent'
     self._write_done = True
     if not self.body:
         return
     if is_streamed(self.body):
         ensure_future(self._write_streamed_data(transport),
                       loop=self._loop)
     else:
         self._write_body_data(transport, self.body, True)
Пример #13
0
 def write_body(self, transport):
     assert not self._write_done, 'Body already sent'
     self._write_done = True
     if not self.data:
         return
     if is_streamed(self.data):
         ensure_future(self._write_streamed_data(transport),
                       loop=transport._loop)
     else:
         self._write_body_data(transport, self.data, True)
Пример #14
0
 def _check_queue(self):
     # Run in the main greenlet of the event-loop thread
     if not self._adjust_greenlet_count():
         self.logger.debug('No greenlet available')
         return self._loop.call_soon(self._check_queue)
     try:
         task = self._queue.pop()
     except IndexError:
         return
     ensure_future(self._green_task(self._available.pop(), task),
                   loop=self._loop)
Пример #15
0
 def test_blocking_timeout_acquire(self):
     key = self.randomkey()
     eq = self.wait.assertEqual
     lock1 = self.client.lock(key)
     lock2 = self.client.lock(key, blocking=5)
     yield from eq(lock1.acquire(), True)
     ensure_future(self._release(lock1, 0.5))
     start = lock2._loop.time()
     yield from eq(lock2.acquire(), True)
     self.assertTrue(5 > lock2._loop.time() - start > 0.5)
     yield from eq(lock2.release(), True)
Пример #16
0
 async def test_blocking_timeout_acquire(self):
     key = self.randomkey()
     eq = self.assertEqual
     lock1 = self.client.lock(key)
     lock2 = self.client.lock(key, blocking=5)
     eq(await lock1.acquire(), True)
     ensure_future(self._release(lock1, 0.5))
     start = lock2._loop.time()
     eq(await lock2.acquire(), True)
     self.assertTrue(5 > lock2._loop.time() - start > 0.5)
     eq(await lock2.release(), True)
Пример #17
0
 def connect(self):
     '''Connect to twitter streaming endpoint.
     If the connection is dropped, the :meth:`reconnect` method is invoked
     according to twitter streaming connection policy_.
     '''
     ensure_future(self.consumer.http.post(
         self.public_stream,
         data=self.filter,
         pre_request=self.oauth,
         on_headers=self._connected,
         data_processed=self._process_data,
         post_request=self._reconnect),
                   loop=self._loop)
Пример #18
0
 def worker_start(self, worker, exc=None):
     '''When the worker starts :meth:`connect` to twitter spreaming
     endpoint
     '''
     api_key = self.get_param('twitter_api_key')
     client_secret = self.get_param('twitter_api_secret')
     access_token = self.get_param('twitter_access_token')
     access_secret = self.get_param('twitter_access_secret')
     self._http = HttpClient(encode_multipart=False)
     oauth1 = OAuth1(api_key,
                     client_secret=client_secret,
                     resource_owner_key=access_token,
                     resource_owner_secret=access_secret)
     self._http.bind_event('pre_request', oauth1)
     self.buffer = []
     ensure_future(self.connect())
Пример #19
0
 def worker_start(self, worker, exc=None):
     '''When the worker starts :meth:`connect` to twitter spreaming
     endpoint
     '''
     api_key = self.get_param('twitter_api_key')
     client_secret = self.get_param('twitter_api_secret')
     access_token = self.get_param('twitter_access_token')
     access_secret = self.get_param('twitter_access_secret')
     self._http = HttpClient(encode_multipart=False)
     oauth1 = OAuth1(api_key,
                     client_secret=client_secret,
                     resource_owner_key=access_token,
                     resource_owner_secret=access_secret)
     self._http.bind_event('pre_request', oauth1)
     self.buffer = []
     ensure_future(self.connect())
Пример #20
0
    def setup(self, environ):
        '''Called once only to setup the WSGI application handler.

        Check :ref:`lazy wsgi handler <wsgi-lazy-handler>`
        section for further information.
        '''
        cfg = environ['pulsar.cfg']
        loop = environ['pulsar.connection']._loop
        self.store = create_store(cfg.data_store, loop=loop)
        pubsub = self.store.pubsub(protocol=Protocol())
        channel = '%s_webchat' % self.name
        ensure_future(pubsub.subscribe(channel), loop=loop)
        return WsgiHandler([Router('/', get=self.home_page),
                            WebSocket('/message', Chat(pubsub, channel)),
                            Router('/rpc', post=Rpc(pubsub, channel),
                                   response_content_types=JSON_CONTENT_TYPES)],
                           [AsyncResponseMiddleware,
                            GZipMiddleware(min_length=20)])
Пример #21
0
    def setup(self, environ):
        '''Called once only by the WSGI server.

        It returns a :class:`.WsgiHandler` with three routes:

        * The base route served by the :meth:`home_page` method
        * The websocket route
        * A route for static files
        '''
        cfg = environ['pulsar.cfg']
        # Create the store and the pubsub handler
        self.store = create_store(cfg.data_store)
        pubsub = self.store.pubsub()
        # subscribe to channel
        ensure_future(self.subscribe(pubsub))
        return WsgiHandler([Router('/', get=self.home_page),
                            MediaRouter('/static', STATIC_DIR),
                            WebSocket('/message',
                                      TweetsWsHandler(pubsub, self.channel))])
Пример #22
0
    def parse(self, mem_limit=None, **kw):
        if self.content_length > self.limit:
            raise_large_body_error(self.limit)
        inp = self.environ.get('wsgi.input') or BytesIO()
        data = inp.read()

        if isawaitable(data):
            return ensure_future(self._async(data))
        else:
            return self._ready(data)
Пример #23
0
    def setup(self, environ):
        '''Called once only by the WSGI server.

        It returns a :class:`.WsgiHandler` with three routes:

        * The base route served by the :meth:`home_page` method
        * The websocket route
        * A route for static files
        '''
        cfg = environ['pulsar.cfg']
        # Create the store and the pubsub handler
        self.store = create_store(cfg.data_store)
        pubsub = self.store.pubsub()
        # subscribe to channel
        ensure_future(self.subscribe(pubsub))
        return WsgiHandler([Router('/', get=self.home_page),
                            MediaRouter('/static', STATIC_DIR),
                            WebSocket('/message',
                                      TweetsWsHandler(pubsub, self.channel))])
Пример #24
0
    def parse(self, mem_limit=None, **kw):
        if self.content_length > self.limit:
            raise_large_body_error(self.limit)
        inp = self.environ.get("wsgi.input") or BytesIO()
        data = inp.read()

        if isawaitable(data):
            return ensure_future(self._async(data))
        else:
            return self._ready(data)
Пример #25
0
    def parse(self, mem_limit=None, **kw):
        mem_limit = mem_limit or DEFAULT_MAXSIZE
        if self.content_length > mem_limit:
            raise HttpException("Request to big. Increase MAXMEM.",
                                status=LARGE_BODY_CODE)
        inp = self.environ.get('wsgi.input') or BytesIO()
        data = inp.read()

        if isawaitable(data):
            return ensure_future(self._async(data))
        else:
            return self._ready(data)
Пример #26
0
    def parse(self, mem_limit=None, **kw):
        mem_limit = mem_limit or DEFAULT_MAXSIZE
        if self.content_length > mem_limit:
            raise HttpException("Request to big. Increase MAXMEM.",
                                status=LARGE_BODY_CODE)
        inp = self.environ.get('wsgi.input') or BytesIO()
        data = inp.read()

        if isawaitable(data):
            return ensure_future(self._async(data))
        else:
            return self._ready(data)
Пример #27
0
    def parse(self):
        boundary = self.options.get("boundary", "")
        if not valid_boundary(boundary):
            raise HttpException("Invalid boundary for multipart/form-data", status=422)
        inp = self.environ.get("wsgi.input") or BytesIO()
        self.buffer = bytearray()

        if isinstance(inp, HttpBodyReader):
            return ensure_future(self._consume(inp, boundary), loop=inp.reader._loop)
        else:
            producer = BytesProducer(inp)
            return producer(self._consume, boundary)
Пример #28
0
    def parse(self):
        boundary = self.options.get('boundary', '')
        if not valid_boundary(boundary):
            raise HttpException("Invalid boundary for multipart/form-data",
                                status=422)
        inp = self.environ.get('wsgi.input') or BytesIO()
        self.buffer = bytearray()

        if isinstance(inp, HttpBodyReader):
            return ensure_future(self._consume(inp, boundary),
                                 loop=inp.reader._loop)
        else:
            producer = BytesProducer(inp)
            return producer(self._consume, boundary)
Пример #29
0
 def __init__(self, consumer, api_key, api_secret, account):
     secret = account.get("access_token_secret")
     self.processed = 0
     self.consumer = consumer
     self.logger = consumer.logger
     self.token = account.get("access_token")
     self.filter = account.get("stream_filter")
     self.oauth = OAuth1(api_key,
                         client_secret=api_secret,
                         resource_owner_key=self.token,
                         resource_owner_secret=secret)
     self._queue = Queue(loop=self._loop)
     self._buffer = []
     self._worker = ensure_future(self._queue_worker(), loop=self._loop)
Пример #30
0
 def __iter__(self):
     '''Iterator over bytes or Futures resulting in bytes
     '''
     if self._streamed:
         raise StreamConsumedError
     self._streamed = True
     self(self._response)
     while True:
         if self.done:
             try:
                 yield self._queue.get_nowait()
             except asyncio.QueueEmpty:
                 break
         else:
             yield ensure_future(self._queue.get())
Пример #31
0
    def _next_class(self, tag, test_classes):
        try:
            test_cls = next(test_classes)
        except StopIteration:
            return self._loop.call_soon(self._next_file)

        test_cls.tag = tag
        test_cls.cfg = self.cfg
        test_cls.wait = AsyncAssert(test_cls)
        try:
            all_tests = self.runner.loadTestsFromTestCase(test_cls)
        except Exception:
            self.logger.exception('Could not load tests')
            run = False
        else:
            run = all_tests.countTestCases()

        if run:
            self.logger.info('Running Tests from %s', test_cls)
            self.runner.startTestClass(test_cls)
            coro = self._run_test_cls(test_cls, test_classes, all_tests)
            ensure_future(coro, loop=self._loop)
        else:
            self._loop.call_soon(self._next_class, tag, test_classes)
Пример #32
0
    def _next_class(self, tag, test_classes):
        try:
            test_cls = next(test_classes)
        except StopIteration:
            return self._loop.call_soon(self._next_file)

        test_cls.tag = tag
        test_cls.cfg = self.cfg
        test_cls.wait = AsyncAssert(test_cls)
        try:
            all_tests = self.runner.loadTestsFromTestCase(test_cls)
        except Exception:
            self.logger.exception('Could not load tests')
            run = False
        else:
            run = all_tests.countTestCases()

        if run:
            self.logger.info('Running Tests from %s', test_cls)
            self.runner.startTestClass(test_cls)
            coro = self._run_test_cls(test_cls, test_classes, all_tests)
            ensure_future(coro, loop=self._loop)
        else:
            self._loop.call_soon(self._next_file)
Пример #33
0
 async def worker_start(self, worker, exc=None):
     if not exc:
         try:
             print(self.script)
             executor = await self.executor(loop=worker._loop)
         except Exception:
             self.logger.exception('Could not initialise')
             worker._loop.call_soon(self.done, 2)
         else:
             if executor.cfg.list_commands:
                 executed = executor.list_commands()
             elif executor.cfg.environ:
                 executed = executor.show_environ()
             else:
                 executed = executor.run()
             fut = ensure_future(executed, loop=worker._loop)
             fut.add_done_callback(self._exit)
Пример #34
0
 async def worker_start(self, worker, exc=None):
     if not exc:
         try:
             print(self.script)
             executor = await self.executor(loop=worker._loop)
         except Exception:
             self.logger.exception('Could not initialise')
             worker._loop.call_soon(self.done, 2)
         else:
             if executor.cfg.list_commands:
                 executed = executor.list_commands()
             elif executor.cfg.environ:
                 executed = executor.show_environ()
             else:
                 executed = executor.run()
             fut = ensure_future(executed, loop=worker._loop)
             fut.add_done_callback(self._exit)
Пример #35
0
    def close(self, msg=None):
        '''Close this :class:`.TaskBackend`.

        Invoked by the :class:`.Actor` when stopping.
        '''
        if not self._closing_waiter:
            if msg:
                self.logger.warning(msg)
            closing = []
            for consumer in self.consumers:
                result = consumer.close()
                if not result.done():
                    closing.append(result)

            self._closing_waiter = ensure_future(_close(
                self, closing, self._loop),
                                                 loop=self._loop)
        return self._closing_waiter
Пример #36
0
    def __call__(self, environ, start_response):
        """The WSGI callable"""
        if self.async:
            return ensure_future(self._async(environ, start_response))
        response = None
        try:
            for middleware in self.middleware:
                response = middleware(environ, start_response)
                if response is not None:
                    break
            if response is None:
                raise Http404

        except Exception as exc:
            response = handle_wsgi_error(environ, exc)

        if isinstance(response, WsgiResponse) and not response.started:
            for middleware in self.response_middleware:
                response = middleware(environ, response) or response
            response.start(start_response)
        return response
Пример #37
0
 def __call__(self, twitter, messages):
     if not self.store:
         self.store = create_store(twitter.cfg.data_store)
         self.pubsub = self.store.pubsub()
     ensure_future(self._publish(messages))
Пример #38
0
    async def _may_poll_task(self, worker):
        # Called in the ``worker`` event loop.
        #
        # It pools a new task if possible, and add it to the queue of
        # tasks consumed by the ``worker`` CPU-bound thread.'''
        task = None
        next_time = None
        lag = 0
        if worker.is_running():
            loop = worker._loop

            if self.num_concurrent_tasks < self.max_concurrent_tasks:
                max_tasks = self.cfg.max_requests
                if max_tasks and self._processed >= max_tasks:
                    self.backend.close(
                        'Processed %s tasks. Stop polling tasks.'
                        % self._processed
                    )

                if not self.closing():
                    try:
                        t0 = loop.time()
                        task = await self.broker.get_message(*self.queues())
                        lag = loop.time() - t0
                    except ConnectionError:
                        if self.broker.connection_error:
                            next_time = backoff(self._next_time)
                        else:
                            next_time = RECONNECT_LAG
                            self.broker.connection_error = True
                        if worker.is_running():
                            self.logger.critical(
                                '%s cannot pool messages - '
                                'connection error - try again in %s seconds',
                                self.broker,
                                next_time
                            )
                    except CANCELLED_ERRORS:
                        self.logger.debug('stopped polling messages')
                        raise
                    except Exception:
                        if worker.is_running():
                            self.logger.exception('server exception')
                    else:
                        self.broker.connection_ok()
                    if task:  # Got a new task
                        self._processed += 1
                        self._concurrent_tasks[task.id] = TaskExecutor(task)
                        ensure_future(self._execute_task(task, worker))
            else:
                self.logger.debug('%s concurrent messages. Cannot poll.',
                                  self.max_concurrent_tasks)

            if next_time is None:
                next_time = poll_time(
                    self.cfg.task_pool_timeout,
                    self.cfg.task_pool_timeout_max,
                    self.num_concurrent_tasks/self.max_concurrent_tasks,
                    lag
                )
            self._next_time = next_time

        self._poll_tasks(worker, next_time)
Пример #39
0
 def __call__(self, request):
     return ensure_future(self._execute_request(request))
Пример #40
0
 def __call__(self, request):
     return ensure_future(self._execute_request(request))
Пример #41
0
 def __call__(self, twitter, messages):
     if not self.store:
         self.store = create_store(twitter.cfg.data_store)
         self.pubsub = self.store.pubsub()
     ensure_future(self._publish(messages))
Пример #42
0
 def put(self, request):
     return ensure_future(self._async_put(request))
Пример #43
0
 def test_coroutine(self):
     future = ensure_future(main(self.loop, 1), loop=self.loop)
     self.loop.run_until_complete(future)
     self.assertEqual(future.result(), 9)
Пример #44
0
 def start_working(self, worker):
     """Don't be lazy"""
     pulsar.ensure_future(self.run(worker))
Пример #45
0
 def start_working(self, worker):
     """Don't be lazy"""
     pulsar.ensure_future(self.run(worker))
Пример #46
0
def start(arbiter, **kw):
    ensure_future(app(arbiter))
Пример #47
0
 def put(self, request):
     return ensure_future(self._async_put(request))