def __init__(self, log_dir: str, debug: bool = False): self.log_dir = log_dir self.debug = debug self.data = self.load_data() self.update_lock = locks.Condition() handlers: tornado.routing._RuleList = [ (r"/api/(.*)", ApiHandler, { "app": self }), (r"/data/(.*)", tornado.web.StaticFileHandler, { "path": log_dir }), # Server Sent Events (SSE) to push new data to the client (r"/events", EventHandler, { "app": self }), # Those are the static files shipped with the package, i.e. the frontend ( r"/(.*)", FrontendFileHandler, { "path": package_dir / "static", "default_filename": "index.html", }, ), ] self.file_watcher = FileWatcher(self.log_dir, self.data, self.update_lock) super().__init__(handlers, debug=debug, compress_response=True)
def __init__(self, stream, server): self.name = None self.is_alive = True self.is_ready = False self.stream = stream self.message_queue = queues.Queue() self.server = server self.inital_data = None self.inital_data_loaded = locks.Condition() self.handlers = { (message_type.CHAT, chat_message.BROADCAST): self.broadcast_chat, (message_type.LOBBY, lobby_message.READY): self.ready, (message_type.LOBBY, lobby_message.START_GAME): self.load_inital_data, (message_type.LOBBY, lobby_message.UNREADY): self.unready, (message_type.GAME, game_message.READY): self.game_ready, (message_type.GAME, game_message.UPDATE): self.send_to_game((message_type.GAME, game_message.UPDATE)), (message_type.GAME, game_message.CALL): self.send_to_game((message_type.GAME, game_message.CALL)), (message_type.GAME, game_message.ACTION_APPEND): self.send_to_game((message_type.GAME, game_message.ACTION_APPEND)), (message_type.GAME, game_message.ACTION_REMOVE): self.send_to_game((message_type.GAME, game_message.ACTION_REMOVE)), }
def __init__(self, *args, **kwargs): """Creates a new instance of the :class:`~rejected.consumer.Consumer` class. """ self._confirmation_futures = {} self._connections = {} self._correlation_id = None self._drop_exchange = kwargs.get('drop_exchange') or self.DROP_EXCHANGE self._drop_invalid = (kwargs.get('drop_invalid_messages') or self.DROP_INVALID_MESSAGES) self._error_exchange = (kwargs.get('error_exchange') or self.ERROR_EXCHANGE) self._error_max_retry = (kwargs.get('error_max_retry') or self.ERROR_MAX_RETRIES or self.ERROR_MAX_RETRY) self._finished = False self._message = None self._message_type = kwargs.get('message_type') or self.MESSAGE_TYPE self._measurement = None self._message_body = None self._process = kwargs['process'] self._settings = kwargs['settings'] self._yield_condition = locks.Condition() # Create a logger that attaches correlation ID to the record logger = logging.getLogger( kwargs.get('settings').get('_import_module', __name__)) self.logger = log.CorrelationIDAdapter(logger, {'parent': self}) # Set a Sentry context for the consumer self.set_sentry_context('consumer', self.name) # Run any child object specified initialization self.initialize()
def __init__(self, settings, process): """Creates a new instance of a Consumer class. To perform initialization tasks, extend Consumer.initialize :param dict settings: The configuration from rejected :param rejected.process.Process: The controlling process """ self._channel = None self._finished = False self._message = None self._message_body = None self._process = process self._settings = settings self._statsd = None self._yield_condition = locks.Condition() # Create a logger that attaches correlation ID to the record self._logger = logging.getLogger(settings.get('_import_module', __name__)) self.logger = log.CorrelationAdapter(self._logger, self) # Set a Sentry context for the consumer self.set_sentry_context('consumer', self.name) try: self.set_sentry_context('version', sys.modules[self.__module__].__version__) except (NameError, AttributeError): pass # Run any child object specified initialization self.initialize()
def test_wait_timeout_preempted(self): c = locks.Condition() # This fires before the wait times out. self.io_loop.call_later(0.01, c.notify) wait = c.wait(timedelta(seconds=0.02)) yield gen.sleep(0.03) yield wait # No TimeoutError.
def test_notify_1(self): c = locks.Condition() self.record_done(c.wait(), 'wait1') self.record_done(c.wait(), 'wait2') c.notify(1) self.history.append('notify1') c.notify(1) self.history.append('notify2') self.assertEqual(['wait1', 'notify1', 'wait2', 'notify2'], self.history)
def test_notify_all(self): c = locks.Condition() for i in range(4): self.record_done(c.wait(), i) c.notify_all() self.history.append('notify_all') # Callbacks execute in the order they were registered. self.assertEqual(list(range(4)) + ['notify_all'], self.history)
def test_notify_all(self): c = locks.Condition() for i in range(4): self.record_done(c.wait(), i) c.notify_all() self.loop_briefly() self.history.append("notify_all") # Callbacks execute in the order they were registered. self.assertEqual(list(range(4)) + ["notify_all"], self.history) # type: ignore
def test_notify_1(self): c = locks.Condition() self.record_done(c.wait(), "wait1") self.record_done(c.wait(), "wait2") c.notify(1) self.loop_briefly() self.history.append("notify1") c.notify(1) self.loop_briefly() self.history.append("notify2") self.assertEqual(["wait1", "notify1", "wait2", "notify2"], self.history)
def test_notify_all_with_timeout(self): c = locks.Condition() self.record_done(c.wait(), 0) self.record_done(c.wait(timedelta(seconds=0.01)), 1) self.record_done(c.wait(), 2) # Wait for callback 1 to time out. yield gen.sleep(0.02) self.assertEqual(['timeout'], self.history) c.notify_all() self.assertEqual(['timeout', 0, 2], self.history)
def __init__( self, ctx, committed_state, unicorn, async_route_provider, **kwargs): super(FeedbackSubmitter, self).__init__(ctx, **kwargs) self._config = ctx.config self._ci_state = committed_state self._dumper = Dumper(ctx, unicorn) self._async_route_provider = async_route_provider self._condition = locks.Condition()
def test_nested_notify(self): # Ensure no notifications lost, even if notify() is reentered by a # waiter calling notify(). c = locks.Condition() # Three waiters. futures = [c.wait() for _ in range(3)] # First and second futures resolved. Second future reenters notify(), # resolving third future. futures[1].add_done_callback(lambda _: c.notify()) c.notify(2) self.assertTrue(all(f.done() for f in futures))
def test_notify_n(self): c = locks.Condition() for i in range(6): self.record_done(c.wait(), i) c.notify(3) # Callbacks execute in the order they were registered. self.assertEqual(list(range(3)), self.history) c.notify(1) self.assertEqual(list(range(4)), self.history) c.notify(2) self.assertEqual(list(range(6)), self.history)
def test_garbage_collection(self): # Test that timed-out waiters are occasionally cleaned from the queue. c = locks.Condition() for _ in range(101): c.wait(timedelta(seconds=0.01)) future = c.wait() self.assertEqual(102, len(c._waiters)) # Let first 101 waiters time out, triggering a collection. yield gen.sleep(0.02) self.assertEqual(1, len(c._waiters)) # Final waiter is still active. self.assertFalse(future.done()) c.notify() self.assertTrue(future.done())
def setUp(self): super(BaseTestCase, self).setUp() # make sure that our logging statements get executed amqp.amqp.LOGGER.enabled = True amqp.amqp.LOGGER.setLevel(logging.DEBUG) amqp.mixins.LOGGER.enabled = True amqp.mixins.LOGGER.setLevel(logging.DEBUG) self.exchange = str(uuid.uuid4()) self.queue = str(uuid.uuid4()) self.routing_key = str(uuid.uuid4()) self.correlation_id = str(uuid.uuid4()) self.message = None self.test_queue_bound = locks.Event() self.get_response = locks.Event() self.amqp_ready = locks.Event() self.condition = locks.Condition() self.config = { "url": AMQP_URL, "reconnect_delay": 1, "timeout": 2, "on_ready_callback": self.on_ready, "on_unavailable_callback": self.on_unavailable, "on_persistent_failure_callback": self.on_persistent_failure, "on_message_returned_callback": self.on_message_returned, "io_loop": self.io_loop, } self.app = web.Application() self.app.settings = {'service': 'unit_tests', 'version': '0.0'} self.handler = TestRequestHandler(self.app) self.clear_event_tracking() amqp.install(self.app, **self.config) yield self.condition.wait(self.io_loop.time() + 5) LOGGER.info('Connected to RabbitMQ, declaring exchange %s', self.exchange) self.app.amqp.channel.exchange_declare(self.on_exchange_declare_ok, self.exchange, auto_delete=True)
def test_notify_n_with_timeout(self): # Register callbacks 0, 1, 2, and 3. Callback 1 has a timeout. # Wait for that timeout to expire, then do notify(2) and make # sure everyone runs. Verifies that a timed-out callback does # not count against the 'n' argument to notify(). c = locks.Condition() self.record_done(c.wait(), 0) self.record_done(c.wait(timedelta(seconds=0.01)), 1) self.record_done(c.wait(), 2) self.record_done(c.wait(), 3) # Wait for callback 1 to time out. yield gen.sleep(0.02) self.assertEqual(['timeout'], self.history) c.notify(2) yield gen.sleep(0.01) self.assertEqual(['timeout', 0, 2], self.history) self.assertEqual(['timeout', 0, 2], self.history) c.notify() self.assertEqual(['timeout', 0, 2, 3], self.history)
def test_wait_timeout(self): c = locks.Condition() self.assertFalse((yield c.wait(timedelta(seconds=0.01))))
def __init__(self): self.waiters = 0 self.lock = locks.Lock() self.condition = locks.Condition()
def test_wait_timeout(self): c = locks.Condition() wait = c.wait(timedelta(seconds=0.01)) self.io_loop.call_later(0.02, c.notify) # Too late. yield gen.sleep(0.03) self.assertFalse((yield wait))
def test_wait_timeout_preempted(self): c = locks.Condition() # This fires before the wait times out. self.io_loop.call_later(0.01, c.notify) yield c.wait(timedelta(seconds=1))
def test_repr(self): c = locks.Condition() self.assertIn("Condition", repr(c)) self.assertNotIn("waiters", repr(c)) c.wait() self.assertIn("waiters", repr(c))
def get_data(cls, account, source_filter, limit=100, skip=0): source_filter = BoxFileFilter(source_filter) if source_filter.file is None: raise ValueError('required parameter file missing') app_log.info("Starting to retrieve file {} => {}".format( source_filter.file, account._id)) client = AsyncHTTPClient() uri = "https://api.box.com/2.0/files/{}/content".format( source_filter.file.lstrip('/')) lock = locks.Condition() def crawl_url(url): # some yummy regex location_header_regex = re.compile( r'^Location:\s?(?P<uri>http:/{2}\S+)') http_status_regex = re.compile(r'^HTTP/[\d\.]+\s(?P<status>\d+)') receiving_file = False # define our callbacks def header_callback(header): m = http_status_regex.match(header) if m is not None: # process our HTTP status header status = m.group('status') if int(status) == 200: # if we're 200, we're receiving the file, not just a redirect app_log.info( "Retrieving file {} for account {}".format( source_filter.file, account._id)) global receiving_file receiving_file = True m = location_header_regex.match(header) if m is not None: # process our location header uri = m.group('uri') # and grab _that_ url app_log.info("Following redirect for file {}".format( source_filter.file)) crawl_url(uri) def stream_callback(chunk): # only dump out chunks that are of the file we're looking for global receiving_file if receiving_file: cls.write(chunk) def on_completed(resp): if 200 <= resp.code <= 299: lock.notify() oauth_client = account.get_client() uri, headers, body = oauth_client.add_token(url) req = HTTPRequest(uri, headers=headers, body=body, header_callback=header_callback, streaming_callback=stream_callback) client.fetch(req, callback=on_completed) crawl_url(uri) # wait for us to complete try: yield lock.wait(timeout=timedelta(seconds=30)) app_log.info("File {} retrieved successfully".format( source_filter.file)) except gen.TimeoutError: app_log.error("Request for file {} => {} timed out!".format( source_filter.file, account._id))
def __init__(self, url, timeout=None, reconnect_delay=None, connection_attempts=None, on_ready_callback=None, on_unavailable_callback=None, on_persistent_failure_callback=None, on_message_returned_callback=None, io_loop=None): """Create a new instance of the consumer class, passing in the AMQP URL used to connect to RabbitMQ. :param str url: The AMQP URL to connect to. :param int timeout: The optional maximum time to wait for a bad state to resolve before treating the failure as persistent. :param int reconnect_delay: The optional time in seconds to wait before reconnecting on connection failure. :param int connection_attempts: The optional number of connection attempts to make before giving up. :param callable on_ready_callback: The optional callback to call when the connection to the AMQP server has been established and is ready. :param callable on_unavailable_callback: The optional callback to call when the connection to the AMQP server becomes unavailable. :param callable on_persistent_failure_callback: The optional callback to call when the connection failure does not resolve itself within the timeout. :param callable on_message_returned_callback: The optional callback to call when the AMQP server returns a message. :param tornado.ioloop.IOLoop io_loop: An optional IOLoop to override the default with. :raises AttributeError: If timeout <= reconnect_delay """ self._channel = None self._ioloop = ioloop.IOLoop.current(io_loop) self._on_ready = on_ready_callback self._on_unavailable = on_unavailable_callback self._on_persistent_failure = on_persistent_failure_callback self.on_message_returned = on_message_returned_callback self._reconnect_delay = reconnect_delay or self.DEFAULT_RECONNECT_DELAY self._timeout = timeout or self.DEFAULT_TIMEOUT self._connection_attempts = (connection_attempts or self.DEFAULT_CONNECTION_ATTEMPTS) self._schemas = {} self._state = self.STATE_IDLE self._url = url self._ready_condition = locks.Condition() if self._timeout <= self._reconnect_delay: raise AttributeError('Reconnection must be less than timeout') # Automatically start the RabbitMQ connection on creation self._connection = self.connect()
def test_notify(self): c = locks.Condition() self.io_loop.call_later(0.01, c.notify) yield c.wait()
def test_repr(self): c = locks.Condition() self.assertIn('Condition', repr(c)) self.assertNotIn('waiters', repr(c)) c.wait() self.assertIn('waiters', repr(c))