def __init__(self, log_dir: str, debug: bool = False):
self.log_dir = log_dir
self.debug = debug
self.data = self.load_data()
self.update_lock = locks.Condition()
handlers: tornado.routing._RuleList = [
(r"/api/(.*)", ApiHandler, {
"app": self
}),
(r"/data/(.*)", tornado.web.StaticFileHandler, {
"path": log_dir
}),
# Server Sent Events (SSE) to push new data to the client
(r"/events", EventHandler, {
"app": self
}),
# Those are the static files shipped with the package, i.e. the frontend
(
r"/(.*)",
FrontendFileHandler,
{
"path": package_dir / "static",
"default_filename": "index.html",
},
),
]
self.file_watcher = FileWatcher(self.log_dir, self.data,
self.update_lock)
super().__init__(handlers, debug=debug, compress_response=True)
def __init__(self, stream, server):
self.name = None
self.is_alive = True
self.is_ready = False
self.stream = stream
self.message_queue = queues.Queue()
self.server = server
self.inital_data = None
self.inital_data_loaded = locks.Condition()
self.handlers = {
(message_type.CHAT, chat_message.BROADCAST):
self.broadcast_chat,
(message_type.LOBBY, lobby_message.READY):
self.ready,
(message_type.LOBBY, lobby_message.START_GAME):
self.load_inital_data,
(message_type.LOBBY, lobby_message.UNREADY):
self.unready,
(message_type.GAME, game_message.READY):
self.game_ready,
(message_type.GAME, game_message.UPDATE):
self.send_to_game((message_type.GAME, game_message.UPDATE)),
(message_type.GAME, game_message.CALL):
self.send_to_game((message_type.GAME, game_message.CALL)),
(message_type.GAME, game_message.ACTION_APPEND):
self.send_to_game((message_type.GAME, game_message.ACTION_APPEND)),
(message_type.GAME, game_message.ACTION_REMOVE):
self.send_to_game((message_type.GAME, game_message.ACTION_REMOVE)),
}
def __init__(self, *args, **kwargs):
"""Creates a new instance of the :class:`~rejected.consumer.Consumer`
class.
"""
self._confirmation_futures = {}
self._connections = {}
self._correlation_id = None
self._drop_exchange = kwargs.get('drop_exchange') or self.DROP_EXCHANGE
self._drop_invalid = (kwargs.get('drop_invalid_messages')
or self.DROP_INVALID_MESSAGES)
self._error_exchange = (kwargs.get('error_exchange')
or self.ERROR_EXCHANGE)
self._error_max_retry = (kwargs.get('error_max_retry')
or self.ERROR_MAX_RETRIES
or self.ERROR_MAX_RETRY)
self._finished = False
self._message = None
self._message_type = kwargs.get('message_type') or self.MESSAGE_TYPE
self._measurement = None
self._message_body = None
self._process = kwargs['process']
self._settings = kwargs['settings']
self._yield_condition = locks.Condition()
# Create a logger that attaches correlation ID to the record
logger = logging.getLogger(
kwargs.get('settings').get('_import_module', __name__))
self.logger = log.CorrelationIDAdapter(logger, {'parent': self})
# Set a Sentry context for the consumer
self.set_sentry_context('consumer', self.name)
# Run any child object specified initialization
self.initialize()
def __init__(self, settings, process):
"""Creates a new instance of a Consumer class. To perform
initialization tasks, extend Consumer.initialize
:param dict settings: The configuration from rejected
:param rejected.process.Process: The controlling process
"""
self._channel = None
self._finished = False
self._message = None
self._message_body = None
self._process = process
self._settings = settings
self._statsd = None
self._yield_condition = locks.Condition()
# Create a logger that attaches correlation ID to the record
self._logger = logging.getLogger(settings.get('_import_module',
__name__))
self.logger = log.CorrelationAdapter(self._logger, self)
# Set a Sentry context for the consumer
self.set_sentry_context('consumer', self.name)
try:
self.set_sentry_context('version',
sys.modules[self.__module__].__version__)
except (NameError, AttributeError):
pass
# Run any child object specified initialization
self.initialize()
def test_wait_timeout_preempted(self):
c = locks.Condition()
# This fires before the wait times out.
self.io_loop.call_later(0.01, c.notify)
wait = c.wait(timedelta(seconds=0.02))
yield gen.sleep(0.03)
yield wait # No TimeoutError.
def test_notify_1(self):
c = locks.Condition()
self.record_done(c.wait(), 'wait1')
self.record_done(c.wait(), 'wait2')
c.notify(1)
self.history.append('notify1')
c.notify(1)
self.history.append('notify2')
self.assertEqual(['wait1', 'notify1', 'wait2', 'notify2'],
self.history)
def test_notify_all(self):
c = locks.Condition()
for i in range(4):
self.record_done(c.wait(), i)
c.notify_all()
self.history.append('notify_all')
# Callbacks execute in the order they were registered.
self.assertEqual(list(range(4)) + ['notify_all'], self.history)
def test_notify_all(self):
c = locks.Condition()
for i in range(4):
self.record_done(c.wait(), i)
c.notify_all()
self.loop_briefly()
self.history.append("notify_all")
# Callbacks execute in the order they were registered.
self.assertEqual(list(range(4)) + ["notify_all"], self.history) # type: ignore
def test_notify_1(self):
c = locks.Condition()
self.record_done(c.wait(), "wait1")
self.record_done(c.wait(), "wait2")
c.notify(1)
self.loop_briefly()
self.history.append("notify1")
c.notify(1)
self.loop_briefly()
self.history.append("notify2")
self.assertEqual(["wait1", "notify1", "wait2", "notify2"], self.history)
def test_notify_all_with_timeout(self):
c = locks.Condition()
self.record_done(c.wait(), 0)
self.record_done(c.wait(timedelta(seconds=0.01)), 1)
self.record_done(c.wait(), 2)
# Wait for callback 1 to time out.
yield gen.sleep(0.02)
self.assertEqual(['timeout'], self.history)
c.notify_all()
self.assertEqual(['timeout', 0, 2], self.history)
def __init__(
self, ctx, committed_state, unicorn, async_route_provider,
**kwargs):
super(FeedbackSubmitter, self).__init__(ctx, **kwargs)
self._config = ctx.config
self._ci_state = committed_state
self._dumper = Dumper(ctx, unicorn)
self._async_route_provider = async_route_provider
self._condition = locks.Condition()
def test_nested_notify(self):
# Ensure no notifications lost, even if notify() is reentered by a
# waiter calling notify().
c = locks.Condition()
# Three waiters.
futures = [c.wait() for _ in range(3)]
# First and second futures resolved. Second future reenters notify(),
# resolving third future.
futures[1].add_done_callback(lambda _: c.notify())
c.notify(2)
self.assertTrue(all(f.done() for f in futures))
def test_notify_n(self):
c = locks.Condition()
for i in range(6):
self.record_done(c.wait(), i)
c.notify(3)
# Callbacks execute in the order they were registered.
self.assertEqual(list(range(3)), self.history)
c.notify(1)
self.assertEqual(list(range(4)), self.history)
c.notify(2)
self.assertEqual(list(range(6)), self.history)
def test_garbage_collection(self):
# Test that timed-out waiters are occasionally cleaned from the queue.
c = locks.Condition()
for _ in range(101):
c.wait(timedelta(seconds=0.01))
future = c.wait()
self.assertEqual(102, len(c._waiters))
# Let first 101 waiters time out, triggering a collection.
yield gen.sleep(0.02)
self.assertEqual(1, len(c._waiters))
# Final waiter is still active.
self.assertFalse(future.done())
c.notify()
self.assertTrue(future.done())
def setUp(self):
super(BaseTestCase, self).setUp()
# make sure that our logging statements get executed
amqp.amqp.LOGGER.enabled = True
amqp.amqp.LOGGER.setLevel(logging.DEBUG)
amqp.mixins.LOGGER.enabled = True
amqp.mixins.LOGGER.setLevel(logging.DEBUG)
self.exchange = str(uuid.uuid4())
self.queue = str(uuid.uuid4())
self.routing_key = str(uuid.uuid4())
self.correlation_id = str(uuid.uuid4())
self.message = None
self.test_queue_bound = locks.Event()
self.get_response = locks.Event()
self.amqp_ready = locks.Event()
self.condition = locks.Condition()
self.config = {
"url": AMQP_URL,
"reconnect_delay": 1,
"timeout": 2,
"on_ready_callback": self.on_ready,
"on_unavailable_callback": self.on_unavailable,
"on_persistent_failure_callback": self.on_persistent_failure,
"on_message_returned_callback": self.on_message_returned,
"io_loop": self.io_loop,
}
self.app = web.Application()
self.app.settings = {'service': 'unit_tests', 'version': '0.0'}
self.handler = TestRequestHandler(self.app)
self.clear_event_tracking()
amqp.install(self.app, **self.config)
yield self.condition.wait(self.io_loop.time() + 5)
LOGGER.info('Connected to RabbitMQ, declaring exchange %s',
self.exchange)
self.app.amqp.channel.exchange_declare(self.on_exchange_declare_ok,
self.exchange,
auto_delete=True)
def test_notify_n_with_timeout(self):
# Register callbacks 0, 1, 2, and 3. Callback 1 has a timeout.
# Wait for that timeout to expire, then do notify(2) and make
# sure everyone runs. Verifies that a timed-out callback does
# not count against the 'n' argument to notify().
c = locks.Condition()
self.record_done(c.wait(), 0)
self.record_done(c.wait(timedelta(seconds=0.01)), 1)
self.record_done(c.wait(), 2)
self.record_done(c.wait(), 3)
# Wait for callback 1 to time out.
yield gen.sleep(0.02)
self.assertEqual(['timeout'], self.history)
c.notify(2)
yield gen.sleep(0.01)
self.assertEqual(['timeout', 0, 2], self.history)
self.assertEqual(['timeout', 0, 2], self.history)
c.notify()
self.assertEqual(['timeout', 0, 2, 3], self.history)
def test_wait_timeout(self):
c = locks.Condition()
self.assertFalse((yield c.wait(timedelta(seconds=0.01))))
def __init__(self):
self.waiters = 0
self.lock = locks.Lock()
self.condition = locks.Condition()
def test_wait_timeout(self):
c = locks.Condition()
wait = c.wait(timedelta(seconds=0.01))
self.io_loop.call_later(0.02, c.notify) # Too late.
yield gen.sleep(0.03)
self.assertFalse((yield wait))
def test_wait_timeout_preempted(self):
c = locks.Condition()
# This fires before the wait times out.
self.io_loop.call_later(0.01, c.notify)
yield c.wait(timedelta(seconds=1))
def test_repr(self):
c = locks.Condition()
self.assertIn("Condition", repr(c))
self.assertNotIn("waiters", repr(c))
c.wait()
self.assertIn("waiters", repr(c))
def get_data(cls, account, source_filter, limit=100, skip=0):
source_filter = BoxFileFilter(source_filter)
if source_filter.file is None:
raise ValueError('required parameter file missing')
app_log.info("Starting to retrieve file {} => {}".format(
source_filter.file, account._id))
client = AsyncHTTPClient()
uri = "https://api.box.com/2.0/files/{}/content".format(
source_filter.file.lstrip('/'))
lock = locks.Condition()
def crawl_url(url):
# some yummy regex
location_header_regex = re.compile(
r'^Location:\s?(?P<uri>http:/{2}\S+)')
http_status_regex = re.compile(r'^HTTP/[\d\.]+\s(?P<status>\d+)')
receiving_file = False
# define our callbacks
def header_callback(header):
m = http_status_regex.match(header)
if m is not None:
# process our HTTP status header
status = m.group('status')
if int(status) == 200:
# if we're 200, we're receiving the file, not just a redirect
app_log.info(
"Retrieving file {} for account {}".format(
source_filter.file, account._id))
global receiving_file
receiving_file = True
m = location_header_regex.match(header)
if m is not None:
# process our location header
uri = m.group('uri')
# and grab _that_ url
app_log.info("Following redirect for file {}".format(
source_filter.file))
crawl_url(uri)
def stream_callback(chunk):
# only dump out chunks that are of the file we're looking for
global receiving_file
if receiving_file:
cls.write(chunk)
def on_completed(resp):
if 200 <= resp.code <= 299:
lock.notify()
oauth_client = account.get_client()
uri, headers, body = oauth_client.add_token(url)
req = HTTPRequest(uri,
headers=headers,
body=body,
header_callback=header_callback,
streaming_callback=stream_callback)
client.fetch(req, callback=on_completed)
crawl_url(uri)
# wait for us to complete
try:
yield lock.wait(timeout=timedelta(seconds=30))
app_log.info("File {} retrieved successfully".format(
source_filter.file))
except gen.TimeoutError:
app_log.error("Request for file {} => {} timed out!".format(
source_filter.file, account._id))
def __init__(self,
url,
timeout=None,
reconnect_delay=None,
connection_attempts=None,
on_ready_callback=None,
on_unavailable_callback=None,
on_persistent_failure_callback=None,
on_message_returned_callback=None,
io_loop=None):
"""Create a new instance of the consumer class, passing in the AMQP
URL used to connect to RabbitMQ.
:param str url: The AMQP URL to connect to.
:param int timeout: The optional maximum time to wait for a bad state
to resolve before treating the failure as
persistent.
:param int reconnect_delay: The optional time in seconds to wait before
reconnecting on connection failure.
:param int connection_attempts: The optional number of connection
attempts to make before giving up.
:param callable on_ready_callback: The optional callback to call when
the connection to the AMQP server
has been established and is ready.
:param callable on_unavailable_callback: The optional callback to call
when the connection to the
AMQP server becomes
unavailable.
:param callable on_persistent_failure_callback: The optional callback
to call when the
connection failure does
not resolve itself
within the timeout.
:param callable on_message_returned_callback: The optional callback
to call when the AMQP
server returns a message.
:param tornado.ioloop.IOLoop io_loop: An optional IOLoop to override
the default with.
:raises AttributeError: If timeout <= reconnect_delay
"""
self._channel = None
self._ioloop = ioloop.IOLoop.current(io_loop)
self._on_ready = on_ready_callback
self._on_unavailable = on_unavailable_callback
self._on_persistent_failure = on_persistent_failure_callback
self.on_message_returned = on_message_returned_callback
self._reconnect_delay = reconnect_delay or self.DEFAULT_RECONNECT_DELAY
self._timeout = timeout or self.DEFAULT_TIMEOUT
self._connection_attempts = (connection_attempts
or self.DEFAULT_CONNECTION_ATTEMPTS)
self._schemas = {}
self._state = self.STATE_IDLE
self._url = url
self._ready_condition = locks.Condition()
if self._timeout <= self._reconnect_delay:
raise AttributeError('Reconnection must be less than timeout')
# Automatically start the RabbitMQ connection on creation
self._connection = self.connect()
def test_notify(self):
c = locks.Condition()
self.io_loop.call_later(0.01, c.notify)
yield c.wait()
def test_repr(self):
c = locks.Condition()
self.assertIn('Condition', repr(c))
self.assertNotIn('waiters', repr(c))
c.wait()
self.assertIn('waiters', repr(c))
