class AsyncParser:
"""
Create a message parser which decodes details about the provided messages and dispatches
the resulting augmented messages to an output queue.
"""
_logger = logging.getLogger('AsyncParser')
def __init__(self, number_workers=5):
self._worker_q = Queue.Queue()
self.out_q = Queue.Queue()
self._number_workers = number_workers
self._threads = []
# Make a "fast" parser, by simply install a url fetcher that return an empty string.
# (sometimes you just have to love the power of dependency injection :)
self._fastParser = HipChatParser(NullUrlFetcher())
def start(self):
"""
Start pulling messages from the queue and dispatching them to the out queue
"""
self._logger.debug('Starting...')
# In a real app, we would manage these threads more intelligently
self._threads = [self._create_worker(i) for i in range(self._number_workers)]
self._logger.info('Started')
def stop(self):
"""
Shutdown this processor in an orderly fashion
"""
self._logger.debug('Stopping...')
for t in self._threads:
t.join()
self._logger.info('Stopped')
def parse(self, msg):
"""
Parses the given message and send the result to the output queue
"""
self._logger.debug('Parsing: %s', msg)
# Quickly decode the details that we can do without delay
msg.details = self._fastParser.parse_to_dict(msg.text)
msg.details_as_json = self._fastParser.dict_to_json(msg.details)
# Pumps out the message. "slow" details are not yet filled in
self.out_q.put(msg)
# If the message had links, send it to the workers, which will
# produced an updated message once the details are filled in
if HipChatParser.DETAIL_LINKS in msg.details:
self._worker_q.put(msg)
def _create_worker(self, worker_id):
"""
Create and start a worker that will collect more costly message details
"""
w = ParserWorkerThread(worker_id, self._worker_q, self.out_q)
w.start()
return w
