def reserve(cls, reserve_id, reserve_data, block=False, block_timeout=90): if block: cursor_id = messenger.get_cursor_id('queue') doc = cls.collection.find_and_modify({ 'state': PENDING, 'reserve_id': reserve_id, 'reserve_data': None, }, {'$set': { 'reserve_data': reserve_data, }}, new=True) if not doc: return if block: for msg in messenger.subscribe('queue', cursor_id=cursor_id, timeout=block_timeout): try: if msg['message'] == [COMPLETE, doc['_id']]: return doc elif msg['message'] == [ERROR, doc['_id']]: raise QueueTaskError('Error occured running ' + 'queue task', { 'queue_id': doc['_id'], 'queue_type': doc['type'], }) except TypeError: pass logger.error('Blocking queue reserve timed out', { 'queue_id': doc['_id'], 'queue_type': doc['type'], }) return doc else: return doc
def get_events(cursor=None): events = [] events_dict = {} for event in messenger.subscribe('events', cursor_id=cursor, timeout=10, yield_delay=0.02): event_type, resource_id = event.pop('message') if (event_type, resource_id) in events_dict: old_event = events_dict[(event_type, resource_id)] old_event['id'] = event['_id'] old_event['timestamp'] = time.mktime( event['timestamp'].timetuple()) continue events_dict[(event_type, resource_id)] = event event['id'] = event.pop('_id') event['type'] = event_type event['resource_id'] = resource_id event['timestamp'] = time.mktime(event['timestamp'].timetuple()) event.pop('channel') events.append(event) return events
def listener_thread(): queue = callqueue.CallQueue() queue.start() lastlog = utils.now() while True: try: for msg in messenger.subscribe(list(listener.channels.keys())): for lstnr in listener.channels[msg['channel']]: try: queue.put(lstnr, msg) size = queue.size() if size >= 50: if utils.now() - lastlog > datetime.timedelta( minutes=3): lastlog = utils.now() logger.warning( 'Message queue flood', 'runners', size=size, ) except: logger.exception('Error in listener callback', 'runners') except GeneratorExit: raise except: logger.exception('Error in listener thread', 'runners') time.sleep(0.3) yield
def get_events(cursor=None): events = [] events_dict = {} if cursor is not None: cursor = bson.ObjectId(cursor) for event in messenger.subscribe('events', cursor_id=cursor, timeout=10, yield_delay=0.02): event_type, resource_id = event.pop('message') if (event_type, resource_id) in events_dict: old_event = events_dict[(event_type, resource_id)] old_event['id'] = str(event['_id']) old_event['timestamp'] = time.mktime( event['timestamp'].timetuple()) continue events_dict[(event_type, resource_id)] = event event['id'] = str(event.pop('_id')) event['type'] = event_type event['resource_id'] = resource_id event['timestamp'] = time.mktime(event['timestamp'].timetuple()) event.pop('channel') events.append(event) return events
def reserve(cls, reserve_id, reserve_data, block=False, block_timeout=30): if block: cursor_id = messenger.get_cursor_id('queue') doc = cls.collection.find_and_modify({ 'state': PENDING, 'reserve_id': reserve_id, 'reserve_data': None, }, {'$set': { 'reserve_data': reserve_data, }}) if not doc: return if block: for msg in messenger.subscribe('queue', cursor_id=cursor_id, timeout=block_timeout): try: if msg['message'] == [COMPLETE, doc['_id']]: return doc elif msg['message'] == [ERROR, doc['_id']]: raise QueueTaskError('Error occured running ' + 'queue task', { 'queue_id': doc['_id'], 'queue_type': doc['type'], }) except TypeError: pass raise QueueTimeout('Blocking queue reserve timed out', { 'queue_id': doc['_id'], 'queue_type': doc['type'], }) else: return doc
def get_events(cursor=None, yield_app_server=False): events = [] events_dict = {} if yield_app_server and check_app_server_interrupt(): return events for event in messenger.subscribe('events', cursor_id=cursor, timeout=10, yield_delay=0.02, yield_app_server=yield_app_server): event_type, resource_id = event.pop('message') if (event_type, resource_id) in events_dict: old_event = events_dict[(event_type, resource_id)] old_event['id'] = event['_id'] old_event['timestamp'] = time.mktime( event['timestamp'].timetuple()) continue events_dict[(event_type, resource_id)] = event event['id'] = event.pop('_id') event['type'] = event_type event['resource_id'] = resource_id event['timestamp'] = time.mktime(event['timestamp'].timetuple()) event.pop('channel') events.append(event) return events
def listener_thread(): while True: try: for msg in messenger.subscribe(listener.channels.keys()): for lstnr in listener.channels[msg['channel']]: try: lstnr(msg) except: logger.exception('Error in listener callback') except: logger.exception('Error in listener thread') time.sleep(0.3)
def start(self, transaction=None, block=False, block_timeout=30): self.ttl_timestamp = utils.now() + \ datetime.timedelta(seconds=self.ttl) self.commit(transaction=transaction) if block: if transaction: raise TypeError('Cannot use transaction when blocking') cursor_id = messenger.get_cursor_id('queue') extra = {'queue_doc': self.export()} messenger.publish('queue', [PENDING, self.id], extra=extra, transaction=transaction) if block: last_update = time.time() while True: for msg in messenger.subscribe('queue', cursor_id=cursor_id, timeout=block_timeout): cursor_id = msg['_id'] try: if msg['message'] == [COMPLETE, self.id]: return elif msg['message'] == [UPDATE, self.id]: last_update = time.time() break elif msg['message'] == [ERROR, self.id]: raise QueueTaskError( 'Error occured running ' + 'queue task', { 'queue_id': self.id, 'queue_type': self.type, }) except TypeError: pass if (time.time() - last_update) >= block_timeout: raise QueueTimeout('Blocking queue timed out.', { 'queue_id': self.id, 'queue_type': self.type, })
def start(self, transaction=None, block=False, block_timeout=30): self.ttl_timestamp = utils.now() + \ datetime.timedelta(seconds=self.ttl) self.commit(transaction=transaction) if block: if transaction: raise TypeError('Cannot use transaction when blocking') cursor_id = messenger.get_cursor_id('queue') extra = { 'queue_doc': self.export() } messenger.publish('queue', [PENDING, self.id], extra=extra, transaction=transaction) if block: last_update = time.time() while True: for msg in messenger.subscribe('queue', cursor_id=cursor_id, timeout=block_timeout): cursor_id = msg['_id'] try: if msg['message'] == [COMPLETE, self.id]: return elif msg['message'] == [UPDATE, self.id]: last_update = time.time() break elif msg['message'] == [ERROR, self.id]: raise QueueTaskError('Error occured running ' + 'queue task', { 'queue_id': self.id, 'queue_type': self.type, }) except TypeError: pass if (time.time() - last_update) >= block_timeout: raise QueueTimeout('Blocking queue timed out.', { 'queue_id': self.id, 'queue_type': self.type, })
def listener_thread(): queue = callqueue.CallQueue() queue.start(10) while True: try: for msg in messenger.subscribe(listener.channels.keys()): for lstnr in listener.channels[msg['channel']]: try: queue.put(lstnr, msg) except: logger.exception('Error in listener callback', 'runners') except GeneratorExit: raise except: logger.exception('Error in listener thread', 'runners') time.sleep(0.3) yield
def subscribe(self, cursor_id=None, timeout=None): for msg in messenger.subscribe('servers', cursor_id=cursor_id, timeout=timeout): if msg.get('server_id') == self.id: yield msg
def subscribe(self, cursor_id=None, timeout=None): for msg in messenger.subscribe('servers', cursor_id=cursor_id, timeout=timeout): if msg.get('server_id') == self.server.id: yield msg