def main(): def print_event(event): type = event.pop("type") s = "Event: {0}\n".format(type) keys = event.keys() keys.sort() for k in keys: v = event[k] s += " {0}: {1}\n".format(k, v) print s print "Initializing event listener." app = app_or_default(None) conn = app.broker_connection() recv = EventReceiver(conn, handlers={"*": print_event}, app=app) try: try: print "Listening for events... (use Ctrl-C to exit)" recv.capture(limit=None) except KeyboardInterrupt, SystemExit: raise SystemExit except Exception: import traceback print "Exception while listening for events:\n" traceback.print_exc()
def run_loop(self): while True: try: with self.broker_conn as conn: recv = EventReceiver(conn, handlers={ 'task-sent': self.on_task_sent, 'task-failed': self.on_task_failed, 'task-retried': self.on_task_retried, 'task-started': self.on_task_started, 'task-succeeded': self.on_task_succeeded, 'task-received': self.on_task_received, 'task-rejected': self.on_task_rejected, 'task-revoked': self.on_task_revoked, 'worker-online': self.on_worker_online, 'worker-heartbeat': self.on_worker_heartbeat, 'worker-offline': self.on_worker_offline, }) recv.capture(limit=None, timeout=None, wakeup=True) except (KeyboardInterrupt, SystemExit): raise except Exception: raise
def run(self): try_interval = 1 while True: try: try_interval *= 2 with self.capp.connection() as conn: recv = EventReceiver(conn, handlers={"*": self.on_event}, app=self.capp) try_interval = 1 recv.capture(limit=None, timeout=None, wakeup=True) except (KeyboardInterrupt, SystemExit): try: import _thread as thread except ImportError: import thread thread.interrupt_main() except Exception as e: logger.error( "Failed to capture events: '%s', " "trying again in %s seconds.", e, try_interval) logger.debug(e, exc_info=True) time.sleep(try_interval)
def run(self): try_interval = 1 while True: try: try_interval *= 2 with self.capp.connection() as conn: recv = EventReceiver(conn, handlers={"*": self.on_event}, app=self.capp) try_interval = 1 recv.capture(limit=None, timeout=None, wakeup=True) except (KeyboardInterrupt, SystemExit): try: import _thread as thread except ImportError: import thread thread.interrupt_main() except Exception as e: logger.error("Failed to capture events: '%s', " "trying again in %s seconds.", e, try_interval) logger.debug(e, exc_info=True) time.sleep(try_interval)
def _capture_events(self): try_interval = 1 while not self._is_root_complete(): try: try_interval *= 2 with self.celery_app.connection() as conn: conn.ensure_connection(max_retries=1, interval_start=0) recv = EventReceiver(conn, handlers={"*": self._on_event}, app=self.celery_app) try_interval = 1 self._ready() recv.capture(limit=None, timeout=None, wakeup=True) except (KeyboardInterrupt, SystemExit): logger.exception("Received external shutdown.") self._on_external_shutdown() # pylint: disable=C0321 except Exception: if self._is_root_complete(): logger.info( "Root task complete; stopping broker receiver thread.") return logger.error(traceback.format_exc()) if try_interval > self.max_try_interval: logger.warning( "Maximum broker retry attempts exceeded, stopping receiver thread)." " Will no longer retry despite incomplete root task.") return logger.debug("Try interval %d secs, still worth retrying." % try_interval) time.sleep(try_interval)
def evcam(camera, freq=1.0, maxrate=None, loglevel=0, logfile=None, app=None): app = app_or_default(app) if not isinstance(loglevel, int): loglevel = LOG_LEVELS[loglevel.upper()] logger = app.log.setup_logger(loglevel=loglevel, logfile=logfile, name="celery.evcam") logger.info( "-> evcam: Taking snapshots with %s (every %s secs.)\n" % ( camera, freq)) state = State() cam = instantiate(camera, state, app=app, freq=freq, maxrate=maxrate, logger=logger) cam.install() conn = app.broker_connection() recv = EventReceiver(conn, app=app, handlers={"*": state.event}) try: try: recv.capture(limit=None) except KeyboardInterrupt: raise SystemExit finally: cam.cancel() conn.close()
async def watch_for_events(event_handlers, q): event_handlers.queue = q print("watching for events") with event_handlers.capp.connection() as conn: while True: receiver = EventReceiver(conn, handlers=event_handlers.to_dict()) receiver.capture(limit=None, timeout=None, wakeup=True) await asyncio.sleep(0.5)
def eventdump(): sys.stderr.write("-> celeryev: starting capture...\n") dumper = Dumper() conn = establish_connection() recv = EventReceiver(conn, handlers={"*": dumper.on_event}) try: recv.capture() except (KeyboardInterrupt, SystemExit): conn and conn.close()
def evdump(): sys.stderr.write("-> evdump: starting capture...\n") dumper = Dumper() conn = establish_connection() recv = EventReceiver(conn, handlers={"*": dumper.on_event}) try: recv.capture() except (KeyboardInterrupt, SystemExit): conn and conn.close()
def run(self): from app import celery celery.control.enable_events() while True: with celery.connection() as conn: recv = EventReceiver(conn, handlers={"*": self.on_event}, app=celery) recv.capture(limit=None, timeout=None, wakeup=True)
def celery_listener(bot, uri): """Listen for events from Celery, relay to IRC.""" connection = Connection(uri) def on_account_created(event): request = event['request'] bot_announce( bot, IRC_CHANNELS_ANNOUNCE, '{user} created ({real_name})'.format( user=request['user_name'], real_name=request['real_name'], ), ) def on_account_submitted(event): request = event['request'] bot_announce( bot, IRC_CHANNELS, '{user} ({real_name}) needs approval: {reasons}'.format( user=request['user_name'], real_name=request['real_name'], reasons=', '.join(request['reasons']), ), ) def on_account_approved(event): request = event['request'] bot_announce( bot, IRC_CHANNELS_ANNOUNCE, '{user} was approved, now pending creation.'.format( user=request['user_name'], ), ) def on_account_rejected(event): request = event['request'] bot_announce( bot, IRC_CHANNELS_ANNOUNCE, '{user} was rejected.'.format(user=request['user_name'], ), ) while True: with connection as conn: recv = EventReceiver( conn, handlers={ 'ocflib.account_created': on_account_created, 'ocflib.account_submitted': on_account_submitted, 'ocflib.account_approved': on_account_approved, 'ocflib.account_rejected': on_account_rejected, }, ) recv.capture(limit=None, timeout=None)
def evdump(app=None): sys.stderr.write("-> evdump: starting capture...\n") app = app_or_default(app) dumper = Dumper() conn = app.broker_connection() recv = EventReceiver(conn, app=app, handlers={"*": dumper.on_event}) try: recv.capture() except (KeyboardInterrupt, SystemExit): conn and conn.close()
def celery_listener(uri): """Listen for events from Celery, relay to IRC.""" connection = Connection(uri) def on_account_created(event): request = event['request'] bot_announce( IRC_CHANNELS_ANNOUNCE, '{user} created ({real_name})'.format( user=request['user_name'], real_name=request['real_name'], ), ) def on_account_submitted(event): request = event['request'] bot_announce( IRC_CHANNELS, '{user} ({real_name}) needs approval: {reasons}'.format( user=request['user_name'], real_name=request['real_name'], reasons=', '.join(request['reasons']), ), ) def on_account_approved(event): request = event['request'] bot_announce( IRC_CHANNELS_ANNOUNCE, '{user} was approved, now pending creation.'.format( user=request['user_name'], ), ) def on_account_rejected(event): request = event['request'] bot_announce( IRC_CHANNELS_ANNOUNCE, '{user} was rejected.'.format( user=request['user_name'], ), ) while True: with connection as conn: recv = EventReceiver( conn, handlers={ 'ocflib.account_created': on_account_created, 'ocflib.account_submitted': on_account_submitted, 'ocflib.account_approved': on_account_approved, 'ocflib.account_rejected': on_account_rejected, }, ) recv.capture(limit=None, timeout=None)
class EventListener(object): """Capture events sent by messages and store them in memory.""" def __init__(self, state=state): self.state = state self.connection = establish_connection() self.receiver = EventReceiver(self.connection, handlers={"*": self.state.event}) def start(self): self.receiver.capture()
class CeleryConsumer(Thread): def __init__(self, state, celery): super(CeleryConsumer, self).__init__() self.celery = celery self.state = state def run(self): with self.celery.connection() as conn: self.receiver = EventReceiver( conn, handlers={'*': self.state.event}, app=self.celery, ) self.receiver.capture()
def run(self): ready = False while not ready: workers = self.celery_client.workers() if not workers: time.sleep(5) continue ready = True if self.enable_events: self.celery_client.enable_events() application = self.celery_client.application with application.connection() as conn: receiver = EventReceiver(conn, handlers={"*": self.on_event}, app=application) receiver.capture(limit=None, timeout=None, wakeup=True)
def run(self): logging.info("Enabling events") celery.control.enable_events() while True: try: with establish_connection() as connection: recv = EventReceiver( connection, handlers={"*": state.event}) with DumpCam(state, freq=1.0): recv.capture(limit=None, timeout=None) except (KeyboardInterrupt, SystemExit): import thread thread.interrupt_main() except Exception as e: logging.error("An error occurred while capturing events" ": %s" % e)
def run(self): try_interval = 1 while True: try: try_interval *= 2 with self._celery_app.connection() as conn: recv = EventReceiver(conn, handlers={"*": self.on_event}, app=self._celery_app) recv.capture(limit=None, timeout=None) try_interval = 1 except (KeyboardInterrupt, SystemExit): import thread thread.interrupt_main() except Exception as e: logging.error("Failed to capture events: '%s', " "trying again in %s seconds." % (e, try_interval)) time.sleep(try_interval)
class MonitorListener(object): """Capture events sent by messages and store them in memory.""" def __init__(self, state): self.connection = DjangoBrokerConnection() self.receiver = EventReceiver(self.connection, handlers={ "task-received": state.receive_task_received, "task-accepted": state.receive_task_event, "task-succeeded": state.receive_task_event, "task-retried": state.receive_task_event, "task-failed": state.receive_task_event, "worker-online": state.receive_worker_event, "worker-offline": state.receive_worker_event, "worker-heartbeat": state.receive_heartbeat, }) def start(self): self.receiver.capture()
def run(self): try_interval = 1 while True: logger.info("Starting to capture events...") try: import celery try_interval *= 2 with self.capp.connection() as conn: recv = EventReceiver(conn, handlers={"*": self.on_event}, app=self.capp) try_interval = 1 recv.capture(limit=None, timeout=None, wakeup=True) except Exception as e: logger.error( "Failed to capture events: '%s', " "trying again in %s seconds.", e, try_interval) logger.debug(e, exc_info=True)
def run(self): self.start_rpc() try_interval = 1 while True: try: try_interval *= 2 with self.app.connection() as conn: recv = EventReceiver(conn, handlers={"*": self.on_shutter}, app=self.app) try_interval = 1 recv.capture(limit=None, timeout=None, wakeup=True) except Exception as e: logger.error( "Failed to capture events: '%s', " "trying again in %s seconds.", e, try_interval) logger.debug(e, exc_info=True) time.sleep(try_interval)
class MonitorListener(threading.Thread): """Capture events sent by messages and store them in memory.""" def __init__(self, state, *args, **kwargs): super(MonitorListener, self).__init__(*args, **kwargs) self.connection = DjangoBrokerConnection() self.receiver = EventReceiver(self.connection, handlers={ "task-received": state.receive_task_received, "task-accepted": state.receive_task_event, "task-succeeded": state.receive_task_event, "task-retried": state.receive_task_event, "task-failed": state.receive_task_event, "worker-online": state.receive_worker_event, "worker-offline": state.receive_worker_event, "worker-heartbeat": state.receive_heartbeat, }) self.setDaemon(True) def run(self): self.receiver.capture()
def run(self): try_interval = 1 while True: try: try_interval *= 2 with self.capp.connection() as conn: recv = EventReceiver(conn, handlers={'*': self.on_event}, app=self.capp) try_interval = 1 recv.capture(limit=None, timeout=None, wakeup=True) except (KeyboardInterrupt, SystemExit): import _thread as thread thread.interrupt_main() except Exception as e: logger.error('捕获 Celery 事件失败: "%s". ' '将在 %s 秒后重试...', e, try_interval) logger.debug(e, exc_info=True) time.sleep(try_interval)
def evtop(): sys.stderr.write("-> evtop: starting capture...\n") state = State() display = CursesMonitor(state) display.init_screen() refresher = DisplayThread(display) refresher.start() conn = establish_connection() recv = EventReceiver(conn, handlers={"*": state.event}) try: recv.capture(limit=None) except Exception: refresher.shutdown = True refresher.join() display.resetscreen() raise except (KeyboardInterrupt, SystemExit): conn and conn.close() refresher.shutdown = True refresher.join() display.resetscreen()
def run(self): # We don't want too frequent retries try_interval = 1 while True: try: try_interval *= 2 with self.capp.connection() as conn: recv = EventReceiver(conn, handlers={'*': self.on_event}, app=self.capp) try_interval = 1 recv.capture(limit=None, timeout=None, wakeup=True) except (KeyboardInterrupt, SystemExit): import _thread _thread.interrupt_main() except Exception as e: logger.error( 'Failed to capture events: "%s", ' 'trying again in %s seconds.', e, try_interval) logger.debug(e, exc_info=True) time.sleep(try_interval)
def evcam(camera, freq=1.0, maxrate=None, loglevel=0, logfile=None): if not isinstance(loglevel, int): loglevel = conf.LOG_LEVELS[loglevel.upper()] logger = log.setup_logger(loglevel=loglevel, logfile=logfile, name="celery.evcam") logger.info( "-> evcam: Taking snapshots with %s (every %s secs.)\n" % ( camera, freq)) state = State() cam = instantiate(camera, state, freq=freq, maxrate=maxrate, logger=logger) cam.install() conn = establish_connection() recv = EventReceiver(conn, handlers={"*": state.event}) try: try: recv.capture(limit=None) except KeyboardInterrupt: raise SystemExit finally: cam.cancel() conn.close()
def my_monitor(): connection = BrokerConnection('amqp://*****:*****@localhost:5672//') import json def on_task_succeeded(event): #print "TASK SUCCEEDED! ", event result = event['result'] #print "RESULT: ", result #result = json.loads(result) # print "RESULT --------------------------", result, " and TYPE OF RESULTTTTTTTTTTTTTTTTTTTTTTTTTTT", type(result) # print "SUBMISSION ID:------------------------------", result['submission_id'] # url_str = build_url(result['submission_id'], result['file_id']) # response = requests.put(url_str, data=serialize(result), headers={'Content-Type' : 'application/json'}) # print "SENT PUT REQUEST. RESPONSE RECEIVED: ", response ## def on_task_failed(event): #print "TASK FAILED!", event exception = event['exception'] def on_update(event): #print "TASK UPDATE!!!!!", event result = event['result'] #print "TYPE OF RESULT IN UPDATE CASEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE", type(result) # url_str = build_url(result['submission_id'], result['file_id']) # response = requests.put(url_str, data=serialize(result), headers={'Content-Type' : 'application/json'}) # print "SENT PUT REQUEST. RESPONSE RECEIVED: ", response def on_event(event): #print "EVENT HAPPENED: ", event pass def on_custom(event): #print "EVENT TYPE: ", event['type'] #if event['type'] != "worker-heartbeat": print("CUSTOM - TASK!", event) def on_worker_online(event): print("worker came onlineeeeee!!! Event: ", str(event)) #try_interval = 3 while True: try: #try_interval *= 2 with connection as conn: recv = EventReceiver(conn, handlers={'task-failed' : on_task_failed, 'task-succeeded' : on_task_succeeded, 'task-sent' : on_event, 'task-received' : on_event, 'task-revoked' : on_event, 'task-retried' : on_event, 'task-started' : on_event, 'task-update' : on_update, 'worker-online' : on_worker_online #'task-update' : on_custom, #'*' : on_custom }) #handlers={'*': on_event}) #print "PRINT FROM Monitor: ", ( vars(recv.consumer) ) recv.capture(limit=None, timeout=None) #try_interval = 3 except (KeyboardInterrupt, SystemExit): print("EXCEPTION KEYBOARD INTERRUPT") sys.exit()
def celery_listener(bot, celery, uri): """Listen for events from Celery, relay to IRC.""" # TODO: this stuff should be moved to ircbot.plugin.create. connection = Connection(uri, ssl={ 'ssl_ca_certs': '/etc/ssl/certs/ca-certificates.crt', 'ssl_cert_reqs': ssl.CERT_REQUIRED, }) def on_account_created(event): request = event['request'] bot_announce( bot, IRC_CHANNELS_ANNOUNCE, '{user} created ({real_name})'.format( user=request['user_name'], real_name=request['real_name'], ), ) def on_account_submitted(event): request = event['request'] bot_announce( bot, IRC_CHANNELS, '{user} ({real_name}) needs approval: {reasons}'.format( user=request['user_name'], real_name=request['real_name'], reasons=', '.join(request['reasons']), ), ) def on_account_approved(event): request = event['request'] bot_announce( bot, IRC_CHANNELS_ANNOUNCE, '{user} was approved, now pending creation.'.format( user=request['user_name'], ), ) def on_account_rejected(event): request = event['request'] bot_announce( bot, IRC_CHANNELS_ANNOUNCE, '{user} was rejected.'.format( user=request['user_name'], ), ) while True: with connection as conn: recv = EventReceiver( conn, app=celery, handlers={ 'ocflib.account_created': on_account_created, 'ocflib.account_submitted': on_account_submitted, 'ocflib.account_approved': on_account_approved, 'ocflib.account_rejected': on_account_rejected, }, ) recv.capture(limit=None, timeout=None)
def my_monitor(): connection = BrokerConnection('amqp://*****:*****@localhost:5672//') import json def on_task_succeeded(event): #print "TASK SUCCEEDED! ", event result = event['result'] #print "RESULT: ", result #result = json.loads(result) # print "RESULT --------------------------", result, " and TYPE OF RESULTTTTTTTTTTTTTTTTTTTTTTTTTTT", type(result) # print "SUBMISSION ID:------------------------------", result['submission_id'] # url_str = build_url(result['submission_id'], result['file_id']) # response = requests.put(url_str, data=serialize(result), headers={'Content-Type' : 'application/json'}) # print "SENT PUT REQUEST. RESPONSE RECEIVED: ", response ## def on_task_failed(event): #print "TASK FAILED!", event exception = event['exception'] def on_update(event): #print "TASK UPDATE!!!!!", event result = event['result'] #print "TYPE OF RESULT IN UPDATE CASEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE", type(result) # url_str = build_url(result['submission_id'], result['file_id']) # response = requests.put(url_str, data=serialize(result), headers={'Content-Type' : 'application/json'}) # print "SENT PUT REQUEST. RESPONSE RECEIVED: ", response def on_event(event): #print "EVENT HAPPENED: ", event pass def on_custom(event): #print "EVENT TYPE: ", event['type'] #if event['type'] != "worker-heartbeat": print "CUSTOM - TASK!", event def on_worker_online(event): print "worker came onlineeeeee!!! Event: ", str(event) #try_interval = 3 while True: try: #try_interval *= 2 with connection as conn: recv = EventReceiver(conn, handlers={'task-failed' : on_task_failed, 'task-succeeded' : on_task_succeeded, 'task-sent' : on_event, 'task-received' : on_event, 'task-revoked' : on_event, 'task-retried' : on_event, 'task-started' : on_event, 'task-update' : on_update, 'worker-online' : on_worker_online #'task-update' : on_custom, #'*' : on_custom }) #handlers={'*': on_event}) #print "PRINT FROM Monitor: ", ( vars(recv.consumer) ) recv.capture(limit=None, timeout=None) #try_interval = 3 except (KeyboardInterrupt, SystemExit): print "EXCEPTION KEYBOARD INTERRUPT" sys.exit()
def celery_listener(bot): """Listen for events from Celery, relay to IRC.""" while not (hasattr(bot, 'connection') and bot.connection.connected): time.sleep(2) celery = Celery( broker=bot.celery_conf['broker'], backend=bot.celery_conf['backend'], ) celery.conf.broker_use_ssl = { 'ssl_ca_certs': '/etc/ssl/certs/ca-certificates.crt', 'ssl_cert_reqs': ssl.CERT_REQUIRED, } # `redis_backend_use_ssl` is an OCF patch which was proposed upstream: # https://github.com/celery/celery/pull/3831 celery.conf.redis_backend_use_ssl = { 'ssl_cert_reqs': ssl.CERT_NONE, } # TODO: stop using pickle celery.conf.task_serializer = 'pickle' celery.conf.result_serializer = 'pickle' celery.conf.accept_content = {'pickle'} bot.tasks = get_tasks(celery) connection = Connection( bot.celery_conf['broker'], ssl={ 'ssl_ca_certs': '/etc/ssl/certs/ca-certificates.crt', 'ssl_cert_reqs': ssl.CERT_REQUIRED, }, ) def bot_announce(targets, message): for target in targets: bot.say(target, message) def on_account_created(event): request = event['request'] if request['calnet_uid']: uid_or_gid = 'Calnet UID: {}'.format(request['calnet_uid']) elif request['callink_oid']: uid_or_gid = 'Callink OID: {}'.format(request['callink_oid']) else: uid_or_gid = 'No Calnet UID or OID set' bot_announce( IRC_CHANNELS_ANNOUNCE, '{user} created ({real_name}, {uid_or_gid})'.format( user=request['user_name'], real_name=request['real_name'], uid_or_gid=uid_or_gid, ), ) def on_account_submitted(event): request = event['request'] bot_announce( IRC_CHANNELS_OPER, '{user} ({real_name}) needs approval: {reasons}'.format( user=request['user_name'], real_name=request['real_name'], reasons=', '.join(request['reasons']), ), ) def on_account_approved(event): request = event['request'] bot_announce( IRC_CHANNELS_ANNOUNCE, '{user} was approved, now pending creation.'.format( user=request['user_name'], ), ) def on_account_rejected(event): request = event['request'] bot_announce( IRC_CHANNELS_ANNOUNCE, '{user} was rejected.'.format(user=request['user_name'], ), ) while True: with connection as conn: recv = EventReceiver( conn, app=celery, handlers={ 'ocflib.account_created': on_account_created, 'ocflib.account_submitted': on_account_submitted, 'ocflib.account_approved': on_account_approved, 'ocflib.account_rejected': on_account_rejected, }, ) recv.capture(limit=None, timeout=None)
def celery_listener(bot, celery, uri): """Listen for events from Celery, relay to IRC.""" connection = Connection( uri, ssl={ 'ssl_ca_certs': '/etc/ssl/certs/ca-certificates.crt', 'ssl_cert_reqs': ssl.CERT_REQUIRED, }, ) def bot_announce(targets, message): for target in targets: bot.say(target, message) def on_account_created(event): request = event['request'] if request['calnet_uid']: uid_or_gid = 'Calnet UID: {}'.format(request['calnet_uid']) elif request['callink_oid']: uid_or_gid = 'Callink OID: {}'.format(request['callink_oid']) else: uid_or_gid = 'No Calnet UID or OID set' bot_announce( IRC_CHANNELS_ANNOUNCE, '{user} created ({real_name}, {uid_or_gid})'.format( user=request['user_name'], real_name=request['real_name'], uid_or_gid=uid_or_gid, ), ) def on_account_submitted(event): request = event['request'] bot_announce( IRC_CHANNELS_OPER, '{user} ({real_name}) needs approval: {reasons}'.format( user=request['user_name'], real_name=request['real_name'], reasons=', '.join(request['reasons']), ), ) def on_account_approved(event): request = event['request'] bot_announce( IRC_CHANNELS_ANNOUNCE, '{user} was approved, now pending creation.'.format( user=request['user_name'], ), ) def on_account_rejected(event): request = event['request'] bot_announce( IRC_CHANNELS_ANNOUNCE, '{user} was rejected.'.format(user=request['user_name'], ), ) while True: with connection as conn: recv = EventReceiver( conn, app=celery, handlers={ 'ocflib.account_created': on_account_created, 'ocflib.account_submitted': on_account_submitted, 'ocflib.account_approved': on_account_approved, 'ocflib.account_rejected': on_account_rejected, }, ) recv.capture(limit=None, timeout=None)