def event_listener(CNX, config): """Processlet responsible for routing and reacting on status change""" D("event listener") cnx = CNX poller = zmq.Poller() out_sign = cnx["_context"].socket(zmq.PUB) out_sign.connect(CONFIG["cnx"]["SUB_orchester_master"] %config) other_in = cnx["orchester_in"] master_sox = cnx["master"] master_sox.setsockopt_string(zmq.SUBSCRIBE,unicode(LOCAL_INFO["where"])) poller.register(master_sox, zmq.POLLIN) poller.register(other_in, zmq.POLLIN) cpt=0 now = time() while True: if abs(now - time()) >= 1: print "%.1f msg/sec %.2f" % (time(), 1.0*cpt/(time() - now)) carbon_send( dict( msg_per_sec = 1.0*cpt/(time() - now))) cpt = 0 now = time() cpt+=1 new={} ready_sox = dict(poller.poll()) if other_in in ready_sox and ready_sox[other_in] == zmq.POLLIN: new = fast_parse_event(other_in) D("rcv from OTHER %s" % repr(new)) elif master_sox in ready_sox and ready_sox[master_sox] == zmq.POLLIN: new = fast_parse_event(master_sox) D("rcv from MASTER %s" % repr(new)) if new == {}: continue if new["where"] != LOCAL_INFO["where"]: log.info("NOT FOR ME Iam %s msg was for %s " % (LOCAL_INFO["where"],new["where"] )) #D("*****") continue try: # only one message at a time can be treated not even sure I need it task_id = new["task_id"] D("RCV%s"%repr(new)) if new["event"] in { "INIT", "BOUNCE"}: re_send_vector(cnx["tracker_out"],new, "ACK", dict( pid = config["pid"])) incr_task_id(new) new["retry"] = "0" new["step"] ="orchester" new["event"] = "INIT" new["next"] = new["type"] D("initing to %s" % repr(new)) D("sending to %r" % cnx[new["type"]]) send_vector(cnx[new["next"]], new) #log.warning("gup %r %r" % (monitor, new)) re_send_vector(cnx["tracker_out"],new, "SEND", dict( pid = config["pid"])) #send_vector(monitor, new) else: log.warning("unknown message caught %r" % _f(new)) if "PROPAGATE" == new["event"]: D("skipping PROPAGATE for %s" % _f(new)) except Exception as e: log.error("Tracker is sad : %r" % e) log.exception( e) D("WAITING FOR")
def event_listener(CNX, config): """Processlet responsible for routing and reacting on status change""" D("event listener") global error_per_stage, busy_per_stage, message_per_stage cnx = CNX propagated_to = dict() def on_failure(new): log.critical("KO for <%(emitter)r-%(task_id)r> %(arg)r" % new) def on_end( new): #D("JOB PROCESSED @ STEP %(step)s" % new) cleanUp(new) def cleanUp(new): #q_madd(timer_q,"cancel", new) global busy_per_stage busy_per_stage -= dict({ new["step"] : 1 }) def on_begin(new): global busy_per_stage busy_per_stage += { new["step"] : 1 } def on_timeout(new): __on_error(new) def on_error(new): __on_error(new) def __on_error(new): global error_per_stage cleanUp(new) error_per_stage+={ new["step"] : 1 } D("OE %s" % ( _f(new) )) #### WHY did I put that? if int(new.get("retry",0)) >= config["max_retry"]: D('retry %(retry)s for <%(emitter)s-%(task_id)s>' % new) new["event"] = "FAILURE" on_failure(new) else: log.critical("unhandled failure for %r" % new) ### could also restart failing processes here def on_send(new): pass def on_init(new): pass def on_propagate( new): D("just passing %s" % _f(new)) on_propagate += { new["next"] : 1 } D("waiting") action = dict( INIT = False, BOUNCE = False, SEND = False, HAPPY_END = False, ACK = False, END = on_end, BEGIN = on_begin, PROPAGATE = on_propagate, ERROR = on_error, TIMEOUT = on_error, O_TIMEOUT = on_failure, ) print("Waiting for socket to be read cf 100% CPU zmq bug") sleep(1) local_in_sox = cnx["tracker_in"] CLOCK_M = int(config.get("tracker_clock_every", 100)) ignore = 0 check_delay = config.get("check_delay", 3) * 2 now = time() while True: new = fast_parse_event(local_in_sox) if ignore == 0: log.debug("RCV %s" % _f(new)) if abs(time() - now) > check_delay: q_madd( watchdog_q, busy_per_stage) log.info("zombie count %r" % busy_per_stage) log.info("MPS %r" % message_per_stage) log.info("EPS %r" % error_per_stage) now = time() ignore += 1 ignore %= CLOCK_M if new["where"] != LOCAL_INFO["where"]: D("NOT FOR ME %s" % _f(new)) continue try: message_per_stage += { new["step"] : 1 } task_id = new["task_id"] emitter = new["emitter"] if int(new["seq"]) > config.get("max_seq", 50): logging.warning("<%r> was bounced <%r> times" %(new, new["seq"])) continue # Wut ? event, U sure? if action[new["event"]]: action[new["event"]](new) except Exception as e: D("MON EXC for %r is %s" % (new, e)) log.exception( e)