def event_listener(CNX, config):
"""Processlet responsible for routing and reacting on status change"""
D("event listener")
cnx = CNX
poller = zmq.Poller()
out_sign = cnx["_context"].socket(zmq.PUB)
out_sign.connect(CONFIG["cnx"]["SUB_orchester_master"] %config)
other_in = cnx["orchester_in"]
master_sox = cnx["master"]
master_sox.setsockopt_string(zmq.SUBSCRIBE,unicode(LOCAL_INFO["where"]))
poller.register(master_sox, zmq.POLLIN)
poller.register(other_in, zmq.POLLIN)
cpt=0
now = time()
while True:
if abs(now - time()) >= 1:
print "%.1f msg/sec %.2f" % (time(), 1.0*cpt/(time() - now))
carbon_send( dict( msg_per_sec = 1.0*cpt/(time() - now)))
cpt = 0
now = time()
cpt+=1
new={}
ready_sox = dict(poller.poll())
if other_in in ready_sox and ready_sox[other_in] == zmq.POLLIN:
new = fast_parse_event(other_in)
D("rcv from OTHER %s" % repr(new))
elif master_sox in ready_sox and ready_sox[master_sox] == zmq.POLLIN:
new = fast_parse_event(master_sox)
D("rcv from MASTER %s" % repr(new))
if new == {}:
continue
if new["where"] != LOCAL_INFO["where"]:
log.info("NOT FOR ME Iam %s msg was for %s " % (LOCAL_INFO["where"],new["where"] ))
#D("*****")
continue
try:
# only one message at a time can be treated not even sure I need it
task_id = new["task_id"]
D("RCV%s"%repr(new))
if new["event"] in { "INIT", "BOUNCE"}:
re_send_vector(cnx["tracker_out"],new, "ACK", dict( pid = config["pid"]))
incr_task_id(new)
new["retry"] = "0"
new["step"] ="orchester"
new["event"] = "INIT"
new["next"] = new["type"]
D("initing to %s" % repr(new))
D("sending to %r" % cnx[new["type"]])
send_vector(cnx[new["next"]], new)
#log.warning("gup %r %r" % (monitor, new))
re_send_vector(cnx["tracker_out"],new, "SEND", dict( pid = config["pid"]))
#send_vector(monitor, new)
else:
log.warning("unknown message caught %r" % _f(new))
if "PROPAGATE" == new["event"]:
D("skipping PROPAGATE for %s" % _f(new))
except Exception as e:
log.error("Tracker is sad : %r" % e)
log.exception( e)
D("WAITING FOR")
def event_listener(CNX, config):
"""Processlet responsible for routing and reacting on status change"""
D("event listener")
global error_per_stage, busy_per_stage, message_per_stage
cnx = CNX
propagated_to = dict()
def on_failure(new):
log.critical("KO for <%(emitter)r-%(task_id)r> %(arg)r" % new)
def on_end( new):
#D("JOB PROCESSED @ STEP %(step)s" % new)
cleanUp(new)
def cleanUp(new):
#q_madd(timer_q,"cancel", new)
global busy_per_stage
busy_per_stage -= dict({ new["step"] : 1 })
def on_begin(new):
global busy_per_stage
busy_per_stage += { new["step"] : 1 }
def on_timeout(new):
__on_error(new)
def on_error(new):
__on_error(new)
def __on_error(new):
global error_per_stage
cleanUp(new)
error_per_stage+={ new["step"] : 1 }
D("OE %s" % ( _f(new) ))
#### WHY did I put that?
if int(new.get("retry",0)) >= config["max_retry"]:
D('retry %(retry)s for <%(emitter)s-%(task_id)s>' % new)
new["event"] = "FAILURE"
on_failure(new)
else:
log.critical("unhandled failure for %r" % new)
### could also restart failing processes here
def on_send(new):
pass
def on_init(new):
pass
def on_propagate( new):
D("just passing %s" % _f(new))
on_propagate += { new["next"] : 1 }
D("waiting")
action = dict(
INIT = False,
BOUNCE = False,
SEND = False,
HAPPY_END = False,
ACK = False,
END = on_end,
BEGIN = on_begin,
PROPAGATE = on_propagate,
ERROR = on_error,
TIMEOUT = on_error,
O_TIMEOUT = on_failure,
)
print("Waiting for socket to be read cf 100% CPU zmq bug")
sleep(1)
local_in_sox = cnx["tracker_in"]
CLOCK_M = int(config.get("tracker_clock_every", 100))
ignore = 0
check_delay = config.get("check_delay", 3) * 2
now = time()
while True:
new = fast_parse_event(local_in_sox)
if ignore == 0:
log.debug("RCV %s" % _f(new))
if abs(time() - now) > check_delay:
q_madd( watchdog_q, busy_per_stage)
log.info("zombie count %r" % busy_per_stage)
log.info("MPS %r" % message_per_stage)
log.info("EPS %r" % error_per_stage)
now = time()
ignore += 1
ignore %= CLOCK_M
if new["where"] != LOCAL_INFO["where"]:
D("NOT FOR ME %s" % _f(new))
continue
try:
message_per_stage += { new["step"] : 1 }
task_id = new["task_id"]
emitter = new["emitter"]
if int(new["seq"]) > config.get("max_seq", 50):
logging.warning("<%r> was bounced <%r> times" %(new,
new["seq"]))
continue
# Wut ? event, U sure?
if action[new["event"]]:
action[new["event"]](new)
except Exception as e:
D("MON EXC for %r is %s" % (new, e))
log.exception( e)
