def do_loop_turn(self): loop_time = time.time() with self.broks_lock: nb_broks = len(self.broks) nb_external_broks = len(self.external_module_broks) logger.debug("[Broks] Begin Loop: managing queue broks [%d]" % nb_broks) self.broks_done = 0 # FIXME: Does it come from a structure only known from enterprise ? # for mod in self.modules_manager.get_internal_instances(): # self.local_module_stats[mod.get_name()] = 0 # Dump modules Queues size external_modules = [ external_module for external_module in self.modules_manager.instances if external_module.is_external ] for external_module in external_modules: try: logger.debug("[Broks] External Queue len (%s): %s" % ( external_module.get_name(), external_module.to_q.qsize() )) except Exception as exp: logger.debug("External Queue len (%s): Exception! %s" % (external_module.get_name(), exp)) # Begin to clean modules self.check_and_del_zombie_modules() # Maybe the arbiter ask us to wait for a new conf # If true, we must restart all... if self.cur_conf is None: # Clean previous run from useless objects and close modules self.clean_previous_run() self.wait_for_initial_conf() # we may have been interrupted or so; then # just return from this loop turn if not self.new_conf: return self.setup_new_conf() # Now we check if arbiter speak to us in the pyro_daemon. # If so, we listen for it # When it pushes conf to us, we reinit connections self.watch_for_new_conf(0.0) if self.new_conf: if self.graceful_enabled and self.switch_process() is True: # Child successfully spawned, we're exiting return self.setup_new_conf() # Maybe the last loop we raised some broks internally we should # integrate them in broks self.interger_internal_broks() # Also reap broks sent from the arbiters self.interger_arbiter_broks() # Main job, go get broks in our distants daemons types = ['scheduler', 'poller', 'reactionner', 'receiver'] for _type in types: _t = time.time() # And from schedulers self.get_new_broks(type=_type) statsmgr.timing('core.broker.get-new-broks.%s' % _type, time.time() - _t, 'perf') # We will works this turn with a copy of the broks, so we won't be # impacted by possible other threads (modules or so) with self.broks_lock: broks = copy.copy(self.broks) to_send = list(self.external_module_broks) self.broks = deque() self.external_module_broks = deque() # and for external queues # REF: doc/broker-modules.png (3) # We put to external queues broks that was not already send t0 = time.time() # We are sending broks as a big list, more efficient than one by one queues = self.modules_manager.get_external_to_queues() for q in queues: try: q.put(to_send) # we catch but the kill detector on the next loop will detect the # fail module and will manage it except Exception: logger.error( 'FAIL TO PUSH DATA TO EXTERNAL MODULE this module will ' 'be detected and restart.' ) statsmgr.timing('core.broker.put-to-external-queue', time.time() - t0, 'perf') logger.debug("[Broks] Time to send [%s] broks to module ([%.3f] secs)" % (len(to_send), time.time() - t0)) start = time.time() while len(broks) != 0: now = time.time() # Do not 'manage' more than 1s, we must get new broks # every 1s if now - start > 1: # so we must remerge our last broks with the main broks to do not # lost them with self.broks_lock: logger.debug( 'Cannot manage all remaining broks [%d] in a loop ' 'turn, push bask this broks in the queue.' % len(broks) ) self.broks.extendleft(broks) break try: b = broks.pop() except IndexError: # no more broks, maybe a daemon stop, not a problem, catch it break # Ok, we can get the brok, and doing something with it # REF: doc/broker-modules.png (4-5) # We un serialize the brok before consume it b.prepare() _t = time.time() self.manage_brok(b) statsmgr.timing('core.broker.manage-brok', time.time() - _t, 'perf') # Maybe external modules raised 'objects' we should get them nb_object_get = self.get_objects_from_from_queues() logger.debug( '[stats] nb object get control queues of external module [%d]' % nb_object_get ) # Say to modules it's a new tick :) self.hook_point('tick') logger.debug('[stats] broks done this loop %d/%d' % (self.broks_done, nb_broks)) time.sleep(max(0.01, min(1.0, 1.0 - (time.time() - loop_time)))) # Checks if memory consumption did not exceed allowed thresold self.check_memory_usage()
def f_wrapper(): t0 = time.time() args_time = aqu_lock_time = calling_time = json_time = 0 need_lock = getattr(f, 'need_lock', True) # Warning : put the bottle.response set inside the wrapper # because outside it will break bottle d = {} method = getattr(f, 'method', 'get').lower() for aname in args: v = None if method == 'post': v = bottle.request.forms.get(aname, None) # Post args are zlibed and cPickled (but in # safemode) if v is not None: v = zlib.decompress(v) v = SafeUnpickler.loads(v) elif method == 'get': v = bottle.request.GET.get(aname, None) if v is None: # Maybe we got a default value? default_args = self.registered_fun_defaults.get(fname, {}) if aname not in default_args: raise Exception('Missing argument %s' % aname) v = default_args[aname] d[aname] = v t1 = time.time() args_time = t1 - t0 if need_lock: logger.debug("HTTP: calling lock for %s", fname) lock.acquire() t2 = time.time() aqu_lock_time = t2 - t1 try: ret = f(**d) # Always call the lock release if need finally: # Ok now we can release the lock if need_lock: lock.release() t3 = time.time() calling_time = t3 - t2 encode = getattr(f, 'encode', 'json').lower() j = json.dumps(ret) t4 = time.time() json_time = t4 - t3 global_time = t4 - t0 logger.debug("Debug perf: %s [args:%s] [aqu_lock:%s]" "[calling:%s] [json:%s] [global:%s]", fname, args_time, aqu_lock_time, calling_time, json_time, global_time) lst = [('args', args_time), ('aqulock', aqu_lock_time), ('calling', calling_time), ('json', json_time), ('global', global_time)] # increase the stats timers for (k, _t) in lst: statsmgr.timing('http.%s.%s' % (fname, k), _t, 'perf') return j
def pynag_con_init(self, id, type='scheduler'): _t = time.time() r = self.do_pynag_con_init(id, type) statsmgr.timing('con-init.%s' % type, time.time() - _t, 'perf') return r
def f_wrapper(): t0 = time.time() args_time = aqu_lock_time = calling_time = json_time = 0 need_lock = getattr(f, 'need_lock', True) # Warning : put the bottle.response set inside the wrapper # because outside it will break bottle d = {} method = getattr(f, 'method', 'get').lower() for aname in args: v = None if method == 'post': v = bottle.request.forms.get(aname, None) # Post args are zlibed and cPickled (but in # safemode) if v is not None: v = zlib.decompress(v) v = SafeUnpickler.loads(v) elif method == 'get': v = bottle.request.GET.get(aname, None) if v is None: # Maybe we got a default value? default_args = self.registered_fun_defaults.get( fname, {}) if aname not in default_args: raise Exception('Missing argument %s' % aname) v = default_args[aname] d[aname] = v t1 = time.time() args_time = t1 - t0 if need_lock: logger.debug("HTTP: calling lock for %s", fname) lock.acquire() t2 = time.time() aqu_lock_time = t2 - t1 try: ret = f(**d) # Always call the lock release if need finally: # Ok now we can release the lock if need_lock: lock.release() t3 = time.time() calling_time = t3 - t2 encode = getattr(f, 'encode', 'json').lower() j = json.dumps(ret) t4 = time.time() json_time = t4 - t3 global_time = t4 - t0 logger.debug( "Debug perf: %s [args:%s] [aqu_lock:%s]" "[calling:%s] [json:%s] [global:%s]", fname, args_time, aqu_lock_time, calling_time, json_time, global_time) lst = [('args', args_time), ('aqulock', aqu_lock_time), ('calling', calling_time), ('json', json_time), ('global', global_time)] # increase the stats timers for (k, _t) in lst: statsmgr.timing('http.%s.%s' % (fname, k), _t, 'perf') logger.info( "[api]:{func}, [args]:{args} [cost]:{cost}".format( func=fname, args=json.dumps(d), cost=calling_time)) return j
def pynag_con_init(self, id, type="scheduler"): _t = time.time() r = self.do_pynag_con_init(id, type) statsmgr.timing("con-init.%s" % type, time.time() - _t, "perf") return r
if self.new_conf: self.setup_new_conf() # Maybe the last loop we raised some broks internally # we should integrate them in broks self.interger_internal_broks() # Also reap broks sent from the arbiters self.interger_arbiter_broks() # Main job, go get broks in our distants daemons types = ["scheduler", "poller", "reactionner", "receiver"] for _type in types: _t = time.time() # And from schedulers self.get_new_broks(type=_type) statsmgr.timing("core.broker.get-new-broks.%s" % _type, time.time() - _t, "perf") # Sort the brok list by id self.broks.sort(sort_by_ids) # and for external queues # REF: doc/broker-modules.png (3) # We put to external queues broks that was not already send t0 = time.time() # We are sending broks as a big list, more efficient than one by one ext_modules = self.modules_manager.get_external_instances() to_send = [b for b in self.broks if getattr(b, "need_send_to_ext", True)] # Send our pack to all external modules to_q queue so they can get the wole packet # beware, the sub-process/queue can be die/close, so we put to restart the whole module # instead of killing ourself :)
if self.new_conf: self.setup_new_conf() # Maybe the last loop we raised some broks internally # we should integrate them in broks self.interger_internal_broks() # Also reap broks sent from the arbiters self.interger_arbiter_broks() # Main job, go get broks in our distants daemons types = ['scheduler', 'poller', 'reactionner', 'receiver'] for _type in types: _t = time.time() # And from schedulers self.get_new_broks(type=_type) statsmgr.timing('core.broker.get-new-broks.%s' % _type, time.time() - _t, 'perf') # Sort the brok list by id self.broks.sort(sort_by_ids) # and for external queues # REF: doc/broker-modules.png (3) # We put to external queues broks that was not already send t0 = time.time() # We are sending broks as a big list, more efficient than one by one ext_modules = self.modules_manager.get_external_instances() to_send = [ b for b in self.broks if getattr(b, 'need_send_to_ext', True) ] # Send our pack to all external modules to_q queue so they can get the wole packet