def do_loop_turn(self):
"""Receiver daemon main loop
:return: None
"""
# Begin to clean modules
self.check_and_del_zombie_modules()
# Maybe the arbiter pushed a new configuration...
if self.watch_for_new_conf(timeout=0.05):
logger.info("I got a new configuration...")
# Manage the new configuration
self.setup_new_conf()
# Maybe external modules raised 'objects'
# we should get them
_t0 = time.time()
self.get_objects_from_from_queues()
statsmgr.timer('core.get-objects-from-queues', time.time() - _t0)
# Get external commands from the arbiters...
_t0 = time.time()
self.get_external_commands_from_arbiters()
statsmgr.timer('external-commands.got.time', time.time() - _t0)
statsmgr.gauge('external-commands.got.count', len(self.unprocessed_external_commands))
_t0 = time.time()
self.push_external_commands_to_schedulers()
statsmgr.timer('external-commands.pushed.time', time.time() - _t0)
# Say to modules it's a new tick :)
_t0 = time.time()
self.hook_point('tick')
statsmgr.timer('hook.tick', time.time() - _t0)
def get_new_broks(self):
"""Get new broks from our satellites
:return: None
"""
for satellites in [
self.schedulers, self.pollers, self.reactionners,
self.receivers
]:
for satellite_link in list(satellites.values()):
logger.debug("Getting broks from %s", satellite_link)
_t0 = time.time()
try:
tmp_broks = satellite_link.get_broks(self.name)
except LinkError:
logger.warning(
"Daemon %s connection failed, I could not get the broks!",
satellite_link)
else:
if tmp_broks:
logger.debug("Got %d Broks from %s in %s",
len(tmp_broks), satellite_link.name,
time.time() - _t0)
statsmgr.gauge(
'get-new-broks-count.%s' % (satellite_link.name),
len(tmp_broks))
statsmgr.timer(
'get-new-broks-time.%s' % (satellite_link.name),
time.time() - _t0)
for brok in tmp_broks:
brok.instance_id = satellite_link.instance_id
# Add the broks to our global list
self.external_broks.extend(tmp_broks)
def get_new_broks(self):
"""Get new broks from our satellites
:return: None
"""
for satellites in [self.schedulers, self.pollers, self.reactionners, self.receivers]:
for satellite_link in list(satellites.values()):
logger.debug("Getting broks from %s", satellite_link)
_t0 = time.time()
try:
tmp_broks = satellite_link.get_broks(self.name)
except LinkError:
logger.warning("Daemon %s connection failed, I could not get the broks!",
satellite_link)
else:
if tmp_broks:
logger.debug("Got %d Broks from %s in %s",
len(tmp_broks), satellite_link.name, time.time() - _t0)
statsmgr.gauge('get-new-broks-count.%s'
% (satellite_link.name), len(tmp_broks))
statsmgr.timer('get-new-broks-time.%s'
% (satellite_link.name), time.time() - _t0)
for brok in tmp_broks:
brok.instance_id = satellite_link.instance_id
# Add the broks to our global list
self.external_broks.extend(tmp_broks)
def get_internal_broks(self):
"""Get all broks from self.broks_internal_raised and append them to our broks
to manage
:return: None
"""
statsmgr.gauge('get-new-broks-count.broker', len(self.internal_broks))
# Add the broks to our global list
self.external_broks.extend(self.internal_broks)
self.internal_broks = []
def get_internal_broks(self):
"""Get all broks from self.broks_internal_raised and append them to our broks
to manage
:return: None
"""
statsmgr.gauge('get-new-broks-count.broker', len(self.internal_broks))
# Add the broks to our global list
self.external_broks.extend(self.internal_broks)
self.internal_broks = []
def get_arbiter_broks(self):
"""Get the broks from the arbiters,
but as the arbiter_broks list can be push by arbiter without Global lock,
we must protect this with a lock
TODO: really? check this arbiter behavior!
:return: None
"""
with self.arbiter_broks_lock:
statsmgr.gauge('get-new-broks-count.arbiter', len(self.arbiter_broks))
# Add the broks to our global list
self.external_broks.extend(self.arbiter_broks)
self.arbiter_broks = []
def do_get_new_actions(self):
"""Get new actions from schedulers
Create a Message and put into the module queue
REF: doc/alignak-action-queues.png (1)
:return: None
"""
# Here are the differences between a poller and a reactionner:
# Poller will only do checks,
# Reactionner will do actions (notifications and event handlers)
do_checks = self.__class__.do_checks
do_actions = self.__class__.do_actions
# We check and get the new actions to execute in each of our schedulers
for scheduler_link_uuid in self.schedulers:
scheduler_link = self.schedulers[scheduler_link_uuid]
if not scheduler_link.active:
logger.warning("My scheduler '%s' is not active currently",
scheduler_link.name)
continue
logger.debug("get new actions, scheduler: %s", scheduler_link.name)
# OK, go for it :)
_t0 = time.time()
actions = scheduler_link.get_actions({
'do_checks':
do_checks,
'do_actions':
do_actions,
'poller_tags':
self.poller_tags,
'reactionner_tags':
self.reactionner_tags,
'worker_name':
self.name,
'module_types':
list(self.q_by_mod.keys())
})
if actions:
logger.debug("Got %d actions from %s", len(actions),
scheduler_link.name)
# We 'tag' them with my_scheduler and put into queue for workers
self.add_actions(actions, scheduler_link.instance_id)
logger.debug("Got %d actions from %s in %s", len(actions),
scheduler_link.name,
time.time() - _t0)
statsmgr.gauge('actions.added.count.%s' % (scheduler_link.name),
len(actions))
def get_arbiter_broks(self):
"""Get the broks from the arbiters,
but as the arbiter_broks list can be push by arbiter without Global lock,
we must protect this with a lock
TODO: really? check this arbiter behavior!
:return: None
"""
with self.arbiter_broks_lock:
statsmgr.gauge('get-new-broks-count.arbiter',
len(self.arbiter_broks))
# Add the broks to our global list
self.external_broks.extend(self.arbiter_broks)
self.arbiter_broks = []
def do_get_new_actions(self):
"""Get new actions from schedulers
Create a Message and put into the module queue
REF: doc/alignak-action-queues.png (1)
:return: None
"""
# Here are the differences between a poller and a reactionner:
# Poller will only do checks,
# Reactionner will do actions (notifications and event handlers)
do_checks = self.__class__.do_checks
do_actions = self.__class__.do_actions
# We check and get the new actions to execute in each of our schedulers
for scheduler_link_uuid in self.schedulers:
scheduler_link = self.schedulers[scheduler_link_uuid]
if not scheduler_link.active:
logger.warning("My scheduler '%s' is not active currently", scheduler_link.name)
continue
logger.debug("get new actions, scheduler: %s", scheduler_link.name)
# OK, go for it :)
_t0 = time.time()
actions = scheduler_link.get_actions({'do_checks': do_checks, 'do_actions': do_actions,
'poller_tags': self.poller_tags,
'reactionner_tags': self.reactionner_tags,
'worker_name': self.name,
'module_types': list(self.q_by_mod.keys())})
if actions:
logger.debug("Got %d actions from %s", len(actions), scheduler_link.name)
# We 'tag' them with my_scheduler and put into queue for workers
self.add_actions(actions, scheduler_link.instance_id)
logger.debug("Got %d actions from %s in %s",
len(actions), scheduler_link.name, time.time() - _t0)
statsmgr.gauge('actions.added.count.%s' % (scheduler_link.name), len(actions))
def do_loop_turn(self):
# pylint: disable=too-many-branches
"""Loop used to:
* get initial status broks
* check if modules are alive, if not restart them
* get broks from ourself, the arbiters and our satellites
* add broks to the queue of each external module
* manage broks with each internal module
If the internal broks management is longer than 0.8 seconds, postpone to hte next
loop turn to avoid overloading the broker daemon.
:return: None
"""
if not self.got_initial_broks:
# Asking initial broks from my schedulers
my_satellites = self.get_links_of_type(s_type='scheduler')
for satellite in list(my_satellites.values()):
logger.info("Asking my initial broks from '%s'",
satellite.name)
_t0 = time.time()
try:
my_initial_broks = satellite.get_initial_broks(self.name)
statsmgr.timer('broks.initial.%s.time' % satellite.name,
time.time() - _t0)
if not my_initial_broks:
logger.info("No initial broks were raised, "
"my scheduler is not yet ready...")
return
self.got_initial_broks = True
logger.debug("Got %d initial broks from '%s'",
my_initial_broks, satellite.name)
statsmgr.gauge('broks.initial.%s.count' % satellite.name,
my_initial_broks)
except LinkError as exp:
logger.warning(
"Scheduler connection failed, I could not get initial broks!"
)
logger.debug("Begin Loop: still some old broks to manage (%d)",
len(self.external_broks))
if self.external_broks:
statsmgr.gauge('unmanaged.broks', len(self.external_broks))
# Try to see if one of my module is dead, and restart previously dead modules
self.check_and_del_zombie_modules()
# Call modules that manage a starting tick pass
_t0 = time.time()
self.hook_point('tick')
statsmgr.timer('hook.tick', time.time() - _t0)
# Maybe the last loop we did raised some broks internally
self.get_internal_broks()
# Also reap broks sent from the arbiters
self.get_arbiter_broks()
# Now get broks from our distant daemons
self.get_new_broks()
# Get the list of broks not yet sent to our external modules
_t0 = time.time()
broks_to_send = [
brok for brok in self.external_broks
if getattr(brok, 'to_be_sent', True)
]
statsmgr.gauge('get-new-broks-count.to_send', len(broks_to_send))
# Send the broks to all external modules to_q queue so they can get the whole packet
# beware, the sub-process/queue can be die/close, so we put to restart the whole module
# instead of killing ourselves :)
for module in self.modules_manager.get_external_instances():
try:
_t00 = time.time()
queue_size = module.to_q.qsize()
statsmgr.gauge(
'queues.external.%s.to.size' % module.get_name(),
queue_size)
module.to_q.put(broks_to_send)
statsmgr.timer('queues.external.%s.to.put' % module.get_name(),
time.time() - _t00)
except Exception as exp: # pylint: disable=broad-except
# first we must find the modules
logger.warning(
"Module %s queue exception: %s, I'm tagging it to restart later",
module.get_name(), str(exp))
logger.exception(exp)
self.modules_manager.set_to_restart(module)
# No more need to send them
for brok in broks_to_send:
brok.to_be_sent = False
logger.debug("Time to send %s broks (%d secs)", len(broks_to_send),
time.time() - _t0)
# Make the internal modules manage the broks
start = time.time()
while self.external_broks:
now = time.time()
# Do not 'manage' more than 0.8s, we must get new broks almost every second
if now - start > 0.8:
logger.info(
"I did not yet managed all my broks, still %d broks",
len(self.external_broks))
break
# Get the first brok in the list
brok = self.external_broks.pop(0)
if self.modules_manager.get_internal_instances():
self.manage_brok(brok)
# Make a very short pause to avoid overloading
self.make_a_pause(0.01, check_time_change=False)
else:
if getattr(brok, 'to_be_sent', False):
self.external_broks.append(brok)
# Maybe our external modules raised 'objects', so get them
if self.get_objects_from_from_queues():
statsmgr.gauge('external-commands.got.count',
len(self.external_commands))
statsmgr.gauge('broks.got.count', len(self.external_broks))
def do_loop_turn(self): # pylint: disable=too-many-branches
"""Satellite main loop::
* Check and delete zombies actions / modules
* Get returns from queues
* Adjust worker number
* Get new actions
:return: None
"""
# Try to see if one of my module is dead, and restart previously dead modules
self.check_and_del_zombie_modules()
# Also if some zombie workers exist...
self.check_and_del_zombie_workers()
# Call modules that manage a starting tick pass
self.hook_point('tick')
# Print stats for debug
for _, sched in self.schedulers.items():
for mod in self.q_by_mod:
# In workers we've got actions sent to queue - queue size
for (worker_id, queue) in list(self.q_by_mod[mod].items()):
try:
actions_count = queue.qsize()
results_count = self.returns_queue.qsize()
logger.debug("[%s][%s][%s] actions queued: %d, results queued: %d",
sched.name, mod, worker_id, actions_count, results_count)
# Update the statistics
statsmgr.gauge('worker.%s.actions-queue-size' % worker_id,
actions_count)
statsmgr.gauge('worker.%s.results-queue-size' % worker_id,
results_count)
except (IOError, EOFError):
pass
# todo temporaray deactivate all this stuff!
# Before return or get new actions, see how we managed
# the former ones: are they still in queue(s)? If so, we
# must wait more or at least have more workers
# wait_ratio = self.wait_ratio.get_load()
# total_q = 0
# try:
# for mod in self.q_by_mod:
# for queue in list(self.q_by_mod[mod].values()):
# total_q += queue.qsize()
# except (IOError, EOFError):
# pass
# if total_q != 0 and wait_ratio < 2 * self.worker_polling_interval:
# logger.debug("I decide to increase the wait ratio")
# self.wait_ratio.update_load(wait_ratio * 2)
# # self.wait_ratio.update_load(self.worker_polling_interval)
# else:
# # Go to self.worker_polling_interval on normal run, if wait_ratio
# # was >2*self.worker_polling_interval,
# # it make it come near 2 because if < 2, go up :)
# self.wait_ratio.update_load(self.worker_polling_interval)
# wait_ratio = self.wait_ratio.get_load()
# statsmgr.timer('core.wait-ratio', wait_ratio)
# if self.log_loop:
# logger.debug("[%s] wait ratio: %f", self.name, wait_ratio)
# Maybe we do not have enough workers, we check for it
# and launch the new ones if needed
self.adjust_worker_number_by_load()
# Manage all messages we've got in the last timeout
# for queue in self.return_messages:
try:
logger.debug("[%s] manage action results: %d results",
self.name, self.returns_queue.qsize())
while self.returns_queue.qsize():
msg = self.returns_queue.get_nowait()
if msg is None:
continue
logger.debug("Got a message: %s", msg)
if msg.get_type() == 'Done':
logger.debug("Got an action result: %s", msg.get_data())
self.manage_action_return(msg.get_data())
logger.debug("Managed action result")
else:
logger.warning("Ignoring message of type: %s", msg.get_type())
except Full:
logger.warning("Returns queue is full")
except Empty:
logger.debug("Returns queue is empty")
except (IOError, EOFError) as exp:
logger.warning("My returns queue is no more available: %s", str(exp))
except Exception as exp: # pylint: disable=broad-except
logger.error("Failed getting messages in returns queue: %s", str(exp))
logger.error(traceback.format_exc())
for _, sched in self.schedulers.items():
if sched.wait_homerun:
logger.debug("scheduler home run: %d results", len(sched.wait_homerun))
if not self.passive:
# If we are an active satellite, we do not initiate the check getting
# and return
try:
# We send to our schedulers the results of all finished checks
logger.debug("pushing results...")
self.push_results()
except LinkError as exp:
logger.warning("Scheduler connection failed, I could not send my results!")
try:
# And we get the new actions from our schedulers
logger.debug("getting new actions...")
self.get_new_actions()
except LinkError as exp:
logger.warning("Scheduler connection failed, I could not get new actions!")
# Get objects from our modules that are not Worker based
if self.log_loop:
logger.debug("[%s] get objects from queues", self.name)
self.get_objects_from_from_queues()
statsmgr.gauge('external-commands.count', len(self.external_commands))
statsmgr.gauge('broks.count', len(self.broks))
statsmgr.gauge('events.count', len(self.events))
def do_loop_turn(self): # pylint: disable=too-many-branches
"""Satellite main loop::
* Check and delete zombies actions / modules
* Get returns from queues
* Adjust worker number
* Get new actions
:return: None
"""
# Try to see if one of my module is dead, and restart previously dead modules
self.check_and_del_zombie_modules()
# Also if some zombie workers exist...
self.check_and_del_zombie_workers()
# Call modules that manage a starting tick pass
self.hook_point('tick')
# Print stats for debug
for _, sched in self.schedulers.items():
for mod in self.q_by_mod:
# In workers we've got actions sent to queue - queue size
for (worker_id, queue) in list(self.q_by_mod[mod].items()):
try:
actions_count = queue.qsize()
results_count = self.returns_queue.qsize()
logger.debug("[%s][%s][%s] actions queued: %d, results queued: %d",
sched.name, mod, worker_id, actions_count, results_count)
# Update the statistics
statsmgr.gauge('worker.%s.actions-queue-size' % worker_id,
actions_count)
statsmgr.gauge('worker.%s.results-queue-size' % worker_id,
results_count)
except (IOError, EOFError):
pass
# todo temporaray deactivate all this stuff!
# Before return or get new actions, see how we managed
# the former ones: are they still in queue(s)? If so, we
# must wait more or at least have more workers
# wait_ratio = self.wait_ratio.get_load()
# total_q = 0
# try:
# for mod in self.q_by_mod:
# for queue in list(self.q_by_mod[mod].values()):
# total_q += queue.qsize()
# except (IOError, EOFError):
# pass
# if total_q != 0 and wait_ratio < 2 * self.worker_polling_interval:
# logger.debug("I decide to increase the wait ratio")
# self.wait_ratio.update_load(wait_ratio * 2)
# # self.wait_ratio.update_load(self.worker_polling_interval)
# else:
# # Go to self.worker_polling_interval on normal run, if wait_ratio
# # was >2*self.worker_polling_interval,
# # it make it come near 2 because if < 2, go up :)
# self.wait_ratio.update_load(self.worker_polling_interval)
# wait_ratio = self.wait_ratio.get_load()
# statsmgr.timer('core.wait-ratio', wait_ratio)
# if self.log_loop:
# logger.debug("[%s] wait ratio: %f", self.name, wait_ratio)
# Maybe we do not have enough workers, we check for it
# and launch the new ones if needed
self.adjust_worker_number_by_load()
# Manage all messages we've got in the last timeout
# for queue in self.return_messages:
try:
logger.debug("[%s] manage action results: %d results",
self.name, self.returns_queue.qsize())
while self.returns_queue.qsize():
msg = self.returns_queue.get_nowait()
if msg is None:
continue
if not isinstance(msg, Message):
logger.warning("Should have received a Message, got a %s!", type(msg))
continue
logger.debug("Got a message: %s", msg)
if msg.get_type() == 'Done':
logger.debug("Got (from %s) an action result: %s",
msg.get_source(), msg.get_data())
self.manage_action_return(msg.get_data())
elif msg.get_type() == 'Stats':
logger.debug("Got (from %s) stats: %s",
msg.get_source(), msg.get_data())
if msg.get_source() in self.workers:
self.workers[msg.get_source()].stats = msg.get_data()
else:
logger.warning("Ignoring message of type: %s", msg.get_type())
except Full:
logger.warning("Returns queue is full")
except Empty:
logger.debug("Returns queue is empty")
except (IOError, EOFError) as exp:
logger.warning("My returns queue is no more available: %s", str(exp))
except Exception as exp: # pylint: disable=broad-except
logger.error("Failed getting messages in returns queue: %s", str(exp))
logger.error(traceback.format_exc())
for _, sched in self.schedulers.items():
if sched.wait_homerun:
logger.debug("scheduler home run: %d results", len(sched.wait_homerun))
if not self.passive:
# If we are an active satellite, we do not initiate the check getting
# and return
try:
# We send to our schedulers the results of all finished checks
logger.debug("pushing results...")
self.push_results()
except LinkError as exp:
logger.warning("Scheduler connection failed, I could not send my results!")
try:
# And we get the new actions from our schedulers
logger.debug("getting new actions...")
self.get_new_actions()
except LinkError as exp:
logger.warning("Scheduler connection failed, I could not get new actions!")
# Get objects from our modules that are not Worker based
if self.log_loop:
logger.debug("[%s] get objects from queues", self.name)
self.get_objects_from_from_queues()
statsmgr.gauge('external-commands.count', len(self.external_commands))
statsmgr.gauge('broks.count', len(self.broks))
statsmgr.gauge('events.count', len(self.events))
def push_external_commands_to_schedulers(self):
"""Send a HTTP request to the schedulers (POST /run_external_commands)
with external command list.
:return: None
"""
if not self.unprocessed_external_commands:
return
commands_to_process = self.unprocessed_external_commands
self.unprocessed_external_commands = []
logger.debug("Commands: %s", commands_to_process)
statsmgr.gauge('external-commands.pushed', len(self.unprocessed_external_commands))
# Now get all external commands and put them into the
# good schedulers
for ext_cmd in commands_to_process:
self.external_commands_manager.resolve_command(ext_cmd)
logger.debug("Resolved command: %s", ext_cmd)
# Now for all alive schedulers, send the commands
for sched_id in self.schedulers:
sched = self.schedulers[sched_id]
# TODO: sched should be a SatelliteLink object and, thus, have a get_name() method
# but sometimes when an exception is raised because the scheduler is not available
# this is not True ... sched is a simple dictionary!
is_active = sched['active']
if not is_active:
logger.warning("The scheduler '%s' is not active, it is not possible to push "
"external commands from its connection!", sched)
return
# If there are some commands...
extcmds = sched['external_commands']
cmds = [extcmd.cmd_line for extcmd in extcmds]
if not cmds:
continue
# ...and the scheduler is alive
con = sched['con']
if con is None:
self.daemon_connection_init(sched_id, s_type='scheduler')
if con is None:
logger.warning("The connection for the scheduler '%s' cannot be established, it is "
"not possible to push external commands.", sched)
continue
sent = False
logger.debug("Sending %d commands to scheduler %s", len(cmds), sched)
try:
# con.run_external_commands(cmds)
con.post('run_external_commands', {'cmds': cmds})
sent = True
except HTTPClientConnectionException as exp: # pragma: no cover, simple protection
logger.warning("[%s] %s", sched, str(exp))
sched['con'] = None
continue
except HTTPClientTimeoutException as exp: # pragma: no cover, simple protection
logger.warning("Connection timeout with the scheduler '%s' when "
"sending external commands: %s", sched, str(exp))
sched['con'] = None
continue
except HTTPClientException as exp: # pragma: no cover, simple protection
logger.error("Error with the scheduler '%s' when "
"sending external commands: %s", sched, str(exp))
sched['con'] = None
continue
except AttributeError as exp: # pragma: no cover, simple protection
logger.warning("The scheduler %s should not be initialized: %s",
sched, str(exp))
logger.exception(exp)
except Exception as exp: # pylint: disable=broad-except
logger.exception("A satellite raised an unknown exception (%s): %s", type(exp), exp)
raise
# Whether we sent the commands or not, clean the scheduler list
self.schedulers[sched_id]['external_commands'] = []
# If we didn't sent them, add the commands to the arbiter list
if not sent:
for extcmd in extcmds:
self.external_commands.append(extcmd)
def push_external_commands_to_schedulers(self):
"""Push received external commands to the schedulers
:return: None
"""
if not self.unprocessed_external_commands:
return
# Those are the global external commands
commands_to_process = self.unprocessed_external_commands
self.unprocessed_external_commands = []
logger.debug("Commands: %s", commands_to_process)
# Now get all external commands and put them into the good schedulers
logger.debug("Commands to process: %d commands", len(commands_to_process))
for ext_cmd in commands_to_process:
cmd = self.external_commands_manager.resolve_command(ext_cmd)
logger.debug("Resolved command: %s, result: %s", ext_cmd.cmd_line, cmd)
if cmd and cmd['global']:
# Send global command to all our schedulers
for scheduler_link_uuid in self.schedulers:
self.schedulers[scheduler_link_uuid].pushed_commands.append(ext_cmd)
# Now for all active schedulers, send the commands
count_pushed_commands = 0
count_failed_commands = 0
for scheduler_link_uuid in self.schedulers:
link = self.schedulers[scheduler_link_uuid]
if not link.active:
logger.debug("The scheduler '%s' is not active, it is not possible to push "
"external commands to its connection!", link.name)
continue
# If there are some commands for this scheduler...
commands = [ext_cmd.cmd_line for ext_cmd in link.pushed_commands]
if not commands:
logger.debug("The scheduler '%s' has no commands.", link.name)
continue
logger.debug("Sending %d commands to scheduler %s", len(commands), link.name)
sent = []
try:
sent = link.push_external_commands(commands)
except LinkError:
logger.warning("Scheduler connection failed, I could not push external commands!")
# Whether we sent the commands or not, clean the scheduler list
link.pushed_commands = []
# If we didn't sent them, add the commands to the arbiter list
if sent:
statsmgr.gauge('external-commands.pushed.%s' % link.name, len(commands))
count_pushed_commands = count_pushed_commands + len(commands)
else:
count_failed_commands = count_failed_commands + len(commands)
statsmgr.gauge('external-commands.failed.%s' % link.name, len(commands))
# Kepp the not sent commands... for a next try
self.external_commands.extend(commands)
statsmgr.gauge('external-commands.pushed.all', count_pushed_commands)
statsmgr.gauge('external-commands.failed.all', count_failed_commands)
def get_new_broks(self, s_type='scheduler'):
"""Get new broks from daemon defined in type parameter
:param s_type: type of object
:type s_type: str
:return: None
"""
# Get the good links tab for looping..
links = self.get_links_from_type(s_type)
if links is None:
logger.debug('Type unknown for connection! %s', s_type)
return
# We check for new check in each schedulers and put
# the result in new_checks
for s_id in links:
logger.debug("Getting broks from %s", links[s_id]['name'])
link = links[s_id]
logger.debug("Link: %s", link)
if not link['active']:
logger.debug("The %s '%s' is not active, "
"do not get broks from its connection!", s_type, link['name'])
continue
if link['con'] is None:
if not self.daemon_connection_init(s_id, s_type=s_type):
if link['connection_attempt'] <= link['max_failed_connections']:
logger.warning("The connection for the %s '%s' cannot be established, "
"it is not possible to get broks from this daemon.",
s_type, link['name'])
else:
logger.error("The connection for the %s '%s' cannot be established, "
"it is not possible to get broks from this daemon.",
s_type, link['name'])
continue
try:
_t0 = time.time()
tmp_broks = link['con'].get('get_broks', {'bname': self.name}, wait='long')
try:
tmp_broks = unserialize(tmp_broks, True)
except AlignakClassLookupException as exp: # pragma: no cover,
# simple protection
logger.error('Cannot un-serialize data received from "get_broks" call: %s',
exp)
continue
if tmp_broks:
logger.debug("Got %d Broks from %s in %s",
len(tmp_broks), link['name'], time.time() - _t0)
statsmgr.timer('con-broks-get.%s' % (link['name']), time.time() - _t0)
statsmgr.gauge('con-broks-count.%s' % (link['name']), len(tmp_broks.values()))
for brok in tmp_broks.values():
brok.instance_id = link['instance_id']
# Ok, we can add theses broks to our queues
_t0 = time.time()
self.add_broks_to_queue(tmp_broks.values())
statsmgr.timer('con-broks-add.%s' % s_type, time.time() - _t0)
except HTTPClientConnectionException as exp: # pragma: no cover, simple protection
logger.warning("[%s] %s", link['name'], str(exp))
link['con'] = None
return
except HTTPClientTimeoutException as exp: # pragma: no cover, simple protection
logger.warning("Connection timeout with the %s '%s' when getting broks: %s",
s_type, link['name'], str(exp))
link['con'] = None
return
except HTTPClientException as exp: # pragma: no cover, simple protection
logger.error("Error with the %s '%s' when getting broks: %s",
s_type, link['name'], str(exp))
link['con'] = None
return
# scheduler must not have checks
# What the F**k? We do not know what happened,
# so.. bye bye :)
except Exception as exp: # pylint: disable=broad-except
logger.exception(exp)
sys.exit(1)
def do_loop_turn(self):
# pylint: disable=too-many-branches
"""Loop used to:
* get initial status broks
* check if modules are alive, if not restart them
* get broks from ourself, the arbiters and our satellites
* add broks to the queue of each external module
* manage broks with each internal module
If the internal broks management is longer than 0.8 seconds, postpone to hte next
loop turn to avoid overloading the broker daemon.
:return: None
"""
if not self.got_initial_broks:
# Asking initial broks from my schedulers
my_satellites = self.get_links_of_type(s_type='scheduler')
for satellite in list(my_satellites.values()):
logger.info("Asking my initial broks from '%s'", satellite.name)
_t0 = time.time()
try:
my_initial_broks = satellite.get_initial_broks(self.name)
statsmgr.timer('broks.initial.%s.time' % satellite.name, time.time() - _t0)
if not my_initial_broks:
logger.info("No initial broks were raised, "
"my scheduler is not yet ready...")
return
self.got_initial_broks = True
logger.debug("Got %d initial broks from '%s'",
my_initial_broks, satellite.name)
statsmgr.gauge('broks.initial.%s.count' % satellite.name, my_initial_broks)
except LinkError as exp:
logger.warning("Scheduler connection failed, I could not get initial broks!")
logger.debug("Begin Loop: still some old broks to manage (%d)", len(self.external_broks))
if self.external_broks:
statsmgr.gauge('unmanaged.broks', len(self.external_broks))
# Try to see if one of my module is dead, and restart previously dead modules
self.check_and_del_zombie_modules()
# Call modules that manage a starting tick pass
_t0 = time.time()
self.hook_point('tick')
statsmgr.timer('hook.tick', time.time() - _t0)
# Maybe the last loop we did raised some broks internally
self.get_internal_broks()
# Also reap broks sent from the arbiters
self.get_arbiter_broks()
# Now get broks from our distant daemons
self.get_new_broks()
# Get the list of broks not yet sent to our external modules
_t0 = time.time()
broks_to_send = [brok for brok in self.external_broks if getattr(brok, 'to_be_sent', True)]
statsmgr.gauge('get-new-broks-count.to_send', len(broks_to_send))
# Send the broks to all external modules to_q queue so they can get the whole packet
# beware, the sub-process/queue can be die/close, so we put to restart the whole module
# instead of killing ourselves :)
for module in self.modules_manager.get_external_instances():
try:
_t00 = time.time()
queue_size = module.to_q.qsize()
statsmgr.gauge('queues.external.%s.to.size' % module.get_name(), queue_size)
module.to_q.put(broks_to_send)
statsmgr.timer('queues.external.%s.to.put' % module.get_name(), time.time() - _t00)
except Exception as exp: # pylint: disable=broad-except
# first we must find the modules
logger.warning("Module %s queue exception: %s, I'm tagging it to restart later",
module.get_name(), str(exp))
logger.exception(exp)
self.modules_manager.set_to_restart(module)
# No more need to send them
for brok in broks_to_send:
brok.to_be_sent = False
logger.debug("Time to send %s broks (%d secs)", len(broks_to_send), time.time() - _t0)
# Make the internal modules manage the broks
start = time.time()
while self.external_broks:
now = time.time()
# Do not 'manage' more than 0.8s, we must get new broks almost every second
if now - start > 0.8:
logger.info("I did not yet managed all my broks, still %d broks",
len(self.external_broks))
break
# Get the first brok in the list
brok = self.external_broks.pop(0)
if self.modules_manager.get_internal_instances():
self.manage_brok(brok)
# Make a very short pause to avoid overloading
self.make_a_pause(0.01, check_time_change=False)
else:
if getattr(brok, 'to_be_sent', False):
self.external_broks.append(brok)
# Maybe our external modules raised 'objects', so get them
if self.get_objects_from_from_queues():
statsmgr.gauge('external-commands.got.count', len(self.external_commands))
statsmgr.gauge('broks.got.count', len(self.external_broks))