def run(self):
self.cn()
self.sendbuf = binascii.unhexlify('00010000' + self.trans_id)
for x in range(self.stun_timeout):
try:
asyncore.poll2(timeout=1.0)
except: pass
self.handle_close()
def update(self):
"""neodym.client.Client.update
Poll asyncore, pass and return.
"""
asyncore.poll2()
pass
return
def _run_once(self):
"""
Run one full iteration of the event loop.
This calls all currently ready callbacks, polls for I/O,
schedules the resulting callbacks, and finally schedules
'call_later' callbacks.
This method is too big, keeping it as is to make it easier to backport
fixes from Python 3.
Changes from Python 3:
- Process events using asyncore.
- Use when > now when checking for ready timers, required for using
utils.monotonic_time using 10 millis resolution.
"""
# Remove delayed calls that were cancelled from head of queue.
while self._scheduled and self._scheduled[0]._cancelled:
heapq.heappop(self._scheduled)
timeout = None
if self._ready:
timeout = 0
elif self._scheduled:
when = self._scheduled[0]._when
timeout = max(0, when - self.time())
# Note: unlike Python 3 version, this run I/O event handlers now. This
# means that handlers scheduled from I/O event handlers will run in
# this cycle instead of the next cycle in Python 3.
asyncore.poll2(timeout, self._channels)
# Handle 'later' callbacks that are ready.
now = self.time()
while self._scheduled:
handle = self._scheduled[0]
if handle._when > now:
break
heapq.heappop(self._scheduled)
self._ready.append(handle)
# This is the only place where callbacks are actually *called*.
# All other places just add them to ready.
# Note: We run all currently scheduled callbacks, but not any
# callbacks scheduled by callbacks run this time around --
# they will be run the next time (after another I/O poll).
# Use an idiom that is thread-safe without using locks.
ntodo = len(self._ready)
for i in range(ntodo):
handle = self._ready.popleft()
if handle._cancelled:
continue
handle._run()
handle = None # Needed to break cycles when an exception occurs.
def _run_once(self):
"""
Run one full iteration of the event loop.
This calls all currently ready callbacks, polls for I/O,
schedules the resulting callbacks, and finally schedules
'call_later' callbacks.
This method is too big, keeping it as is to make it easier to backport
fixes from Python 3.
Changes from Python 3:
- Process events using asyncore.
- Use when > now when checking for ready timers, required for using
utils.monotonic_time using 10 millis resolution.
"""
# Remove delayed calls that were cancelled from head of queue.
while self._scheduled and self._scheduled[0]._cancelled:
heapq.heappop(self._scheduled)
timeout = None
if self._ready:
timeout = 0
elif self._scheduled:
when = self._scheduled[0]._when
timeout = max(0, when - self.time())
# Note: unlike Python 3 version, this run I/O event handlers now. This
# means that handlers scheduled from I/O event handlers will run in
# this cycle instead of the next cycle in Python 3.
asyncore.poll2(timeout, self._channels)
# Handle 'later' callbacks that are ready.
now = self.time()
while self._scheduled:
handle = self._scheduled[0]
if handle._when > now:
break
heapq.heappop(self._scheduled)
self._ready.append(handle)
# This is the only place where callbacks are actually *called*.
# All other places just add them to ready.
# Note: We run all currently scheduled callbacks, but not any
# callbacks scheduled by callbacks run this time around --
# they will be run the next time (after another I/O poll).
# Use an idiom that is thread-safe without using locks.
ntodo = len(self._ready)
for i in range(ntodo):
handle = self._ready.popleft()
if handle._cancelled:
continue
handle._run()
handle = None # Needed to break cycles when an exception occurs.
def do_work(self, s, returns_queue, c):
logger.info("[NRPEPoller] Module started!")
## restore default signal handler for the workers:
signal.signal(signal.SIGTERM, signal.SIG_DFL)
self.set_proctitle(self.name)
self.returns_queue = returns_queue
self.s = s
self.t_each_loop = time.time()
has_running = False
received = 0
launched = 0
waiting = 0
while True:
# We check if all new things in connections
# NB : using poll2 instead of poll (poll1 is with select
# call that is limited to 1024 connexions, poll2 is ... poll).
asyncore.poll2(1)
# If we are dying (big problem!) we do not
# take new jobs, we just finished the current one
if not self.i_am_dying:
# REF: doc/shinken-action-queues.png (3)
received = self.get_new_checks(has_running)
# REF: doc/shinken-action-queues.png (4)
launched = self.launch_new_checks()
# REF: doc/shinken-action-queues.png (5)
waiting = self.manage_finished_checks()
if received > 0 or launched > 0 or waiting > 0:
has_running = True
logger.debug(
"[nrpebooster][do_work] received={0} launched={1} waiting={2}"
.format(received, launched, waiting))
else:
has_running = False
# Now get order from master, if any..
try:
msg = c.get(block=False)
except Empty:
pass
else:
if msg.get_type() == 'Die':
logger.info("[NRPEPoller] Dad says we should die...")
break
def poll(self, timeout=0.1, maxIterations=10):
if not self.connected:
self.connect()
count = 0
self._movedData = True
while count < maxIterations:
self._movedData = False
asyncore.poll2(timeout=timeout, map=self._map)
if not self._movedData:
break
count += 1
if count:
return True
return False
def poll(self, timeout=0.1, maxIterations=10):
if not self.connected:
self.connect()
count = 0
self._movedData = True
while count < maxIterations:
self._movedData = False
asyncore.poll2(timeout=timeout, map=self._map)
if not self._movedData:
break
count += 1
if count:
return True
return False
def manage_finished_checks(self):
to_del = []
# First look for checks in timeout
for c in self.checks:
if c.status == 'launched':
c.con.look_for_timeout()
# We check if all new things in connections
# WARNING : using poll2 instead of poll (poll1 is with select
# call that is limited to 1024 connexions, poll2 is ... poll).
asyncore.poll2(timeout=1)
# Now we look for finished checks
for c in self.checks:
# First manage check in error, bad formed
if c.status == 'done':
to_del.append(c)
try:
self.returns_queue.put(c)
except IOError, exp:
logger.error("[NRPEPoller] Exiting: %s" % exp)
sys.exit(2)
continue
# Then we check for good checks
if c.status == 'launched' and c.con.is_done():
n = c.con
c.status = 'done'
c.exit_status = getattr(n, 'rc', 3)
c.get_outputs(getattr(n, 'message', 'Error in launching command.'), 8012)
c.execution_time = getattr(n, 'execution_time', 0.0)
# unlink our object from the original check
if hasattr(c, 'con'):
delattr(c, 'con')
# and set this check for deleting
# and try to send it
to_del.append(c)
try:
self.returns_queue.put(c)
except IOError, exp:
logger.error("[NRPEPoller]Exiting: %s" % exp)
sys.exit(2)
def update(self):
"""neodym.server.Server.update
Call asyncore.poll2 and then iterate over all connections and handle
any given input.
"""
asyncore.poll2()
pass
connections = [c for c in Connection.__all__]
if connections:
for c in connections:
if not c.recv_queue.empty():
message = c.recv_queue.get()
self.logger.debug('Received message: %s' % message)
# perform handshake operation on new connections
if c.is_connected is False:
if message.unique_identifier == 'handshake':
self.logger.debug('Handling handshake request...')
if message.get_attr('msg_map_hash') == self.__hash__:
self.logger.info('Handshake: OK -> Connection-%s' % id(c))
c.is_connected = True
c.put(message)
c.handle_write()
else:
self.logger.info('Hash mismatch!')
# handle the message for all connected clients
elif c.is_connected is True:
handlers = Handler.get_handlers(message)
callbacks = [h(message, c) for h in handlers]
self.logger.debug('Handling message through: %s' % (
str(callbacks)
))
else:
self.logger.debug('Internal server error!')
else:
time.sleep(0.1)
def do_work(self, s, returns_queue, c):
logger.info("[NRPEPoller] Module started!")
## restore default signal handler for the workers:
signal.signal(signal.SIGTERM, signal.SIG_DFL)
self.set_proctitle(self.name)
self.returns_queue = returns_queue
self.s = s
self.t_each_loop = time.time()
while True:
# We check if all new things in connections
# NB : using poll2 instead of poll (poll1 is with select
# call that is limited to 1024 connexions, poll2 is ... poll).
asyncore.poll2(1)
# If we are dying (big problem!) we do not
# take new jobs, we just finished the current one
if not self.i_am_dying:
# REF: doc/shinken-action-queues.png (3)
self.get_new_checks()
# REF: doc/shinken-action-queues.png (4)
self.launch_new_checks()
# REF: doc/shinken-action-queues.png (5)
self.manage_finished_checks()
# Now get order from master, if any..
try:
msg = c.get(block=False)
except Empty:
pass
else:
if msg.get_type() == 'Die':
logger.info("[NRPEPoller] Dad says we should die...")
break
def serve_once(self):
asyncore.poll2(map=self._map)
def test_retry_on_io_error(self):
fake_server = self.fake_server
inst = self._setup_nrpe(modconf)
inst.returns_queue = mock.MagicMock()
# We prepare a check in the to_queue
command = (
"$USER1$/check_nrpe -H 127.0.0.1 -p %s -n -u -t 5 -c check_load3 -a 20"
% fake_server.port)
chk = Check('queue', command, None, time.time())
# GO
inst.add_new_check(chk)
self.assertFalse(
fake_server.cli_socks, 'there should have no connected client '
'to our fake server at this point')
inst.launch_new_checks()
self.assertEqual('launched', chk.status)
self.assertEqual(0, chk.retried)
self.assertEqual('Sending request and waiting response..',
chk.con.message, "what? chk=%s " % chk)
# launch_new_checks() really launch a new check :
# it creates the nrpe client and directly make it to connect
# to the server.
# To give a bit of time to our fake server thread to accept
# the incoming connection from the client we actually need
# to sleep just a bit of time:
time.sleep(0.1)
if not chk.con.connected:
asyncore.poll2(0)
self.assertTrue(
fake_server.cli_socks,
'the client should have connected to our fake server.\n'
'-> %s' % chk.con.message)
# that should make the client to send us its request:
asyncore.poll2(0)
# give some time to the server thread to read it and
# send its response:
time.sleep(0.1)
m = mock.MagicMock(side_effect=socket.error('boum'))
chk.con.recv = m # this is what will trigger the desired effect..
self.assertEqual('Sending request and waiting response..',
chk.con.message)
# that should make the client to have its recv() method called:
asyncore.poll2(0)
self.assertEqual("Error on read: boum", chk.con.message)
save_con = chk.con # we have to retain the con because its unset..
log_mock = mock.MagicMock(wraps=booster_nrpe.logger)
booster_nrpe.logger = log_mock
# ..by manage_finished_checks :
inst.manage_finished_checks()
log_mock.warning.assert_called_once_with(
'%s: Got an IO error (%s), retrying 1 more time.. (cur=%s)',
chk.command, save_con.message, 0)
self.assertEqual('queue', chk.status)
self.assertEqual(1, chk.retried,
"the client has got the error we raised")
# now the check is going to be relaunched:
inst.launch_new_checks()
# this makes sure for it to be fully processed.
for _ in range(2):
asyncore.poll2(0)
time.sleep(0.1)
inst.manage_finished_checks()
log_mock.info.assert_called_once_with(
'%s: Successfully retried check :)', command)
self.assertEqual(
[], inst.checks,
"the check should have be moved out to the nrpe internal checks list"
)
inst.returns_queue.put.assert_called_once_with(chk)
self.assertEqual(0, chk.exit_status)
self.assertEqual(1, chk.retried)
def test_retry_on_io_error(self):
"""
:return:
"""
self.print_header()
# Obliged to call to get a self.logger...
self.setup_with_file('cfg/cfg_default.cfg')
self.assertTrue(self.conf_is_correct)
fake_server = self.fake_server
my_module = self._setup_nrpe()
my_module.returns_queue = mock.MagicMock()
# We prepare a check in the to_queue
command = (
"$USER1$/check_nrpe -H 127.0.0.1 -p %s -n -u -t 5 -c check_load3 -a 20"
% fake_server.port)
data = {
'is_a': 'check',
'status': 'queue',
'command': command,
'timeout': 10,
'poller_tag': None,
't_to_go': time.time(),
'ref': None,
}
chk = Check(data)
# GO
my_module.add_new_check(chk)
self.assertFalse(
fake_server.cli_socks,
'there should have no connected client to our fake server at this point'
)
my_module.launch_new_checks()
self.assertEqual('launched', chk.status)
self.assertEqual(0, chk.retried)
self.assertEqual('Sending request and waiting response...',
chk.con.message, "what? chk=%s " % chk)
# launch_new_checks() really launch a new check :
# it creates the nrpe client and directly make it to connect
# to the server.
# To give a bit of time to our fake server thread to accept
# the incoming connection from the client we actually need
# to sleep just a bit of time:
time.sleep(0.1)
# if not chk.con.connected:
# asyncore.poll2(0)
# if not chk.con.connected:
# asyncore.poll2(0)
self.assertTrue(
fake_server.cli_socks,
'the client should have connected to our fake server.\n'
'-> %s' % chk.con.message)
# that should make the client to send us its request:
asyncore.poll2(0)
# give some time to the server thread to read it and
# send its response:
time.sleep(0.1)
m = mock.MagicMock(side_effect=socket.error('boum'))
chk.con.recv = m # this is what will trigger the desired effect..
self.assertEqual('Sending request and waiting response...',
chk.con.message)
# that should make the client to have its recv() method called:
asyncore.poll2(0)
self.assertEqual("Error on read: boum", chk.con.message)
save_con = chk.con # we have to retain the con because its unset..
# Clear logs
self.clear_logs()
# ..by manage_finished_checks :
my_module.manage_finished_checks()
self.assert_log_match(
re.escape(
'%s: Got an IO error (%s), retrying 1 more time.. (cur=%s)' %
(chk.command, save_con.message, 0)), 0)
self.assertEqual('queue', chk.status)
self.assertEqual(1, chk.retried,
"the client has got the error we raised")
# now the check is going to be relaunched:
my_module.launch_new_checks()
# this makes sure for it to be fully processed.
for _ in range(2):
asyncore.poll2(0)
time.sleep(0.1)
my_module.manage_finished_checks()
self.assert_any_log_match(
re.escape('%s: Successfully retried check' % (chk.command)))
self.assertEqual(
[], my_module.checks,
"the check should have be moved out to the nrpe internal checks list"
)
my_module.returns_queue.put.assert_called_once_with(chk)
self.assertEqual(0, chk.exit_status)
self.assertEqual(1, chk.retried)
def handleRequestIfReady(self, sleepTime):
asyncore.poll2(timeout=sleepTime, map=self._map)
def update_event(self, inp=-1):
self.set_output_val(0, asyncore.poll2(self.input(0), self.input(1)))
def serve_once(self):
asyncore.poll2(map=self._map)
def handleRequestIfReady(self, sleepTime):
asyncore.poll2(timeout=sleepTime, map=self._map)
def test_retry_on_io_error(self):
fake_server = self.fake_server
inst = self._setup_nrpe(modconf)
inst.returns_queue = mock.MagicMock()
# We prepare a check in the to_queue
command = ("$USER1$/check_nrpe -H 127.0.0.1 -p %s -n -u -t 5 -c check_load3 -a 20"
% fake_server.port)
chk = Check('queue', command, None, time.time())
# GO
inst.add_new_check(chk)
self.assertFalse(fake_server.cli_socks,
'there should have no connected client '
'to our fake server at this point')
inst.launch_new_checks()
self.assertEqual('launched', chk.status)
self.assertEqual(0, chk.retried)
self.assertEqual('Sending request and waiting response..',
chk.con.message,
"what? chk=%s " % chk)
# launch_new_checks() really launch a new check :
# it creates the nrpe client and directly make it to connect
# to the server.
# To give a bit of time to our fake server thread to accept
# the incoming connection from the client we actually need
# to sleep just a bit of time:
time.sleep(0.1)
if not chk.con.connected:
asyncore.poll2(0)
self.assertTrue(fake_server.cli_socks,
'the client should have connected to our fake server.\n'
'-> %s' % chk.con.message)
# that should make the client to send us its request:
asyncore.poll2(0)
# give some time to the server thread to read it and
# send its response:
time.sleep(0.1)
m = mock.MagicMock(side_effect=socket.error('boum'))
chk.con.recv = m # this is what will trigger the desired effect..
self.assertEqual('Sending request and waiting response..',
chk.con.message)
# that should make the client to have its recv() method called:
asyncore.poll2(0)
self.assertEqual("Error on read: boum", chk.con.message)
save_con = chk.con # we have to retain the con because its unset..
log_mock = mock.MagicMock(wraps=booster_nrpe.logger)
booster_nrpe.logger = log_mock
# ..by manage_finished_checks :
inst.manage_finished_checks()
log_mock.warning.assert_called_once_with(
'%s: Got an IO error (%s), retrying 1 more time.. (cur=%s)',
chk.command, save_con.message, 0)
self.assertEqual('queue', chk.status)
self.assertEqual(1, chk.retried,
"the client has got the error we raised")
# now the check is going to be relaunched:
inst.launch_new_checks()
# this makes sure for it to be fully processed.
for _ in range(2):
asyncore.poll2(0)
time.sleep(0.1)
inst.manage_finished_checks()
log_mock.info.assert_called_once_with(
'%s: Successfully retried check :)', command)
self.assertEqual(
[], inst.checks,
"the check should have be moved out to the nrpe internal checks list")
inst.returns_queue.put.assert_called_once_with(chk)
self.assertEqual(0, chk.exit_status)
self.assertEqual(1, chk.retried)
print time.asctime(), 'received %s' % body
print sent
try: del(sent[body])
except KeyError: pass
ch.basic_ack(delivery_tag=meth.delivery_tag)
for i in range(N):
body = str(random())
publisher_ch.basic_publish(exchange='', routing_key=Q, body=body)
sent[body] = 1
attempts = N
while attempts > 0:
attempts -= 1
print attempts
asyncore.poll2(timeout=1.0)
if len(sent) == N and len(consumer_ch.callbacks) == 0:
print 'Starting consumer'
consumer_ch.basic_consume(handle_delivery, queue=Q)
elif len(sent) == 0:
print 'All consumed!'
break
if attempts == 0:
print 'failed. did you run step1? did you run remsh stuff?'
else:
print 'ok'
# exiting
jerry_conn.close()
tom_conn.close()
def test_no_command(self):
""" No command required
:return:
"""
self.print_header()
# Obliged to call to get a self.logger...
self.setup_with_file('cfg/cfg_default.cfg')
self.assertTrue(self.conf_is_correct)
fake_server = self.fake_server
my_module = self._setup_nrpe()
my_module.returns_queue = mock.MagicMock()
# We prepare a check in the to_queue
command = ("$USER1$/check_nrpe -H 127.0.0.1 -p %s -n -u -t 5 -a 20" %
fake_server.port)
data = {
'is_a': 'check',
'status': 'queue',
'command': command,
'timeout': 10,
'poller_tag': None,
't_to_go': time.time(),
'ref': None,
}
chk = Check(data)
# GO
my_module.add_new_check(chk)
self.assertFalse(
fake_server.cli_socks,
'there should have no connected client to our fake server at this point'
)
# Clear logs
self.clear_logs()
my_module.launch_new_checks()
# Check is launched
self.assertEqual('launched', chk.status)
self.assertIsNotNone(chk.con)
self.assertEqual(0, chk.retried)
self.assertEqual('Sending request and waiting response...',
chk.con.message, "what? chk=%s " % chk)
# launch_new_checks() really launch a new check :
# it creates the nrpe client and directly make it to connect
# to the server.
# To give a bit of time to our fake server thread to accept
# the incoming connection from the client we actually need
# to sleep just a bit of time:
time.sleep(0.3)
# if not chk.con.connected:
# asyncore.poll2(0)
self.assertTrue(
fake_server.cli_socks,
'the client should have connected to our fake server.\n'
'-> %s' % chk.con.message)
# this makes sure for it to be fully processed.
for _ in range(2):
asyncore.poll2(0)
time.sleep(0.1)
my_module.manage_finished_checks()
self.assertEqual(
[], my_module.checks,
"the check should have be moved out from the nrpe internal checks list"
)
my_module.returns_queue.put.assert_called_once_with(chk)
self.assertEqual(0, chk.exit_status)
self.assertEqual(0, chk.retried)
