def do_work(self, run_type, group):
if run_type == "poll":
info("Polling billing")
LibreNMS.call_script("poll-billing.php")
else: # run_type == 'calculate'
info("Calculating billing")
LibreNMS.call_script("billing-calculate.php")
def create_lock_manager(self):
"""
Create a new LockManager. Tries to create a Redis LockManager, but falls
back to python's internal threading lock implementation.
Exits if distributing poller is enabled and a Redis LockManager cannot be created.
:return: Instance of LockManager
"""
try:
return LibreNMS.RedisLock(
namespace='librenms.lock',
host=self.config.redis_host,
port=self.config.redis_port,
db=self.config.redis_db,
password=self.config.redis_pass,
unix_socket_path=self.config.redis_socket,
sentinel=self.config.redis_sentinel,
sentinel_service=self.config.redis_sentinel_service,
socket_timeout=self.config.redis_timeout)
except ImportError:
if self.config.distributed:
critical(
"ERROR: Redis connection required for distributed polling")
critical(
"Please install redis-py, either through your os software repository or from PyPI"
)
self.exit(2)
except Exception as e:
if self.config.distributed:
critical(
"ERROR: Redis connection required for distributed polling")
critical("Could not connect to Redis. {}".format(e))
self.exit(2)
return LibreNMS.ThreadingLock()
def do_work(self, context, group):
if self.lock(group, 'group', timeout=self.config.ping.frequency):
try:
info("Running fast ping")
LibreNMS.call_script('ping.php', ('-g', group))
finally:
self.unlock(group, 'group')
def do_work(self, context, group):
if self.lock(group, 'group', timeout=self.config.ping.frequency):
try:
info("Running fast ping")
LibreNMS.call_script('ping.php', ('-g', group))
finally:
self.unlock(group, 'group')
def do_work(self, run_type, group):
if run_type == 'poll':
info("Polling billing")
LibreNMS.call_script('poll-billing.php')
else: # run_type == 'calculate'
info("Calculating billing")
LibreNMS.call_script('billing-calculate.php')
def __init__(self):
self.start_time = time.time()
self.config.populate()
self._db = LibreNMS.DB(self.config)
self.config.load_poller_config(self._db)
threading.current_thread(
).name = self.config.name # rename main thread
self.attach_signals()
self._lm = self.create_lock_manager()
self.daily_timer = LibreNMS.RecurringTimer(
self.config.update_frequency, self.run_maintenance, 'maintenance')
self.stats_timer = LibreNMS.RecurringTimer(
self.config.poller.frequency, self.log_performance_stats,
'performance')
if self.config.watchdog_enabled:
info("Starting watchdog timer for log file: {}".format(
self.config.watchdog_logfile))
self.watchdog_timer = LibreNMS.RecurringTimer(
self.config.poller.frequency, self.logfile_watchdog,
'watchdog')
else:
info("Watchdog is disabled.")
self.is_master = False
def __init__(self):
self.config.populate()
threading.current_thread(
).name = self.config.name # rename main thread
self.attach_signals()
# init database connections different ones for different threads
self._db = LibreNMS.DB(self.config) # main
self._services_db = LibreNMS.DB(self.config) # services dispatch
self._discovery_db = LibreNMS.DB(self.config) # discovery dispatch
self._lm = self.create_lock_manager()
self.daily_timer = LibreNMS.RecurringTimer(
self.config.update_frequency, self.run_maintenance, 'maintenance')
self.stats_timer = LibreNMS.RecurringTimer(
self.config.poller.frequency, self.log_performance_stats,
'performance')
self.is_master = False
self.performance_stats = {
'poller': PerformanceCounter(),
'discovery': PerformanceCounter(),
'services': PerformanceCounter()
}
def _create_queue(self, queue_type, group):
"""
Create a queue (not thread safe)
:param queue_type:
:param group:
:return:
"""
info("Creating queue {}".format(self.queue_name(queue_type, group)))
try:
return LibreNMS.RedisUniqueQueue(
self.queue_name(queue_type, group),
namespace='librenms.queue',
host=self.config.redis_host,
port=self.config.redis_port,
db=self.config.redis_db,
password=self.config.redis_pass,
unix_socket_path=self.config.redis_socket)
except ImportError:
if self.config.distributed:
critical(
"ERROR: Redis connection required for distributed polling")
critical(
"Please install redis-py, either through your os software repository or from PyPI"
)
exit(2)
except Exception as e:
if self.config.distributed:
critical(
"ERROR: Redis connection required for distributed polling")
critical("Could not connect to Redis. {}".format(e))
exit(2)
return LibreNMS.UniqueQueue()
def do_work(self, context, group):
if self.lock(group, "group", timeout=self.config.ping.frequency):
try:
info("Running fast ping")
LibreNMS.call_script("ping.php", ("-g", group))
finally:
self.unlock(group, "group")
def do_work(self, device_id, group):
try:
info("Checking alerts")
LibreNMS.call_script('alerts.php')
except subprocess.CalledProcessError as e:
if e.returncode == 1:
warning("There was an error issuing alerts: {}".format(e.output))
else:
raise
def do_work(self, device_id, group):
try:
info("Checking alerts")
LibreNMS.call_script("alerts.php")
except subprocess.CalledProcessError as e:
if e.returncode == 1:
warning("There was an error issuing alerts: {}".format(
e.output))
else:
raise
def __init__(self):
self.config.populate()
threading.current_thread().name = self.config.name # rename main thread
self.attach_signals()
self._db = LibreNMS.DB(self.config)
self._lm = self.create_lock_manager()
self.daily_timer = LibreNMS.RecurringTimer(self.config.update_frequency, self.run_maintenance, 'maintenance')
self.stats_timer = LibreNMS.RecurringTimer(self.config.poller.frequency, self.log_performance_stats, 'performance')
self.is_master = False
def do_work(self, device_id, group):
if self.lock(device_id, timeout=self.config.services.frequency):
try:
info("Checking services on device {}".format(device_id))
LibreNMS.call_script('check-services.php', ('-h', device_id))
except subprocess.CalledProcessError as e:
if e.returncode == 5:
info("Device {} is down, cannot poll service, waiting {}s for retry"
.format(device_id, self.config.down_retry))
self.lock(device_id, allow_relock=True, timeout=self.config.down_retry)
else:
self.unlock(device_id)
def do_work(self, run_type, group):
if run_type == "poll":
logger.info("Polling billing")
exit_code, output = LibreNMS.call_script("poll-billing.php")
if exit_code != 0:
logger.warning("Error {} in Polling billing:\n{}".format(
exit_code, output))
else: # run_type == 'calculate'
logger.info("Calculating billing")
exit_code, output = LibreNMS.call_script("billing-calculate.php")
if exit_code != 0:
logger.warning("Error {} in Calculating billing:\n{}".format(
exit_code, output))
def do_work(self, device_id, group):
if self.lock(device_id, timeout=LibreNMS.normalize_wait(self.config.discovery.frequency)):
try:
info("Discovering device {}".format(device_id))
LibreNMS.call_script('discovery.php', ('-h', device_id))
except subprocess.CalledProcessError as e:
if e.returncode == 5:
info("Device {} is down, cannot discover, waiting {}s for retry"
.format(device_id, self.config.down_retry))
self.lock(device_id, allow_relock=True, timeout=self.config.down_retry)
else:
self.unlock(device_id)
else:
self.unlock(device_id)
def do_work(self, device_id, group):
if self.lock(device_id, timeout=self.config.services.frequency):
try:
info("Checking services on device {}".format(device_id))
LibreNMS.call_script("check-services.php", ("-h", device_id))
except subprocess.CalledProcessError as e:
if e.returncode == 5:
info(
"Device {} is down, cannot poll service, waiting {}s for retry"
.format(device_id, self.config.down_retry))
self.lock(device_id,
allow_relock=True,
timeout=self.config.down_retry)
else:
self.unlock(device_id)
def run_maintenance(self):
"""
Runs update and cleanup tasks by calling daily.sh. Reloads the python script after the update.
Sets a schema-update lock so no distributed pollers will update until the schema has been updated.
"""
attempt = 0
wait = 5
max_runtime = 86100
max_tries = int(max_runtime / wait)
logger.info("Waiting for schema lock")
while not self._lm.lock("schema-update", self.config.unique_name, max_runtime):
attempt += 1
if attempt >= max_tries: # don't get stuck indefinitely
logger.warning(
"Reached max wait for other pollers to update, updating now"
)
break
sleep(wait)
logger.info("Running maintenance tasks")
exit_code, output = LibreNMS.call_script("daily.sh")
if exit_code == 0:
logger.info("Maintenance tasks complete\n{}".format(output))
else:
logger.error("Error {} in daily.sh:\n{}".format(exit_code, output))
self._lm.unlock("schema-update", self.config.unique_name)
self.restart()
def do_work(self, device_id, group):
if self.lock(device_id, timeout=self.config.poller.frequency):
logger.info("Polling device {}".format(device_id))
exit_code, output = LibreNMS.call_script("poller.php",
("-h", device_id))
if exit_code == 0:
self.unlock(device_id)
else:
if exit_code == 6:
logger.warning(
"Polling device {} unreachable, waiting {}s for retry".
format(device_id, self.config.down_retry))
# re-lock to set retry timer
self.lock(device_id,
allow_relock=True,
timeout=self.config.down_retry)
else:
logger.error(
"Polling device {} failed with exit code {}: {}".
format(device_id, exit_code, output))
self.unlock(device_id)
else:
logger.debug(
"Tried to poll {}, but it is locked".format(device_id))
def run_maintenance(self):
"""
Runs update and cleanup tasks by calling daily.sh. Reloads the python script after the update.
Sets a schema-update lock so no distributed pollers will update until the schema has been updated.
"""
attempt = 0
wait = 5
max_runtime = 86100
max_tries = int(max_runtime / wait)
info("Waiting for schema lock")
while not self._lm.lock('schema-update', self.config.unique_name,
max_runtime):
attempt += 1
if attempt >= max_tries: # don't get stuck indefinitely
warning(
'Reached max wait for other pollers to update, updating now'
)
break
sleep(wait)
info("Running maintenance tasks")
try:
output = LibreNMS.call_script('daily.sh')
info("Maintenance tasks complete\n{}".format(output))
except subprocess.CalledProcessError as e:
error("Error in daily.sh:\n" +
(e.output.decode() if e.output is not None else 'No output'))
self._lm.unlock('schema-update', self.config.unique_name)
self.restart()
def test_redis_queue(self):
if 'redis' not in sys.modules:
self.assertTrue(True, 'Skipped Redis tests')
else:
rc = redis.Redis()
rc.delete('queue:testing') # make sure no previous data exists
qm = LibreNMS.RedisUniqueQueue('testing', namespace='queue')
thread = threading.Thread(target=self.queue_thread,
args=(qm, None, False))
thread.daemon = True
thread.start()
thread = threading.Thread(target=self.queue_thread, args=(qm, '2'))
thread.daemon = True
thread.start()
qm.put(2)
qm.put(3)
qm.put(4)
sleep(0.05)
self.assertEqual(2, qm.qsize())
self.assertEqual('3', qm.get())
self.assertEqual('4', qm.get(), "Did not get second item in queue")
self.assertEqual(None, qm.get_nowait(),
"Did not get None when queue should be empty")
self.assertTrue(qm.empty(), "Queue should be empty")
def test_threading_lock(self):
lm = LibreNMS.ThreadingLock()
thread = threading.Thread(target=self.lock_thread,
args=(lm, 'first.lock', 2, 1))
thread.daemon = True
thread.start()
sleep(0.05)
self.assertFalse(lm.lock('first.lock', 'main_thread', 0),
"Acquired lock when it is held by thread")
self.assertFalse(lm.unlock('first.lock', 'main_thread'),
"Unlocked lock main doesn't own")
sleep(1.1)
self.assertTrue(lm.lock('first.lock', 'main_thread', 1),
"Could not acquire lock previously held by thread")
self.assertFalse(lm.lock('first.lock', 'main_thread', 1, False),
"Was able to re-lock a lock main owns")
self.assertTrue(lm.lock('first.lock', 'main_thread', 1, True),
"Could not re-lock a lock main owns")
self.assertTrue(lm.check_lock('first.lock'))
self.assertTrue(lm.unlock('first.lock', 'main_thread'),
"Could not unlock lock main holds")
self.assertFalse(lm.unlock('first.lock', 'main_thread'),
"Unlocked an unlocked lock?")
self.assertFalse(lm.check_lock('first.lock'))
def test_redis_lock(self):
if 'redis' not in sys.modules:
self.assertTrue(True, 'Skipped Redis tests')
else:
rc = redis.Redis()
rc.delete('lock:redis.lock') # make sure no previous data exists
lm = LibreNMS.RedisLock(namespace='lock')
thread = threading.Thread(target=self.lock_thread,
args=(lm, 'redis.lock', 2, 1))
thread.daemon = True
thread.start()
sleep(0.05)
self.assertFalse(lm.lock('redis.lock', 'main_thread', 1),
"Acquired lock when it is held by thread")
self.assertFalse(lm.unlock('redis.lock', 'main_thread'),
"Unlocked lock main doesn't own")
sleep(1.1)
self.assertTrue(
lm.lock('redis.lock', 'main_thread', 1),
"Could not acquire lock previously held by thread")
self.assertFalse(lm.lock('redis.lock', 'main_thread', 1),
"Relocked an existing lock")
self.assertTrue(lm.lock('redis.lock', 'main_thread', 1, True),
"Could not re-lock a lock main owns")
self.assertTrue(lm.unlock('redis.lock', 'main_thread'),
"Could not unlock lock main holds")
self.assertFalse(lm.unlock('redis.lock', 'main_thread'),
"Unlocked an unlocked lock?")
def do_work(self, device_id, group):
logger.info("Checking alerts")
exit_code, output = LibreNMS.call_script("alerts.php")
if exit_code != 0:
if exit_code == 1:
logger.warning(
"There was an error issuing alerts: {}".format(output))
else:
raise CalledProcessError
def __init__(self, config, lock_manager):
"""
A TimedQueueManager to manage dispatch and workers for Alerts
:param config: LibreNMS.ServiceConfig reference to the service config object
:param lock_manager: the single instance of lock manager
"""
TimedQueueManager.__init__(self, config, lock_manager, "alerting")
self._db = LibreNMS.DB(self.config)
def do_work(self, device_id, group):
if self.lock(device_id,
timeout=LibreNMS.normalize_wait(
self.config.discovery.frequency)):
try:
info("Discovering device {}".format(device_id))
LibreNMS.call_script("discovery.php", ("-h", device_id))
except subprocess.CalledProcessError as e:
if e.returncode == 5:
info(
"Device {} is down, cannot discover, waiting {}s for retry"
.format(device_id, self.config.down_retry))
self.lock(device_id,
allow_relock=True,
timeout=self.config.down_retry)
else:
self.unlock(device_id)
else:
self.unlock(device_id)
def do_work(self, device_id, group):
if self.lock(device_id, timeout=self.config.poller.frequency):
info('Polling device {}'.format(device_id))
try:
LibreNMS.call_script('poller.php', ('-h', device_id))
except subprocess.CalledProcessError as e:
if e.returncode == 6:
warning('Polling device {} unreachable, waiting {}s for retry'.format(device_id,
self.config.down_retry))
# re-lock to set retry timer
self.lock(device_id, allow_relock=True, timeout=self.config.down_retry)
else:
error('Polling device {} failed! {}'.format(device_id, e))
self.unlock(device_id)
else:
self.unlock(device_id)
else:
debug('Tried to poll {}, but it is locked'.format(device_id))
def __init__(self, config, lock_manager):
"""
A TimedQueueManager with two timers dispatching poll billing and calculate billing to the same work queue
:param config: LibreNMS.ServiceConfig reference to the service config object
:param lock_manager: the single instance of lock manager
"""
TimedQueueManager.__init__(self, config, lock_manager, 'billing')
self.calculate_timer = LibreNMS.RecurringTimer(
self.get_poller_config().calculate,
self.dispatch_calculate_billing, 'calculate_billing_timer')
def __init__(self, config, type_desc, work_function, dispatch_function, auto_start=True):
"""
A queue manager that periodically dispatches work to the queue
The times are normalized like they started at 0:00
:param config: LibreNMS.ServiceConfig reference to the service config object
:param type_desc: description for this queue manager type
:param work_function: function that will be called to perform the task
:param dispatch_function: function that will be called when the timer is up, should call post_work()
:param auto_start: automatically start worker threads
"""
QueueManager.__init__(self, config, type_desc, work_function, auto_start)
self.timer = LibreNMS.RecurringTimer(self.get_poller_config().frequency, dispatch_function)
def do_work(self, context, group):
if self.lock(group, "group", timeout=self.config.ping.frequency):
try:
logger.info("Running fast ping")
exit_code, output = LibreNMS.call_script(
"ping.php", ("-g", group))
if exit_code != 0:
logger.warning(
"Running fast ping for {} failed with error code {}: {}"
.format(group, exit_code, output))
finally:
self.unlock(group, "group")
def __init__(self, config, work_function, poll_dispatch_function, calculate_dispatch_function,
auto_start=True):
"""
A TimedQueueManager with two timers dispatching poll billing and calculate billing to the same work queue
:param config: LibreNMS.ServiceConfig reference to the service config object
:param work_function: function that will be called to perform the task
:param poll_dispatch_function: function that will be called when the timer is up, should call post_work()
:param calculate_dispatch_function: function that will be called when the timer is up, should call post_work()
:param auto_start: automatically start worker threads
"""
TimedQueueManager.__init__(self, config, 'billing', work_function, poll_dispatch_function, auto_start)
self.calculate_timer = LibreNMS.RecurringTimer(self.get_poller_config().calculate, calculate_dispatch_function, 'calculate_billing_timer')
def do_work(self, device_id, group):
if self.lock(device_id, timeout=self.config.poller.frequency):
info("Polling device {}".format(device_id))
try:
LibreNMS.call_script("poller.php", ("-h", device_id))
except subprocess.CalledProcessError as e:
if e.returncode == 6:
warning(
"Polling device {} unreachable, waiting {}s for retry".
format(device_id, self.config.down_retry))
# re-lock to set retry timer
self.lock(device_id,
allow_relock=True,
timeout=self.config.down_retry)
else:
error("Polling device {} failed! {}".format(device_id, e))
self.unlock(device_id)
else:
self.unlock(device_id)
else:
debug("Tried to poll {}, but it is locked".format(device_id))
def do_work(self, device_id, group):
if self.lock(device_id,
timeout=LibreNMS.normalize_wait(
self.config.discovery.frequency)):
logger.info("Discovering device {}".format(device_id))
exit_code, output = LibreNMS.call_script("discovery.php",
("-h", device_id))
if exit_code == 0:
self.unlock(device_id)
else:
if exit_code == 5:
logger.info(
"Device {} is down, cannot discover, waiting {}s for retry"
.format(device_id, self.config.down_retry))
self.lock(device_id,
allow_relock=True,
timeout=self.config.down_retry)
else:
logger.error(
"Discovering device {} failed with exit code {}: {}".
format(device_id, exit_code, output))
self.unlock(device_id)
def __init__(self, config, lock_manager, auto_start=True):
"""
A TimedQueueManager to manage dispatch and workers for Alerts
:param config: LibreNMS.ServiceConfig reference to the service config object
:param lock_manager: the single instance of lock manager
:param auto_start: automatically start worker threads
"""
TimedQueueManager.__init__(self,
config,
lock_manager,
'alerting',
auto_start=auto_start)
self._db = LibreNMS.DB(self.config)
def test_recurring_timer(self):
self.assertEqual(0, self.counter)
timer = LibreNMS.RecurringTimer(0.5, self.count)
timer.start()
self.assertEqual(0, self.counter)
sleep(0.5)
self.assertEqual(1, self.counter)
self.assertEqual(1, self.counter)
sleep(0.5)
self.assertEqual(2, self.counter)
timer.stop()
self.assertTrue(timer._event.is_set())
sleep(0.5)
self.assertEqual(2, self.counter)
timer.start()
sleep(0.5)
self.assertEqual(3, self.counter)
timer.stop()
def __init__(self,
config,
lock_manager,
type_desc,
uses_groups=False,
auto_start=True):
"""
A queue manager that periodically dispatches work to the queue
The times are normalized like they started at 0:00
:param config: LibreNMS.ServiceConfig reference to the service config object
:param type_desc: description for this queue manager type
:param uses_groups: If this queue respects assigned groups or there is only one group
:param auto_start: automatically start worker threads
"""
QueueManager.__init__(self, config, lock_manager, type_desc,
uses_groups, auto_start)
self.timer = LibreNMS.RecurringTimer(
self.get_poller_config().frequency, self.do_dispatch)
def do_work(self, device_id, group):
if self.lock(device_id, timeout=self.config.services.frequency):
logger.info("Checking services on device {}".format(device_id))
exit_code, output = LibreNMS.call_script("check-services.php",
("-h", device_id))
if exit_code == 0:
self.unlock(device_id)
else:
if exit_code == 5:
logger.info(
"Device {} is down, cannot poll service, waiting {}s for retry"
.format(device_id, self.config.down_retry))
self.lock(device_id,
allow_relock=True,
timeout=self.config.down_retry)
else:
logger.warning(
"Unknown error while checking services on device {} with exit code {}: {}"
.format(device_id, exit_code, output))
def run_maintenance(self):
"""
Runs update and cleanup tasks by calling daily.sh. Reloads the python script after the update.
Sets a schema-update lock so no distributed pollers will update until the schema has been updated.
"""
attempt = 0
wait = 5
max_runtime = 86100
max_tries = int(max_runtime / wait)
info("Waiting for schema lock")
while not self._lm.lock('schema-update', self.config.unique_name, max_runtime):
attempt += 1
if attempt >= max_tries: # don't get stuck indefinitely
warning('Reached max wait for other pollers to update, updating now')
break
sleep(wait)
info("Running maintenance tasks")
output = LibreNMS.call_script('daily.sh')
info("Maintenance tasks complete\n{}".format(output))
self.restart()
def lock_discovery(self, device_id, retry=False):
lock_name = self.gen_lock_name('discovery', device_id)
timeout = self.config.down_retry if retry else LibreNMS.normalize_wait(self.config.discovery.frequency)
return self._lm.lock(lock_name, self.gen_lock_owner(), timeout, retry)
