def main():
"""Parse CLI arguments and launches main program."""
args = docopt(__doc__, version=VERSION)
config = ConfigParser(interpolation=ExtendedInterpolation())
# Set defaults for all sections
config.read_dict(copy.copy(DEFAULT_OPTIONS))
try:
config.read(args['--file'])
except ParsingError as exc:
sys.exit(str(exc))
incoming_dir = config.get('process', 'src-dir')
if args['--print']:
for section in sorted(DEFAULT_OPTIONS):
if section == 'pull':
continue
print("[{}]".format(section))
for key, value in sorted(DEFAULT_OPTIONS[section].items()):
print("{k} = {v}".format(k=key, v=value))
print()
sys.exit(0)
if args['--print-conf']:
for section in sorted(config):
if section == 'pull':
continue
print("[{}]".format(section))
for key, value in sorted(config[section].items()):
print("{k} = {v}".format(k=key, v=value))
print()
sys.exit(0)
try:
configuration_check(config, 'paths')
configuration_check(config, 'process')
configuration_check(config, 'graphite')
read_write_access(config.get('process', 'src-dir'))
check_metrics(config)
except ValueError as exc:
sys.exit(str(exc))
tasks = multiprocessing.Queue()
handler = EventHandler(tasks=tasks)
notifier = pyinotify.Notifier(watcher, handler)
num_consumers = config.getint('process', 'workers')
incoming_dir = config.get('process', 'src-dir')
loglevel =\
config.get('process', 'loglevel').upper() # pylint: disable=no-member
log.setLevel(getattr(logging, loglevel, None))
log.info('haproxystats-processs %s version started', VERSION)
# process incoming data which were retrieved while processing was stopped
for pathname in glob.iglob(incoming_dir + '/*'):
if os.path.isdir(pathname):
log.info('putting %s in queue', pathname)
tasks.put(pathname)
def shutdown(signalnb=None, frame=None):
"""Signal processes to exit.
It adds STOP_SIGNAL to the queue, which causes processes to exit in a
clean way.
Arguments:
signalnb (int): The ID of signal
frame (obj): Frame object at the time of receiving the signal
"""
log.info('received %s at %s', signalnb, frame)
notifier.stop()
for _ in range(num_consumers):
log.info('sending stop signal to worker')
tasks.put(STOP_SIGNAL)
log.info('waiting for workers to finish their work')
for consumer in consumers:
consumer.join()
log.info('exiting')
sys.exit(0)
# Register our graceful shutdown process to termination signals
signal.signal(signal.SIGHUP, shutdown)
signal.signal(signal.SIGTERM, shutdown)
# Add our watcher
while True:
try:
log.info('adding a watch for %s', incoming_dir)
watcher.add_watch(incoming_dir, MASK, quiet=False, rec=False)
except pyinotify.WatchManagerError as error:
log.error('received error (%s), going to retry in few seconds',
error)
time.sleep(3)
else:
break
log.info('creating %d consumers', num_consumers)
consumers = [Consumer(tasks, config) for i in range(num_consumers)]
for consumer in consumers:
consumer.start()
log.info('watching %s directory for incoming data', incoming_dir)
notifier.loop(daemonize=False)
def main():
"""Parse CLI arguments and launches main program."""
args = docopt(__doc__, version=VERSION)
config = ConfigParser(interpolation=ExtendedInterpolation())
# Set defaults for all sections
config.read_dict(copy.copy(DEFAULT_OPTIONS))
try:
config.read(args['--file'])
except ParsingError as exc:
sys.exit(str(exc))
incoming_dir = config.get('process', 'src-dir')
if args['--print']:
for section in sorted(DEFAULT_OPTIONS):
if section == 'pull':
continue
print("[{}]".format(section))
for key, value in sorted(DEFAULT_OPTIONS[section].items()):
print("{k} = {v}".format(k=key, v=value))
print()
sys.exit(0)
if args['--print-conf']:
for section in sorted(config):
if section == 'pull':
continue
print("[{}]".format(section))
for key, value in sorted(config[section].items()):
print("{k} = {v}".format(k=key, v=value))
print()
sys.exit(0)
try:
configuration_check(config, 'paths')
configuration_check(config, 'process')
configuration_check(config, 'graphite')
read_write_access(config.get('process', 'src-dir'))
check_metrics(config)
except ValueError as exc:
sys.exit(str(exc))
tasks = multiprocessing.Queue()
handler = EventHandler(tasks=tasks)
notifier = pyinotify.Notifier(watcher, handler)
num_consumers = config.getint('process', 'workers')
incoming_dir = config.get('process', 'src-dir')
loglevel =\
config.get('process', 'loglevel').upper() # pylint: disable=no-member
log.setLevel(getattr(logging, loglevel, None))
log.info('haproxystats-processs %s version started', VERSION)
# process incoming data which were retrieved while processing was stopped
for pathname in glob.iglob(incoming_dir + '/*'):
if os.path.isdir(pathname):
log.info('putting %s in queue', pathname)
tasks.put(pathname)
def shutdown(signalnb=None, frame=None):
"""Signal processes to exit.
It adds STOP_SIGNAL to the queue, which causes processes to exit in a
clean way.
Arguments:
signalnb (int): The ID of signal
frame (obj): Frame object at the time of receiving the signal
"""
log.info('received %s at %s', signalnb, frame)
notifier.stop()
for _ in range(num_consumers):
log.info('sending stop signal to worker')
tasks.put(STOP_SIGNAL)
log.info('waiting for workers to finish their work')
for consumer in consumers:
consumer.join()
log.info('exiting')
sys.exit(0)
# Register our graceful shutdown process to termination signals
signal.signal(signal.SIGHUP, shutdown)
signal.signal(signal.SIGTERM, shutdown)
# Add our watcher
while True:
try:
log.info('adding a watch for %s', incoming_dir)
watcher.add_watch(incoming_dir, MASK, quiet=False, rec=False)
except pyinotify.WatchManagerError as error:
log.error('received error (%s), going to retry in few seconds',
error)
time.sleep(3)
else:
break
log.info('creating %d consumers', num_consumers)
consumers = [Consumer(tasks, config) for i in range(num_consumers)]
for consumer in consumers:
consumer.start()
log.info('watching %s directory for incoming data', incoming_dir)
notifier.loop(daemonize=False)
def main():
"""Parse CLI arguments and launch main program."""
args = docopt(__doc__, version=VERSION)
config = ConfigParser(interpolation=ExtendedInterpolation())
# Set defaults for all sections
config.read_dict(copy.copy(DEFAULT_OPTIONS))
# Load configuration from a file. NOTE: ConfigParser doesn't warn if user
# sets a filename which doesn't exist, in this case defaults will be used.
try:
config.read(args['--file'])
except ParsingError as exc:
sys.exit(str(exc))
if args['--print']:
for section in sorted(DEFAULT_OPTIONS):
if section == 'pull' or section == 'DEFAULT':
print("[{}]".format(section))
for key, value in sorted(DEFAULT_OPTIONS[section].items()):
print("{k} = {v}".format(k=key, v=value))
print()
sys.exit(0)
if args['--print-conf']:
for section in sorted(config):
if section == 'pull' or section == 'DEFAULT':
print("[{}]".format(section))
for key, value in sorted(config[section].items()):
print("{k} = {v}".format(k=key, v=value))
print()
sys.exit(0)
try:
configuration_check(config, 'pull')
except ValueError as exc:
sys.exit(str(exc))
loglevel = (config.get('pull', 'loglevel') # pylint: disable=no-member
.upper())
log.setLevel(getattr(logging, loglevel, None))
log.info('haproxystats-pull %s version started', VERSION)
# Setup our event loop
loop = asyncio.get_event_loop()
executor = ThreadPoolExecutor(max_workers=config.getint('pull',
'workers'))
# Register shutdown to signals
def shutdown(signalname):
"""Perform a clean shutdown.
Arguments:
signalname (str): Signal name
"""
tasks_running = False
log.info('received %s', signalname)
for task in asyncio.Task.all_tasks():
if not task.done():
tasks_running = True
log.info('cancelling %s task', task)
task.cancel()
if not tasks_running:
log.info('no tasks were running when %s signal received', signal)
log.info('waiting for threads to finish any pending IO tasks')
executor.shutdown(wait=True)
sys.exit(0)
loop.add_signal_handler(signal.SIGHUP, partial(shutdown, 'SIGHUP'))
loop.add_signal_handler(signal.SIGTERM, partial(shutdown, 'SIGTERM'))
# a temporary directory to store fetched data
tmp_dst_dir = config['pull']['tmp-dst-dir']
# a permanent directory to move data from the temporary directory. Data are
# picked up by the process daemon from that directory.
dst_dir = config['pull']['dst-dir']
for directory in dst_dir, tmp_dst_dir:
try:
os.makedirs(directory)
except OSError as exc:
# errno 17 => file exists
if exc.errno != 17:
sys.exit("failed to make directory {d}:{e}"
.format(d=directory, e=exc))
supervisor(loop, config, executor)
def main():
"""Parse CLI arguments and launch main program."""
args = docopt(__doc__, version=VERSION)
config = ConfigParser(interpolation=ExtendedInterpolation())
# Set defaults for all sections
config.read_dict(copy.copy(DEFAULT_OPTIONS))
# Load configuration from a file. NOTE: ConfigParser doesn't warn if user
# sets a filename which doesn't exist, in this case defaults will be used.
try:
config.read(args['--file'])
except ParsingError as exc:
sys.exit(str(exc))
if args['--print']:
for section in sorted(DEFAULT_OPTIONS):
if section == 'pull' or section == 'DEFAULT':
print("[{}]".format(section))
for key, value in sorted(DEFAULT_OPTIONS[section].items()):
print("{k} = {v}".format(k=key, v=value))
print()
sys.exit(0)
if args['--print-conf']:
for section in sorted(config):
if section == 'pull' or section == 'DEFAULT':
print("[{}]".format(section))
for key, value in sorted(config[section].items()):
print("{k} = {v}".format(k=key, v=value))
print()
sys.exit(0)
try:
configuration_check(config, 'pull')
except ValueError as exc:
sys.exit(str(exc))
loglevel = (config.get('pull', 'loglevel') # pylint: disable=no-member
.upper())
log.setLevel(getattr(logging, loglevel, None))
log.info('haproxystats-pull %s version started', VERSION)
# Setup our event loop
loop = asyncio.get_event_loop()
executor = ThreadPoolExecutor(max_workers=config.getint('pull',
'workers'))
# Register shutdown to signals
def shutdown(signalname):
"""Perform a clean shutdown.
Arguments:
signalname (str): Signal name
"""
tasks_running = False
log.info('received %s', signalname)
for task in asyncio.Task.all_tasks():
if not task.done():
tasks_running = True
log.info('cancelling %s task', task)
task.cancel()
if not tasks_running:
log.info('no tasks were running when %s signal received', signal)
log.info('waiting for threads to finish any pending IO tasks')
executor.shutdown(wait=True)
sys.exit(0)
loop.add_signal_handler(signal.SIGHUP, partial(shutdown, 'SIGHUP'))
loop.add_signal_handler(signal.SIGTERM, partial(shutdown, 'SIGTERM'))
# a temporary directory to store fetched data
tmp_dst_dir = config['pull']['tmp-dst-dir']
# a permanent directory to move data from the temporary directory. Data are
# picked up by the process daemon from that directory.
dst_dir = config['pull']['dst-dir']
for directory in dst_dir, tmp_dst_dir:
try:
os.makedirs(directory)
except OSError as exc:
# errno 17 => file exists
if exc.errno != 17:
sys.exit("failed to make directory {d}:{e}"
.format(d=directory, e=exc))
supervisor(loop, config, executor)