def run(self):
"""Consume item from queue and process it.
It is the target function of Process class. Consumes items from
the queue, processes data which are pulled down by haproxystats-pull
program and uses Pandas to perform all computations of statistics.
It exits when it receives STOP_SIGNAL as item.
To avoid orphan processes on the system, it must be robust against
failures and try very hard recover from failures.
"""
if self.config.has_section('local-store'):
self.local_store = self.config.get('local-store', 'dir')
self.file_handler = FileHandler()
dispatcher.register('open', self.file_handler.open)
dispatcher.register('send', self.file_handler.send)
dispatcher.register('flush', self.file_handler.flush)
dispatcher.register('loop', self.file_handler.loop)
timeout = self.config.getfloat('graphite', 'timeout')
connect_timeout = self.config.getfloat('graphite',
'connect-timeout',
fallback=timeout)
write_timeout = self.config.getfloat('graphite',
'write-timeout',
fallback=timeout)
graphite = GraphiteHandler(
server=self.config.get('graphite', 'server'),
port=self.config.getint('graphite', 'port'),
connect_timeout=connect_timeout,
write_timeout=write_timeout,
retries=self.config.getint('graphite', 'retries'),
interval=self.config.getfloat('graphite', 'interval'),
delay=self.config.getfloat('graphite', 'delay'),
backoff=self.config.getfloat('graphite', 'backoff'),
queue_size=self.config.getint('graphite', 'queue-size'))
dispatcher.register('open', graphite.open)
dispatcher.register('send', graphite.send)
dispatcher.signal('open')
try:
while True:
log.info('waiting for item from the queue')
incoming_dir = self.tasks.get()
log.info('received item %s', incoming_dir)
if incoming_dir == STOP_SIGNAL:
break
start_time = time.time()
# incoming_dir => /var/lib/haproxystats/incoming/1454016646
# timestamp => 1454016646
self.timestamp = os.path.basename(incoming_dir)
# update filename for file handler.
# This *does not* error if a file handler is not registered.
dispatcher.signal('loop',
local_store=self.local_store,
timestamp=self.timestamp)
self.process_stats(incoming_dir)
# This flushes data to file
dispatcher.signal('flush')
# Remove directory as data have been successfully processed.
log.debug('removing %s', incoming_dir)
try:
shutil.rmtree(incoming_dir)
except (FileNotFoundError, PermissionError, OSError) as exc:
log.critical(
'failed to remove directory %s with:%s. '
'This should not have happened as it means '
'another worker processed data from this '
'directory or something/someone removed the '
'directory!', incoming_dir, exc)
elapsed_time = time.time() - start_time
log.info('total wall clock time in seconds %.3f', elapsed_time)
data = ("{p}.haproxystats.{m} {v} {t}\n".format(
p=self.graphite_path,
m='TotalWallClockTime',
v="{t:.3f}".format(t=elapsed_time),
t=self.timestamp))
dispatcher.signal('send', data=data)
log.info('finished with %s', incoming_dir)
except KeyboardInterrupt:
log.critical('Ctrl-C received')
return
def run(self):
"""Consume item from queue and process it.
It is the target function of Process class. Consumes items from
the queue, processes data which are pulled down by haproxystats-pull
program and uses Pandas to perform all computations of statistics.
It exits when it receives STOP_SIGNAL as item.
To avoid orphan processes on the system, it must be robust against
failures and try very hard recover from failures.
"""
if self.config.has_section('local-store'):
self.local_store = self.config.get('local-store', 'dir')
self.file_handler = FileHandler()
dispatcher.register('open', self.file_handler.open)
dispatcher.register('send', self.file_handler.send)
dispatcher.register('flush', self.file_handler.flush)
dispatcher.register('loop', self.file_handler.loop)
timeout = self.config.getfloat('graphite', 'timeout')
connect_timeout = self.config.getfloat('graphite',
'connect-timeout',
fallback=timeout)
write_timeout = self.config.getfloat('graphite',
'write-timeout',
fallback=timeout)
graphite = GraphiteHandler(
server=self.config.get('graphite', 'server'),
port=self.config.getint('graphite', 'port'),
connect_timeout=connect_timeout,
write_timeout=write_timeout,
retries=self.config.getint('graphite', 'retries'),
interval=self.config.getfloat('graphite', 'interval'),
delay=self.config.getfloat('graphite', 'delay'),
backoff=self.config.getfloat('graphite', 'backoff'),
queue_size=self.config.getint('graphite', 'queue-size')
)
dispatcher.register('open', graphite.open)
dispatcher.register('send', graphite.send)
dispatcher.signal('open')
try:
while True:
log.info('waiting for item from the queue')
incoming_dir = self.tasks.get()
log.info('received item %s', incoming_dir)
if incoming_dir == STOP_SIGNAL:
break
start_time = time.time()
# incoming_dir => /var/lib/haproxystats/incoming/1454016646
# timestamp => 1454016646
self.timestamp = os.path.basename(incoming_dir)
# update filename for file handler.
# This *does not* error if a file handler is not registered.
dispatcher.signal('loop',
local_store=self.local_store,
timestamp=self.timestamp)
self.process_stats(incoming_dir)
# This flushes data to file
dispatcher.signal('flush')
# Remove directory as data have been successfully processed.
log.debug('removing %s', incoming_dir)
try:
shutil.rmtree(incoming_dir)
except (FileNotFoundError, PermissionError, OSError) as exc:
log.critical('failed to remove directory %s with:%s. '
'This should not have happened as it means '
'another worker processed data from this '
'directory or something/someone removed the '
'directory!', incoming_dir, exc)
elapsed_time = time.time() - start_time
log.info('total wall clock time in seconds %.3f', elapsed_time)
data = ("{p}.haproxystats.{m} {v} {t}\n"
.format(p=self.graphite_path,
m='TotalWallClockTime',
v="{t:.3f}".format(t=elapsed_time),
t=self.timestamp))
dispatcher.signal('send', data=data)
log.info('finished with %s', incoming_dir)
except KeyboardInterrupt:
log.critical('Ctrl-C received')
return