class EventletInbox(object):
def __init__(self, logger=None):
''' __init__ '''
self.__inbox = EventletQueue()
if logger is None:
self._logger = getLogger('%s.EventletInbox' % __name__)
else:
self._logger = logger
def get(self):
''' get data from inbox '''
try:
result = self.__inbox.get_nowait()
except EventletEmpty:
raise EmptyInboxException
return result
def put(self, message):
''' put message to inbox '''
self.__inbox.put(message)
def __len__(self):
''' return length of inbox '''
return self.__inbox.qsize()
class StatsdLog(object):
"""Simple server to monitor a syslog udp stream for statsd events"""
def __init__(self, conf):
TRUE_VALUES = set(('true', '1', 'yes', 'on', 't', 'y'))
self.conf = conf
self.logger = logging.getLogger('statsdlogd')
self.logger.setLevel(logging.INFO)
self.syslog = SysLogHandler(address='/dev/log')
self.formatter = logging.Formatter('%(name)s: %(message)s')
self.syslog.setFormatter(self.formatter)
self.logger.addHandler(self.syslog)
self.debug = conf.get('debug', 'false').lower() in TRUE_VALUES
self.statsd_host = conf.get('statsd_host', '127.0.0.1')
self.statsd_port = int(conf.get('statsd_port', '8125'))
self.listen_addr = conf.get('listen_addr', '127.0.0.1')
self.listen_port = int(conf.get('listen_port', 8126))
self.report_internal_stats = conf.get('report_internal_stats',
'true').lower() in TRUE_VALUES
self.int_stats_interval = int(conf.get('internal_stats_interval', 5))
self.buff = int(conf.get('buffer_size', 8192))
self.max_q_size = int(conf.get('max_line_backlog', 512))
self.statsd_sample_rate = float(conf.get('statsd_sample_rate', '.5'))
self.counter = 0
self.skip_counter = 0
self.hits = 0
self.q = Queue(maxsize=self.max_q_size)
# key: regex
self.patterns_file = conf.get('patterns_file',
'/etc/statsdlog/patterns.json')
self.json_patterns = conf.get('json_pattern_file',
'true').lower() in TRUE_VALUES
try:
self.patterns = self.load_patterns()
except Exception as err:
self.logger.exception(err)
print "Encountered exception at startup: %s" % err
sys.exit(1)
self.statsd_addr = (self.statsd_host, self.statsd_port)
self.comp_patterns = {}
for item in self.patterns:
self.comp_patterns[item] = re.compile(self.patterns[item])
def load_patterns(self):
if self.json_patterns:
self.logger.info("Using json based patterns file: %s" %
self.patterns_file)
with open(self.patterns_file) as pfile:
return json.loads(pfile.read())
else:
self.logger.info("Using plain text patterns file: %s" %
self.patterns_file)
patterns = {}
with open(self.patterns_file) as f:
for line in f:
if line:
pattern = [x.strip() for x in line.split("=", 1)]
else:
pattern = None
if len(pattern) != 2:
# skip this line
self.logger.error("Skipping pattern. Unable to parse: %s" %
line)
else:
if pattern[0] and pattern[1]:
patterns[pattern[0]] = pattern[1]
else:
self.logger.error(
"Skipping pattern. Unable to parse: %s" % line)
return patterns
def check_line(self, line):
"""
Check if a line matches our search patterns.
:param line: The string to check
:returns: List of regex entries that matched (or empty list if none)
"""
matches = []
for entry in self.comp_patterns:
if self.comp_patterns[entry].match(line):
matches.append(entry)
return matches
def internal_stats(self):
"""
Periodically send our own stats to statsd.
"""
lastcount = 0
lasthit = 0
while True:
eventlet.sleep(self.int_stats_interval)
self.send_event("statsdlog.lines:%s|c" %
(self.counter - lastcount))
lastcount = self.counter
self.send_event("statsdlog.hits:%s|c" % (self.hits - lasthit))
lasthit = self.hits
def stats_print(self):
"""
Periodically dump some stats to the logs.
"""
lastcount = 0
lasthit = 0
while True:
eventlet.sleep(2)
lps = (self.counter - lastcount) / 60
hps = (self.hits - lasthit) / 60
lastcount = self.counter
lasthit = self.hits
self.logger.info('per second: %d lines - hits %d' % (lps, hps))
self.logger.info('totals: %d hits - %d lines' %
(self.hits, self.counter))
if self.skip_counter is not 0:
self.logger.info('Had to skip %d log lines so far' %
self.skip_counter)
def send_event(self, payload):
"""
Fire event to statsd
:param payload: The payload of the udp packet to send.
"""
try:
udp_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
udp_socket.sendto(payload, self.statsd_addr)
except Exception:
# udp sendto failed (socket already in use?), but thats ok
self.logger.error("Error trying to send statsd event")
def statsd_counter_increment(self, stats, delta=1):
"""
Increment multiple statsd stats counters
:param stats: list of stats items to package and send
:param delta: delta of stats items
"""
if self.statsd_sample_rate < 1:
if random() <= self.statsd_sample_rate:
for item in stats:
payload = "%s:%s|c|@%s" % (item, delta,
self.statsd_sample_rate)
self.send_event(payload)
else:
for item in stats:
payload = "%s:%s|c" % (item, delta)
self.send_event(payload)
def worker(self):
"""
Check for and process log lines in queue
"""
while True:
msg = self.q.get()
matches = self.check_line(msg)
for match in matches:
self.statsd_counter_increment([match])
if self.hits >= maxint:
self.logger.info("hit maxint, reset hits counter")
self.hits = 0
self.hits += 1
def listener(self):
"""
syslog udp listener
"""
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
bind_addr = (self.listen_addr, self.listen_port)
sock.bind(bind_addr)
self.logger.info("listening on %s:%d" % bind_addr)
while 1:
data, addr = sock.recvfrom(self.buff)
if not data:
break
else:
if self.q.qsize() < self.max_q_size:
self.q.put(data)
if self.counter >= maxint:
self.logger.info("hit maxint, reset seen counter")
self.counter = 0
self.counter += 1
else:
if self.debug:
self.logger.notice("max log lines in queue, skipping")
if self.skip_counter >= maxint:
self.logger.info("hit maxint, reset skip counter")
self.skip_counter = 0
self.skip_counter += 1
def start(self):
"""
Start the listener, worker, and mgmt server.
"""
eventlet.spawn_n(self.worker)
if self.debug:
eventlet.spawn_n(self.stats_print)
if self.report_internal_stats:
eventlet.spawn_n(self.internal_stats)
while True:
try:
self.listener()
except Exception as err:
self.logger.error(err)
class AsyncClient(baseasync.BaseAsync):
def __init__(self, *args, **kwargs):
super(AsyncClient, self).__init__(*args, **kwargs)
self.pool = eventlet.greenpool.GreenPool(DEFAULT_POOL_SIZE)
self.reader_thread = None
self.writer_thread = None
self.queue = Queue(DEFAULT_MAX_QUEUE_SIZE)
self.max_pending = MAX_PENDING
self.closing = False
def build_socket(self, family=socket.AF_INET):
return socket.socket(family)
def connect(self):
super(AsyncClient, self).connect()
self.closing = False
self.reader_thread = eventlet.greenthread.spawn(self._reader_run)
self.writer_thread = eventlet.greenthread.spawn(self._writer_run)
def dispatch(self, fn, *args, **kwargs):
if LOG.isEnabledFor(logging.DEBUG):
LOG.debug("Dispatching: Pending {0}".format(len(self._pending)))
self.pool.spawn_n(fn, *args, **kwargs)
def shutdown(self):
self.closing = True
if len(self._pending) + self.queue.qsize() == 0:
self._end_close()
def close(self):
self.shutdown()
self.wait()
def _end_close(self):
self.writer_thread.kill()
self.reader_thread.kill()
super(AsyncClient, self).close()
self.writer_thread = None
self.reader_thread = None
def sendAsync(self, header, value, onSuccess, onError):
if self.closing:
raise common.ConnectionClosed(
"Client is closing, can't queue more operations.")
if self.faulted:
self._raise(
common.ConnectionFaulted(
"Can't send message when connection is on a faulted state."
), onError)
return #skip the rest
# fail fast on NotConnected
if not self.isConnected:
self._raise(common.NotConnected("Not connected."), onError)
return #skip the rest
if LOG.isEnabledFor(logging.DEBUG):
LOG.debug("Queue: {0}".format(self.queue.qsize()))
self.queue.put((header, value, onSuccess, onError))
eventlet.sleep(0)
def wait(self):
self.queue.join()
def send(self, header, value):
done = eventlet.event.Event()
class Dummy:
pass
d = Dummy()
d.error = None
d.result = None
def innerSuccess(m, r, value):
d.result = (m, r, value)
done.send()
def innerError(e):
d.error = e
done.send()
self.sendAsync(header, value, innerSuccess, innerError)
done.wait() # TODO(Nacho): should be add a default timeout?
if d.error: raise d.error
return d.result
def _writer_run(self):
while self.isConnected and not self.faulted:
try:
while len(self._pending) > self.max_pending:
eventlet.sleep(0)
(header, value, onSuccess, onError) = self.queue.get()
super(AsyncClient, self).sendAsync(header, value, onSuccess,
onError)
except common.ConnectionFaulted:
pass
except common.ConnectionClosed:
pass
except Exception as ex:
self._fault_client(ex)
# Yield execution, don't starve the reader
eventlet.sleep(0)
def _reader_run(self):
while self.isConnected and not self.faulted:
try:
self._async_recv()
self.queue.task_done()
if self.closing and len(
self._pending) + self.queue.qsize() == 0:
self._end_close()
except common.ConnectionFaulted:
pass
except Exception as ex:
self._fault_client(ex)
class Client(baseasync.BaseAsync):
def __init__(self, *args, **kwargs):
super(Client, self).__init__(*args, **kwargs)
self.pool = eventlet.greenpool.GreenPool(DEFAULT_POOL_SIZE)
self.reader_thread = None
self.writer_thread = None
self.queue = Queue(DEFAULT_MAX_QUEUE_SIZE)
self.max_pending = MAX_PENDING
self.closing = False
def build_socket(self, family=socket.AF_INET):
return socket.socket(family)
def wrap_secure_socket(self, s, ssl_version):
return GreenSSLSocket(s, ssl_version=ssl_version)
def connect(self):
super(Client, self).connect()
self.closing = False
self.reader_thread = eventlet.greenthread.spawn(self._reader_run)
self.writer_thread = eventlet.greenthread.spawn(self._writer_run)
def dispatch(self, fn, *args, **kwargs):
if LOG.isEnabledFor(logging.DEBUG):
LOG.debug("Dispatching: Pending {0}".format(len(self._pending)))
self.pool.spawn_n(fn, *args, **kwargs)
def shutdown(self):
self.closing = True
if len(self._pending) + self.queue.qsize() == 0:
self._end_close()
def close(self):
self.shutdown()
self.wait()
def _end_close(self):
self.writer_thread.kill()
self.reader_thread.kill()
super(Client, self).close()
self.writer_thread = None
self.reader_thread = None
def sendAsync(self, header, value, onSuccess, onError, no_ack=False):
if self.closing:
raise common.ConnectionClosed("Client is closing, can't queue more operations.")
if self.faulted:
self._raise(common.ConnectionFaulted("Can't send message when connection is on a faulted state."), onError)
return # skip the rest
# fail fast on NotConnected
if not self.isConnected:
self._raise(common.NotConnected("Not connected."), onError)
return # skip the rest
if LOG.isEnabledFor(logging.DEBUG):
LOG.debug("Queue: {0}".format(self.queue.qsize()))
self.queue.put((header, value, onSuccess, onError, no_ack))
eventlet.sleep(0)
def wait(self):
self.queue.join()
def send(self, header, value):
done = eventlet.event.Event()
class Dummy:
pass
d = Dummy()
d.error = None
d.result = None
def innerSuccess(m, r, value):
d.result = (m, r, value)
done.send()
def innerError(e):
d.error = e
done.send()
self.sendAsync(header, value, innerSuccess, innerError)
done.wait() # TODO(Nacho): should be add a default timeout?
if d.error:
raise d.error
return d.result
def _writer_run(self):
while self.isConnected and not self.faulted:
try:
while len(self._pending) > self.max_pending:
eventlet.sleep(0)
(header, value, onSuccess, onError, no_ack) = self.queue.get()
super(Client, self).sendAsync(header, value, onSuccess, onError, no_ack)
except common.ConnectionFaulted:
pass
except common.ConnectionClosed:
pass
except Exception as ex:
self._fault_client(ex)
# Yield execution, don't starve the reader
eventlet.sleep(0)
def _reader_run(self):
while self.isConnected and not self.faulted:
try:
self._async_recv()
self.queue.task_done()
if self.closing and len(self._pending) + self.queue.qsize() == 0:
self._end_close()
except common.ConnectionFaulted:
pass
except Exception as ex:
self._fault_client(ex)
class StatsdLog(object):
def __init__(self, conf):
TRUE_VALUES = set(("true", "1", "yes", "on", "t", "y"))
self.conf = conf
self.logger = logging.getLogger("statsdlogd")
self.logger.setLevel(logging.INFO)
self.syslog = SysLogHandler(address="/dev/log")
self.formatter = logging.Formatter("%(name)s: %(message)s")
self.syslog.setFormatter(self.formatter)
self.logger.addHandler(self.syslog)
if conf.get("debug", False) in TRUE_VALUES:
self.debug = True
else:
self.debug = False
self.statsd_host = conf.get("statsd_host", "127.0.0.1")
self.statsd_port = int(conf.get("statsd_port", "8125"))
self.listen_addr = conf.get("listen_addr", "127.0.0.1")
self.listen_port = int(conf.get("listen_port", 8126))
if conf.get("report_internal_stats", False) in TRUE_VALUES:
self.report_internal_stats = True
else:
self.report_internal_stats = False
self.int_stats_interval = int(conf.get("internal_stats_interval", 5))
self.buff = int(conf.get("buffer_size", 8192))
self.max_q_size = int(conf.get("max_line_backlog", 512))
self.statsd_sample_rate = float(conf.get("statsd_sample_rate", ".5"))
self.counter = 0
self.skip_counter = 0
self.hits = 0
self.q = Queue(maxsize=self.max_q_size)
# key: regex
self.patterns_file = conf.get("patterns_file", "patterns.json")
try:
with open(self.patterns_file) as pfile:
self.patterns = json.loads(pfile.read())
except Exception as err:
self.logger.critical(err)
print err
sys.exit(1)
self.statsd_addr = (self.statsd_host, self.statsd_port)
self.comp_patterns = {}
for item in self.patterns:
self.comp_patterns[item] = re.compile(self.patterns[item])
def check_line(self, line):
"""
Check if a line matches our search patterns.
:param line: The string to check
:returns: None or regex entry that matched
"""
for entry in self.comp_patterns:
if self.comp_patterns[entry].match(line):
return entry
return None
def internal_stats(self):
"""
Periodically send our own stats to statsd.
"""
lastcount = 0
lasthit = 0
while True:
eventlet.sleep(self.int_stats_interval)
self.send_event("statsdlog.lines:%s|c" % (self.counter - lastcount))
lastcount = self.counter
self.send_event("statsdlog.hits:%s|c" % (self.hits - lasthit))
lasthit = self.hits
def stats_print(self):
"""
Periodically dump some stats to the logs.
"""
lastcount = 0
lasthit = 0
while True:
eventlet.sleep(2)
lps = (self.counter - lastcount) / 60
hps = (self.hits - lasthit) / 60
lastcount = self.counter
lasthit = self.hits
self.logger.info("per second: %d lines - hits %d" % (lps, hps))
self.logger.info("totals: %d hits - %d lines" % (self.hits, self.counter))
if self.skip_counter is not 0:
self.logger.info("Had to skip %d log lines so far" % self.skip_counter)
def send_event(self, payload):
"""
Fire event to statsd
:param payload: The payload of the udp packet to send.
"""
try:
udp_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
udp_socket.sendto(payload, self.statsd_addr)
except Exception:
# udp sendto failed (socket already in use?), but thats ok
self.logger.error("Error trying to send statsd event")
def statsd_counter_increment(self, stats, delta=1):
"""
Increment multiple statsd stats counters
:param stats: list of stats items to package and send
:param delta: delta of stats items
"""
if self.statsd_sample_rate < 1:
if random() <= self.statsd_sample_rate:
for item in stats:
payload = "%s:%s|c|@%s" % (item, delta, self.statsd_sample_rate)
self.send_event(payload)
else:
for item in stats:
payload = "%s:%s|c" % (item, delta)
self.send_event(payload)
def worker(self):
"""
Check for and process log lines in queue
"""
while True:
msg = self.q.get()
matched = self.check_line(msg)
if matched:
self.statsd_counter_increment([matched])
if self.hits >= maxint:
self.logger.info("hit maxint, reset hits counter")
self.hits = 0
self.hits += 1
else:
pass
def listener(self):
"""
syslog udp listener
"""
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
bind_addr = (self.listen_addr, self.listen_port)
sock.bind(bind_addr)
self.logger.info("listening on %s:%d" % bind_addr)
while 1:
data, addr = sock.recvfrom(self.buff)
if not data:
break
else:
if self.q.qsize() < self.max_q_size:
self.q.put(data)
if self.counter >= maxint:
self.logger.info("hit maxint, reset seen counter")
self.counter = 0
self.counter += 1
else:
if self.debug:
self.logger.notice("max log lines in queue, skipping")
if self.skip_counter >= maxint:
self.logger.info("hit maxint, reset skip counter")
self.skip_counter = 0
self.skip_counter += 1
def start(self):
"""
Start the listener, worker, and mgmt server.
"""
eventlet.spawn_n(self.worker)
if self.debug:
eventlet.spawn_n(self.stats_print)
if self.report_internal_stats:
eventlet.spawn_n(self.internal_stats)
while True:
try:
self.listener()
except Exception as err:
self.logger.error(err)
class StatsdLog(object):
"""Simple server to monitor a syslog udp stream for statsd events"""
def __init__(self, conf):
TRUE_VALUES = set(('true', '1', 'yes', 'on', 't', 'y'))
self.conf = conf
self.logger = logging.getLogger('statsdlogd')
self.logger.setLevel(logging.INFO)
self.syslog = SysLogHandler(address='/dev/log')
self.formatter = logging.Formatter('%(name)s: %(message)s')
self.syslog.setFormatter(self.formatter)
self.logger.addHandler(self.syslog)
self.debug = conf.get('debug', 'false').lower() in TRUE_VALUES
self.statsd_host = conf.get('statsd_host', '127.0.0.1')
self.statsd_port = int(conf.get('statsd_port', '8125'))
self.listen_addr = conf.get('listen_addr', '127.0.0.1')
self.listen_port = int(conf.get('listen_port', 8126))
self.report_internal_stats = conf.get('report_internal_stats',
'true').lower() in TRUE_VALUES
self.int_stats_interval = int(conf.get('internal_stats_interval', 5))
self.buff = int(conf.get('buffer_size', 8192))
self.max_q_size = int(conf.get('max_line_backlog', 512))
self.statsd_sample_rate = float(conf.get('statsd_sample_rate', '.5'))
self.counter = 0
self.skip_counter = 0
self.hits = 0
self.q = Queue(maxsize=self.max_q_size)
# key: regex
self.patterns_file = conf.get('patterns_file',
'/etc/statsdlog/patterns.json')
self.json_patterns = conf.get('json_pattern_file',
'true').lower() in TRUE_VALUES
try:
self.patterns = self.load_patterns()
except Exception as err:
self.logger.exception(err)
print "Encountered exception at startup: %s" % err
sys.exit(1)
self.statsd_addr = (self.statsd_host, self.statsd_port)
self.comp_patterns = {}
for item in self.patterns:
self.comp_patterns[item] = re.compile(self.patterns[item])
def load_patterns(self):
if self.json_patterns:
self.logger.info("Using json based patterns file: %s" %
self.patterns_file)
with open(self.patterns_file) as pfile:
return json.loads(pfile.read())
else:
self.logger.info("Using plain text patterns file: %s" %
self.patterns_file)
patterns = {}
with open(self.patterns_file) as f:
for line in f:
if line:
pattern = [x.strip() for x in line.split("=", 1)]
else:
pattern = None
if len(pattern) != 2:
# skip this line
self.logger.error(
"Skipping pattern. Unable to parse: %s" % line)
else:
if pattern[0] and pattern[1]:
patterns[pattern[0]] = pattern[1]
else:
self.logger.error(
"Skipping pattern. Unable to parse: %s" % line)
return patterns
def check_line(self, line):
"""
Check if a line matches our search patterns.
:param line: The string to check
:returns: List of regex entries that matched (or empty list if none)
"""
matches = []
for entry in self.comp_patterns:
if self.comp_patterns[entry].match(line):
matches.append(entry)
return matches
def internal_stats(self):
"""
Periodically send our own stats to statsd.
"""
lastcount = 0
lasthit = 0
while True:
eventlet.sleep(self.int_stats_interval)
self.send_event("statsdlog.lines:%s|c" %
(self.counter - lastcount))
lastcount = self.counter
self.send_event("statsdlog.hits:%s|c" % (self.hits - lasthit))
lasthit = self.hits
def stats_print(self):
"""
Periodically dump some stats to the logs.
"""
lastcount = 0
lasthit = 0
while True:
eventlet.sleep(2)
lps = (self.counter - lastcount) / 60
hps = (self.hits - lasthit) / 60
lastcount = self.counter
lasthit = self.hits
self.logger.info('per second: %d lines - hits %d' % (lps, hps))
self.logger.info('totals: %d hits - %d lines' %
(self.hits, self.counter))
if self.skip_counter is not 0:
self.logger.info('Had to skip %d log lines so far' %
self.skip_counter)
def send_event(self, payload):
"""
Fire event to statsd
:param payload: The payload of the udp packet to send.
"""
try:
udp_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
udp_socket.sendto(payload, self.statsd_addr)
except Exception:
# udp sendto failed (socket already in use?), but thats ok
self.logger.error("Error trying to send statsd event")
def statsd_counter_increment(self, stats, delta=1):
"""
Increment multiple statsd stats counters
:param stats: list of stats items to package and send
:param delta: delta of stats items
"""
if self.statsd_sample_rate < 1:
if random() <= self.statsd_sample_rate:
for item in stats:
payload = "%s:%s|c|@%s" % (item, delta,
self.statsd_sample_rate)
self.send_event(payload)
else:
for item in stats:
payload = "%s:%s|c" % (item, delta)
self.send_event(payload)
def worker(self):
"""
Check for and process log lines in queue
"""
while True:
msg = self.q.get()
matches = self.check_line(msg)
for match in matches:
self.statsd_counter_increment([match])
if self.hits >= maxint:
self.logger.info("hit maxint, reset hits counter")
self.hits = 0
self.hits += 1
def listener(self):
"""
syslog udp listener
"""
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
bind_addr = (self.listen_addr, self.listen_port)
sock.bind(bind_addr)
self.logger.info("listening on %s:%d" % bind_addr)
while 1:
data, addr = sock.recvfrom(self.buff)
if not data:
break
else:
if self.q.qsize() < self.max_q_size:
self.q.put(data)
if self.counter >= maxint:
self.logger.info("hit maxint, reset seen counter")
self.counter = 0
self.counter += 1
else:
if self.debug:
self.logger.notice("max log lines in queue, skipping")
if self.skip_counter >= maxint:
self.logger.info("hit maxint, reset skip counter")
self.skip_counter = 0
self.skip_counter += 1
def start(self):
"""
Start the listener, worker, and mgmt server.
"""
eventlet.spawn_n(self.worker)
if self.debug:
eventlet.spawn_n(self.stats_print)
if self.report_internal_stats:
eventlet.spawn_n(self.internal_stats)
while True:
try:
self.listener()
except Exception as err:
self.logger.error(err)