def __init__(
self,
my_host,
my_port,
carbon_host,
carbon_port,
flush_interval,
percent_thresholds,
*,
keys=None,
connection_timeout=10,
top_namespace="stats",
auth_header_name=None
):
self.my_host = my_host
self.my_port = my_port
self.carbon_host = carbon_host
self.carbon_port = carbon_port
self.flush_interval = flush_interval
self.metrics = {}
self.percent_thresholds = percent_thresholds
self.connection_timeout = connection_timeout
self.ns_prefix = top_namespace
self.keys = {}
self.auth_header_name = auth_header_name
if keys:
# auth header is required if keys are present so check that
if not self.auth_header_name:
raise Exception(
"auth_header_name cannot be None with keys \
provided"
)
self.auth_header_name = self.auth_header_name.lower()
for key, regex in keys:
self.keys[key] = re.compile(regex)
self.app = Application([(r"^/v1/stat/?$", "POST", self.v1_stat)])
response.send(body)
# coroutine handler receiving arguments from re patterns in the route path
@coroutine
def groups(g1, g2):
return 'groups! {0} {1}'.format(g1, g2)
if __name__ == '__main__':
logging.basicConfig(stream=sys.stdout, level=logging.ERROR,
format='%(asctime)s | %(levelname)s | %(message)s')
p = argparse.ArgumentParser()
p.add_argument('-l', '--log-level', default='ERROR')
args = p.parse_args()
logging.getLogger().setLevel(getattr(logging, args.log_level))
a = Application([
(r'^/echo$', 'POST', echo),
(r'^/slow/(?P<t>.*)', 'GET', slow),
(r'^/groups/(?P<g1>.*)/(?P<g2>.*)$', 'GET', groups),
(r'^/site/(?P<path>.*)$', 'GET', Application.static,
{'doc_root':'/tmp/site'}),
(r'^/hi3', 'GET', hi3),
(r'^/.*$', 'GET', hi),
])
a.serve('0.0.0.0', 2020, keep_alive=True)
# coroutine handler receiving arguments from re patterns in the route path
@coroutine
def groups(g1, g2):
return 'groups! {0} {1}'.format(g1, g2)
if __name__ == '__main__':
logging.basicConfig(stream=sys.stdout,
level=logging.ERROR,
format='%(asctime)s | %(levelname)s | %(message)s')
p = argparse.ArgumentParser()
p.add_argument('-l', '--log-level', default='ERROR')
args = p.parse_args()
logging.getLogger().setLevel(getattr(logging, args.log_level))
a = Application([
(r'^/echo$', 'POST', echo),
(r'^/slow/(?P<t>.*)', 'GET', slow),
(r'^/groups/(?P<g1>.*)/(?P<g2>.*)$', 'GET', groups),
(r'^/site/(?P<path>.*)$', 'GET', Application.static, {
'doc_root': '/tmp/site'
}),
(r'^/hi3', 'GET', hi3),
(r'^/.*$', 'GET', hi),
])
a.serve('0.0.0.0', 2020, keep_alive=True)
class Agg:
def __init__(
self,
my_host,
my_port,
carbon_host,
carbon_port,
flush_interval,
percent_thresholds,
*,
keys=None,
connection_timeout=10,
top_namespace="stats",
auth_header_name=None
):
self.my_host = my_host
self.my_port = my_port
self.carbon_host = carbon_host
self.carbon_port = carbon_port
self.flush_interval = flush_interval
self.metrics = {}
self.percent_thresholds = percent_thresholds
self.connection_timeout = connection_timeout
self.ns_prefix = top_namespace
self.keys = {}
self.auth_header_name = auth_header_name
if keys:
# auth header is required if keys are present so check that
if not self.auth_header_name:
raise Exception(
"auth_header_name cannot be None with keys \
provided"
)
self.auth_header_name = self.auth_header_name.lower()
for key, regex in keys:
self.keys[key] = re.compile(regex)
self.app = Application([(r"^/v1/stat/?$", "POST", self.v1_stat)])
def serve(self):
loop = get_event_loop()
try:
log.info("starting event loop")
log.info("listening on {0}:{1}".format(self.my_host, self.my_port))
log.info("sending data to carbon-cache at {0}:{1}".format(self.carbon_host, self.carbon_port))
log.info("flush interval to carbon is {0}s".format(self.flush_interval))
log.info("calculating {}th percentile(s)".format(",".join(map(str, self.percent_thresholds))))
log.info("listening on {0}:{1}".format(self.my_host, self.my_port))
loop.call_soon(self.flusher)
self.app.serve(self.my_host, self.my_port, keep_alive=False)
except KeyboardInterrupt as k:
log.info("Keyboard Interrupt. Stopping server.")
finally:
loop.close()
log.info("done serving")
@coroutine
def v1_stat(self, request, response):
body = yield from request.body()
metrics = json.loads(body.decode("utf-8"))
log.debug("loaded {}".format(metrics))
# fixme, validate json with validictory
for metric in metrics:
# get namespace
ns = metric["name"].split(".")[0]
# check auth if we have keys
if len(self.keys) > 0:
key = request.headers.get(self.auth_header_name, None)
if key is None or not key in self.keys or not self.keys[key].match(ns):
log.warning("{0} Unauthorized for {1}".format(key, ns))
raise HTTPException(401, "Unauthorized")
# got here, then auth is ok
self.handle_stat(metric["name"], metric["value"], metric["type"])
def handle_stat(self, metric_name, value, metric_type):
if metric_type == "c":
if not metric_name in self.metrics:
self.metrics[metric_name] = CountMetric(metric_name, self.ns_prefix, self.flush_interval)
elif metric_type == "ms":
if not metric_name in self.metrics:
self.metrics[metric_name] = TimerMetric(
metric_name, self.ns_prefix, self.flush_interval, self.percent_thresholds
)
self.metrics[metric_name].accumulate(value)
def flusher(self):
now = time.time()
messages = []
for metric in self.metrics.values():
if (now - metric.last_flush_time) > self.flush_interval:
messages += metric.flush(now)
if len(messages) > 0:
log.debug("async send messages")
async(self.send_messages(messages))
get_event_loop().call_later(0.25, self.flusher)
@coroutine
def send_messages(self, messages):
log.info("sending message {}".format(messages))
# use pickle protocol = 2, so python2 can read it
payload = pickle.dumps(messages, 2)
header = struct.pack("!L", len(payload))
message = header + payload
try:
reader, writer = yield from wait_for(
open_connection(self.carbon_host, self.carbon_port), self.connection_timeout
)
# FIXME, how to ensure messages are sent
writer.write(message)
writer.write_eof()
except Exception as e:
log.warning("Could not connect to carbon {}".format(e))
log.exception(e)
