def __init__( self, my_host, my_port, carbon_host, carbon_port, flush_interval, percent_thresholds, *, keys=None, connection_timeout=10, top_namespace="stats", auth_header_name=None ): self.my_host = my_host self.my_port = my_port self.carbon_host = carbon_host self.carbon_port = carbon_port self.flush_interval = flush_interval self.metrics = {} self.percent_thresholds = percent_thresholds self.connection_timeout = connection_timeout self.ns_prefix = top_namespace self.keys = {} self.auth_header_name = auth_header_name if keys: # auth header is required if keys are present so check that if not self.auth_header_name: raise Exception( "auth_header_name cannot be None with keys \ provided" ) self.auth_header_name = self.auth_header_name.lower() for key, regex in keys: self.keys[key] = re.compile(regex) self.app = Application([(r"^/v1/stat/?$", "POST", self.v1_stat)])
response.send(body) # coroutine handler receiving arguments from re patterns in the route path @coroutine def groups(g1, g2): return 'groups! {0} {1}'.format(g1, g2) if __name__ == '__main__': logging.basicConfig(stream=sys.stdout, level=logging.ERROR, format='%(asctime)s | %(levelname)s | %(message)s') p = argparse.ArgumentParser() p.add_argument('-l', '--log-level', default='ERROR') args = p.parse_args() logging.getLogger().setLevel(getattr(logging, args.log_level)) a = Application([ (r'^/echo$', 'POST', echo), (r'^/slow/(?P<t>.*)', 'GET', slow), (r'^/groups/(?P<g1>.*)/(?P<g2>.*)$', 'GET', groups), (r'^/site/(?P<path>.*)$', 'GET', Application.static, {'doc_root':'/tmp/site'}), (r'^/hi3', 'GET', hi3), (r'^/.*$', 'GET', hi), ]) a.serve('0.0.0.0', 2020, keep_alive=True)
# coroutine handler receiving arguments from re patterns in the route path @coroutine def groups(g1, g2): return 'groups! {0} {1}'.format(g1, g2) if __name__ == '__main__': logging.basicConfig(stream=sys.stdout, level=logging.ERROR, format='%(asctime)s | %(levelname)s | %(message)s') p = argparse.ArgumentParser() p.add_argument('-l', '--log-level', default='ERROR') args = p.parse_args() logging.getLogger().setLevel(getattr(logging, args.log_level)) a = Application([ (r'^/echo$', 'POST', echo), (r'^/slow/(?P<t>.*)', 'GET', slow), (r'^/groups/(?P<g1>.*)/(?P<g2>.*)$', 'GET', groups), (r'^/site/(?P<path>.*)$', 'GET', Application.static, { 'doc_root': '/tmp/site' }), (r'^/hi3', 'GET', hi3), (r'^/.*$', 'GET', hi), ]) a.serve('0.0.0.0', 2020, keep_alive=True)
class Agg: def __init__( self, my_host, my_port, carbon_host, carbon_port, flush_interval, percent_thresholds, *, keys=None, connection_timeout=10, top_namespace="stats", auth_header_name=None ): self.my_host = my_host self.my_port = my_port self.carbon_host = carbon_host self.carbon_port = carbon_port self.flush_interval = flush_interval self.metrics = {} self.percent_thresholds = percent_thresholds self.connection_timeout = connection_timeout self.ns_prefix = top_namespace self.keys = {} self.auth_header_name = auth_header_name if keys: # auth header is required if keys are present so check that if not self.auth_header_name: raise Exception( "auth_header_name cannot be None with keys \ provided" ) self.auth_header_name = self.auth_header_name.lower() for key, regex in keys: self.keys[key] = re.compile(regex) self.app = Application([(r"^/v1/stat/?$", "POST", self.v1_stat)]) def serve(self): loop = get_event_loop() try: log.info("starting event loop") log.info("listening on {0}:{1}".format(self.my_host, self.my_port)) log.info("sending data to carbon-cache at {0}:{1}".format(self.carbon_host, self.carbon_port)) log.info("flush interval to carbon is {0}s".format(self.flush_interval)) log.info("calculating {}th percentile(s)".format(",".join(map(str, self.percent_thresholds)))) log.info("listening on {0}:{1}".format(self.my_host, self.my_port)) loop.call_soon(self.flusher) self.app.serve(self.my_host, self.my_port, keep_alive=False) except KeyboardInterrupt as k: log.info("Keyboard Interrupt. Stopping server.") finally: loop.close() log.info("done serving") @coroutine def v1_stat(self, request, response): body = yield from request.body() metrics = json.loads(body.decode("utf-8")) log.debug("loaded {}".format(metrics)) # fixme, validate json with validictory for metric in metrics: # get namespace ns = metric["name"].split(".")[0] # check auth if we have keys if len(self.keys) > 0: key = request.headers.get(self.auth_header_name, None) if key is None or not key in self.keys or not self.keys[key].match(ns): log.warning("{0} Unauthorized for {1}".format(key, ns)) raise HTTPException(401, "Unauthorized") # got here, then auth is ok self.handle_stat(metric["name"], metric["value"], metric["type"]) def handle_stat(self, metric_name, value, metric_type): if metric_type == "c": if not metric_name in self.metrics: self.metrics[metric_name] = CountMetric(metric_name, self.ns_prefix, self.flush_interval) elif metric_type == "ms": if not metric_name in self.metrics: self.metrics[metric_name] = TimerMetric( metric_name, self.ns_prefix, self.flush_interval, self.percent_thresholds ) self.metrics[metric_name].accumulate(value) def flusher(self): now = time.time() messages = [] for metric in self.metrics.values(): if (now - metric.last_flush_time) > self.flush_interval: messages += metric.flush(now) if len(messages) > 0: log.debug("async send messages") async(self.send_messages(messages)) get_event_loop().call_later(0.25, self.flusher) @coroutine def send_messages(self, messages): log.info("sending message {}".format(messages)) # use pickle protocol = 2, so python2 can read it payload = pickle.dumps(messages, 2) header = struct.pack("!L", len(payload)) message = header + payload try: reader, writer = yield from wait_for( open_connection(self.carbon_host, self.carbon_port), self.connection_timeout ) # FIXME, how to ensure messages are sent writer.write(message) writer.write_eof() except Exception as e: log.warning("Could not connect to carbon {}".format(e)) log.exception(e)