def main(): global log_tracebacks # can be updated import argparse parser = argparse.ArgumentParser(description="Collect metrics from amqp and dispatch them to carbon daemon.") parser.add_argument( "-c", "--config", action="append", default=list(), help="Additional configuration files to read. Can be specified" " multiple times, values from later ones override values in the former.", ) parser.add_argument( "--delete-queue", nargs="?", default=False, help="Delete queue before re-declaring it," ' useful to change bindings. Accepts "if-empty" argument,' " overrides net.amqp.queue.delete_first configuration parameter.", ) parser.add_argument("-n", "--dry-run", action="store_true", help="Do not actually send data.") parser.add_argument("--dump", action="store_true", help="Dump polled data to stdout.") parser.add_argument("--debug", action="store_true", help="Verbose operation mode.") optz = parser.parse_args() cfg = AttrDict.from_yaml("{}.yaml".format(os.path.splitext(os.path.realpath(__file__))[0]), if_exists=True) for k in optz.config: cfg.update_yaml(k) configure_logging(cfg.logging, logging.DEBUG if optz.debug else logging.WARNING) logging.captureWarnings(cfg.logging.warnings) cfg.net.amqp.queue.delete_first = optz.delete_queue if optz.delete_queue is not None else True optz.dump = optz.dump or cfg.debug.dump_data optz.dry_run = optz.dry_run or cfg.debug.dry_run log_tracebacks = cfg.logging.tracebacks dst = cfg.net.carbon.host if isinstance(dst, types.StringTypes): dst = dst.rsplit(":", 1) dst = dst[0], int(dst[1]) if len(dst) > 1 else cfg.net.carbon.default_port dump = ( (lambda metric, val, ts: print("{} {} {}".format(metric, val, ts))) if optz.dump else lambda metric, val, ts: None ) if not optz.dry_run: carbon = CarbonClient(dst) dst = lambda metric, ts, val, val_raw: (dump(metric, val, ts), carbon.send(metric, val, ts)) else: dst = lambda metric, ts, val, val_raw: dump(metric, val, ts) amqp = AMQPHarvester( host=cfg.net.amqp.host, auth=(cfg.net.amqp.user, cfg.net.amqp.password), exchange=cfg.net.amqp.exchange, queue=cfg.net.amqp.queue, heartbeat=cfg.net.amqp.heartbeat, log=logging.getLogger("amqp_carbon.amqp_link"), callback=dst, exclusive=cfg.net.amqp.consume.exclusive, ack_batch=cfg.net.amqp.consume.ack_batch, ) amqp.harvest()
def main(): global graphite_min_cycle # can be updated import argparse parser = argparse.ArgumentParser( description='Collect various metrics from gmond and dispatch' ' them graphite-style at regular intervals to amqp (so they can be routed to carbon).') parser.add_argument('-c', '--config', action='append', default=list(), help='Additional configuration files to read. Can be specified' ' multiple times, values from later ones override values in the former.') parser.add_argument('-n', '--dry-run', action='store_true', help='Do not actually send data.') parser.add_argument('--dump', action='store_true', help='Dump polled data to stdout.') parser.add_argument('--debug', action='store_true', help='Verbose operation mode.') optz = parser.parse_args() cfg = AttrDict.from_yaml('{}.yaml'.format( os.path.splitext(os.path.realpath(__file__))[0] ), if_exists=True) for k in optz.config: cfg.update_yaml(k) configure_logging( cfg.logging, logging.DEBUG if optz.debug else logging.WARNING ) logging.captureWarnings(cfg.logging.warnings) optz.dump = optz.dump or cfg.debug.dump_data optz.dry_run = optz.dry_run or cfg.debug.dry_run graphite_min_cycle = cfg.metrics.interval mangler = DataMangler( name_template=cfg.metrics.name.full, name_rewrite=cfg.metrics.name.rewrite, name_aliases=cfg.metrics.name.aliases ) log = logging.getLogger('gmond_amqp.amqp_link') if not cfg.logging.tracebacks: log.exception = log.error amqp = AMQPPublisher( host=cfg.net.amqp.host, auth=(cfg.net.amqp.user, cfg.net.amqp.password), exchange=cfg.net.amqp.exchange, heartbeat=cfg.net.amqp.heartbeat, log=log, libc_gethostbyname=gethostbyname\ if not cfg.net.bypass_libc_gethostbyname else False ) log = logging.getLogger('gmond_amqp.main_loop') ts, data = time(), list() self_profiling = cfg.metrics.self_profiling and '{}.gmond_amqp'.format( socket.gethostname() if cfg.net.bypass_libc_gethostbyname else gethostname() ) while True: ts_now = time() xml = gmond_poll( cfg.net.gmond.hosts, libc_gethostbyname=gethostbyname\ if not cfg.net.bypass_libc_gethostbyname else False, default_port=cfg.net.gmond.default_port ) if self_profiling: ts_new, ts_prof = time(), ts_now val, ts_prof = ts_new - ts_prof, ts_new data.append(('{}.poll'.format(self_profiling), ts_now, val, val)) xml = gmond_xml_process( xml, validate=cfg.net.gmond.validate_xml, validate_strict=cfg.net.gmond.validate_strict ) if self_profiling: ts_new = time() val, ts_prof = ts_new - ts_prof, ts_new data.append(('{}.process'.format(self_profiling), ts_now, val, val)) data.extend(it.chain.from_iterable(it.starmap( ft.partial(mangler.process_cluster, ts=ts_now), xml ))) log.debug('Publishing {} datapoints'.format(len(data))) if optz.dump: pprint(data) if not optz.dry_run: amqp.publish(data) if self_profiling: ts_new = time() val, ts_prof = ts_new - ts_prof, ts_new data = [('{}.publish'.format(self_profiling), ts_now, val, val)] while ts <= ts_now: ts += cfg.metrics.interval ts_sleep = max(0, ts - time()) log.debug('Sleep: {}s'.format(ts_sleep)) sleep(ts_sleep)