def main():
global log_tracebacks # can be updated
import argparse
parser = argparse.ArgumentParser(description="Collect metrics from amqp and dispatch them to carbon daemon.")
parser.add_argument(
"-c",
"--config",
action="append",
default=list(),
help="Additional configuration files to read. Can be specified"
" multiple times, values from later ones override values in the former.",
)
parser.add_argument(
"--delete-queue",
nargs="?",
default=False,
help="Delete queue before re-declaring it,"
' useful to change bindings. Accepts "if-empty" argument,'
" overrides net.amqp.queue.delete_first configuration parameter.",
)
parser.add_argument("-n", "--dry-run", action="store_true", help="Do not actually send data.")
parser.add_argument("--dump", action="store_true", help="Dump polled data to stdout.")
parser.add_argument("--debug", action="store_true", help="Verbose operation mode.")
optz = parser.parse_args()
cfg = AttrDict.from_yaml("{}.yaml".format(os.path.splitext(os.path.realpath(__file__))[0]), if_exists=True)
for k in optz.config:
cfg.update_yaml(k)
configure_logging(cfg.logging, logging.DEBUG if optz.debug else logging.WARNING)
logging.captureWarnings(cfg.logging.warnings)
cfg.net.amqp.queue.delete_first = optz.delete_queue if optz.delete_queue is not None else True
optz.dump = optz.dump or cfg.debug.dump_data
optz.dry_run = optz.dry_run or cfg.debug.dry_run
log_tracebacks = cfg.logging.tracebacks
dst = cfg.net.carbon.host
if isinstance(dst, types.StringTypes):
dst = dst.rsplit(":", 1)
dst = dst[0], int(dst[1]) if len(dst) > 1 else cfg.net.carbon.default_port
dump = (
(lambda metric, val, ts: print("{} {} {}".format(metric, val, ts)))
if optz.dump
else lambda metric, val, ts: None
)
if not optz.dry_run:
carbon = CarbonClient(dst)
dst = lambda metric, ts, val, val_raw: (dump(metric, val, ts), carbon.send(metric, val, ts))
else:
dst = lambda metric, ts, val, val_raw: dump(metric, val, ts)
amqp = AMQPHarvester(
host=cfg.net.amqp.host,
auth=(cfg.net.amqp.user, cfg.net.amqp.password),
exchange=cfg.net.amqp.exchange,
queue=cfg.net.amqp.queue,
heartbeat=cfg.net.amqp.heartbeat,
log=logging.getLogger("amqp_carbon.amqp_link"),
callback=dst,
exclusive=cfg.net.amqp.consume.exclusive,
ack_batch=cfg.net.amqp.consume.ack_batch,
)
amqp.harvest()
def main():
global graphite_min_cycle # can be updated
import argparse
parser = argparse.ArgumentParser(
description='Collect various metrics from gmond and dispatch'
' them graphite-style at regular intervals to amqp (so they can be routed to carbon).')
parser.add_argument('-c', '--config', action='append', default=list(),
help='Additional configuration files to read. Can be specified'
' multiple times, values from later ones override values in the former.')
parser.add_argument('-n', '--dry-run', action='store_true', help='Do not actually send data.')
parser.add_argument('--dump', action='store_true', help='Dump polled data to stdout.')
parser.add_argument('--debug', action='store_true', help='Verbose operation mode.')
optz = parser.parse_args()
cfg = AttrDict.from_yaml('{}.yaml'.format(
os.path.splitext(os.path.realpath(__file__))[0] ), if_exists=True)
for k in optz.config: cfg.update_yaml(k)
configure_logging( cfg.logging,
logging.DEBUG if optz.debug else logging.WARNING )
logging.captureWarnings(cfg.logging.warnings)
optz.dump = optz.dump or cfg.debug.dump_data
optz.dry_run = optz.dry_run or cfg.debug.dry_run
graphite_min_cycle = cfg.metrics.interval
mangler = DataMangler(
name_template=cfg.metrics.name.full,
name_rewrite=cfg.metrics.name.rewrite,
name_aliases=cfg.metrics.name.aliases )
log = logging.getLogger('gmond_amqp.amqp_link')
if not cfg.logging.tracebacks: log.exception = log.error
amqp = AMQPPublisher( host=cfg.net.amqp.host,
auth=(cfg.net.amqp.user, cfg.net.amqp.password),
exchange=cfg.net.amqp.exchange, heartbeat=cfg.net.amqp.heartbeat,
log=log, libc_gethostbyname=gethostbyname\
if not cfg.net.bypass_libc_gethostbyname else False )
log = logging.getLogger('gmond_amqp.main_loop')
ts, data = time(), list()
self_profiling = cfg.metrics.self_profiling and '{}.gmond_amqp'.format(
socket.gethostname() if cfg.net.bypass_libc_gethostbyname else gethostname() )
while True:
ts_now = time()
xml = gmond_poll( cfg.net.gmond.hosts,
libc_gethostbyname=gethostbyname\
if not cfg.net.bypass_libc_gethostbyname else False,
default_port=cfg.net.gmond.default_port )
if self_profiling:
ts_new, ts_prof = time(), ts_now
val, ts_prof = ts_new - ts_prof, ts_new
data.append(('{}.poll'.format(self_profiling), ts_now, val, val))
xml = gmond_xml_process( xml,
validate=cfg.net.gmond.validate_xml,
validate_strict=cfg.net.gmond.validate_strict )
if self_profiling:
ts_new = time()
val, ts_prof = ts_new - ts_prof, ts_new
data.append(('{}.process'.format(self_profiling), ts_now, val, val))
data.extend(it.chain.from_iterable(it.starmap(
ft.partial(mangler.process_cluster, ts=ts_now), xml )))
log.debug('Publishing {} datapoints'.format(len(data)))
if optz.dump: pprint(data)
if not optz.dry_run: amqp.publish(data)
if self_profiling:
ts_new = time()
val, ts_prof = ts_new - ts_prof, ts_new
data = [('{}.publish'.format(self_profiling), ts_now, val, val)]
while ts <= ts_now: ts += cfg.metrics.interval
ts_sleep = max(0, ts - time())
log.debug('Sleep: {}s'.format(ts_sleep))
sleep(ts_sleep)