def main(): """ Main entry point for Grok Custom Metric forwarder """ parser = OptionParser() parser.add_option("--server", dest="server", help="Grok server") AvogadroAgent.addParserOptions(parser) (options, _args) = parser.parse_args() grok = GrokSession(server=options.server) with grok.connect() as sock: _fetchAndForward(sock, AvogadroCPUTimesAgent, options) _fetchAndForward(sock, AvogadroMemoryAgent, options) _fetchAndForward(sock, AvogadroDiskReadBytesAgent, options) _fetchAndForward(sock, AvogadroDiskWriteBytesAgent, options) _fetchAndForward(sock, AvogadroDiskReadTimeAgent, options) _fetchAndForward(sock, AvogadroDiskWriteTimeAgent, options) _fetchAndForward(sock, AvogadroNetworkBytesSentAgent, options) _fetchAndForward(sock, AvogadroNetworkBytesReceivedAgent, options) _fetchAndForward(sock, AvogadroKeyCountAgent, options) _fetchAndForward(sock, AvogadroKeyDownDownAgent, options) _fetchAndForward(sock, AvogadroKeyUpDownAgent, options) _fetchAndForward(sock, AvogadroKeyHoldAgent, options)
def run(server, apiKey, metricName, resource, numRecords): grok = GrokSession(server=server, apikey=apiKey) inc = 300 currentTimestamp = int(time.time()) - (numRecords * inc) with grok.connect() as sock: for i in xrange(numRecords): value = random.random() sock.sendall("%s %f %d\n" % (metricName, value, currentTimestamp)) currentTimestamp += inc if i % 100 == 99: print ".", sys.stdout.flush() if i == RECORDS_BEFORE_MONITOR: print print "Creating model...", sys.stdout.flush() # Monitor the metric modelSpec = {"metric": metricName, "datasource": "custom"} if resource is not None: modelSpec["resource"] = resource model = grok.createModel(modelSpec) print "done"
def run(server, apiKey, metricName, resource, numRecords): grok = GrokSession(server=server, apikey=apiKey) inc = 300 currentTimestamp = int(time.time()) - (numRecords * inc) with grok.connect() as sock: for i in xrange(numRecords): value = random.random() sock.sendall("%s %f %d\n" % (metricName, value, currentTimestamp)) currentTimestamp += inc if i % 100 == 99: print ".", sys.stdout.flush() if i == RECORDS_BEFORE_MONITOR: print print "Creating model...", sys.stdout.flush() # Monitor the metric modelSpec = {"metric": metricName, "datasource": "custom"} if resource is not None: modelSpec["resource"] = resource model = grok.createModel(modelSpec) print "done"
def main(): """ Main entry point for Grok Custom Metric forwarder """ parser = OptionParser() parser.add_option("--server", dest="server", help="Grok server") AvogadroAgent.addParserOptions(parser) (options, _args) = parser.parse_args() grok = GrokSession(server=options.server) with grok.connect() as sock: _fetchAndForward(sock, AvogadroCPUTimesAgent, options) _fetchAndForward(sock, AvogadroMemoryAgent, options) _fetchAndForward(sock, AvogadroDiskReadBytesAgent, options) _fetchAndForward(sock, AvogadroDiskWriteBytesAgent, options) _fetchAndForward(sock, AvogadroDiskReadTimeAgent, options) _fetchAndForward(sock, AvogadroDiskWriteTimeAgent, options) _fetchAndForward(sock, AvogadroNetworkBytesSentAgent, options) _fetchAndForward(sock, AvogadroNetworkBytesReceivedAgent, options) _fetchAndForward(sock, AvogadroKeyCountAgent, options) _fetchAndForward(sock, AvogadroKeyDownDownAgent, options) _fetchAndForward(sock, AvogadroKeyUpDownAgent, options) _fetchAndForward(sock, AvogadroKeyHoldAgent, options)
of files open by all processes). """ import datetime import subprocess import time from grokcli.api import GrokSession try: from sample_credentials import (GROK_API_KEY, GROK_SERVER, METRIC_NAME) except (SyntaxError, ImportError): print( "\nERROR: You must update Grok credentials in sample_credentials.py " "before you can continue.\n") import sys sys.exit(1) if __name__ == "__main__": # Grok client grok = GrokSession(server=GROK_SERVER, apikey=GROK_API_KEY) # Add custom metric data with grok.connect() as sock: print 'Collecting "Open file descriptors" sample...', count = subprocess.check_output("/usr/sbin/lsof | /usr/bin/wc -l", shell=True).strip() print count print 'Sending sample to Grok Metric named "%s"' % METRIC_NAME ts = time.mktime(datetime.datetime.utcnow().timetuple()) sock.sendall("%s %s %d\n" % (METRIC_NAME, count, ts)) print "Done!"
def sendMetricsToGrok(opt): """Collects data for burnrate metrics, writes it to a csv file and sends it to Grok. Collects the following metrics (toggled with CL flags): - Total hourly burnrate - Regional hourly burnrate (use -b) - Total number running instances - Regional number running instances (use -r) - Total number stopped instances - Regional number stopped instances (use -p) - Total number all instances - Regional number all instances (use -t) """ grok = GrokSession(server=opt.server, apikey=opt.key) regionalData = getDataByRegions() ts = time.mktime(datetime.datetime.utcnow().timetuple()) with open(opt.outputfile, "ab") as csvfile: csvwriter = csv.writer(csvfile) with grok.connect() as sock: # Regional burn rate calculation/send if opt.regionalBurnrates: if opt.verbose: print "Calculating / sending regional hourly burn rates" for region in regionalData.items(): sock.sendall("%s %s %d\n" % (opt.prefix + "." + region[0] + ".burnrate", (region[1]["burnrate"]*int(opt.scale)), ts)) csvwriter.writerow([(opt.prefix + "." + region[0] + ".burnrate"), str(region[1]["burnrate"]), str(ts)]) # Total burn rate calculation/send if opt.verbose: print "Calculating / sending total hourly burn rate" burnrate = sum(region[1]["burnrate"] for region in regionalData.items()) sock.sendall("%s %s %d\n" % (opt.prefix + ".total.burnrate", (burnrate*int(opt.scale)), ts)) csvwriter.writerow([opt.prefix + ".total.burnrate", str(burnrate), str(ts)]) # Regional running instances calculate/send if opt.regionalRunning: if opt.verbose: print "Calculating / sending regional running instances" for region in regionalData.items(): sock.sendall("%s %s %d\n" % (opt.prefix + "." + region[0] + ".runningInstances", (region[1]["numberRunningInstances"] *int(opt.scale)), ts)) csvwriter.writerow([(opt.prefix + "." + region[0] + ".runningInstances"), str(region[1]["numberRunningInstances"]), str(ts)]) # Total running instances calculate/send if opt.verbose: print "Calculating / sending total running instances" numRunning = sum(region[1]["numberRunningInstances"] for region in regionalData.items()) sock.sendall("%s %s %d\n" % (opt.prefix + ".total.runningInstances", (numRunning*int(opt.scale)), ts)) csvwriter.writerow([opt.prefix + ".total.runningInstances", str(numRunning), str(ts)]) # Regional stopped instances calculate/send if opt.regionalStopped: if opt.verbose: print "Calculating / sending regional stopped instances" for region in regionalData.items(): sock.sendall("%s %s %d\n" % (opt.prefix + "." + region[0] + ".stoppedInstances", (region[1]["numberStoppedInstances"] *int(opt.scale)), ts)) csvwriter.writerow([(opt.prefix + "." + region[0] + ".stoppedInstances"), str(region[1]["numberStoppedInstances"]), str(ts)]) # Total stopped instances calculate/send if opt.verbose: print "Calculating / sending total stopped instances" numStopped = sum(region[1]["numberStoppedInstances"] for region in regionalData.items()) sock.sendall("%s %s %d\n" % (opt.prefix + ".total.stoppedInstances", (numStopped*int(opt.scale)), ts)) csvwriter.writerow([opt.prefix + ".total.stoppedInstances", str(numStopped), str(ts)]) # Regional all instances calculate/send if opt.regionalAll: if opt.verbose: print "Calculating / sending regional all instances" for region in regionalData.items(): sock.sendall("%s %s %d\n" % (opt.prefix + "." + region[0] + ".allInstances", (region[1]["numberAllInstances"] *int(opt.scale)), ts)) csvwriter.writerow([(opt.prefix + "." + region[0] + ".AllInstances"), str(region[1]["numberAllInstances"]), str(ts)]) # Total all instances calculate/send if opt.verbose: print "Calculating / sending total all instances" numAll = sum(region[1]["numberAllInstances"] for region in regionalData.items()) sock.sendall("%s %s %d\n" % (opt.prefix + ".total.allInstances", (numAll*int(opt.scale)), ts)) csvwriter.writerow([opt.prefix + ".total.allInstances", str(numAll), str(ts)]) if opt.verbose: print "Done!"
opt, arg = parser.parse_args(sys.argv[1:]) path = os.path.dirname(os.path.abspath(__file__)) if ((opt.server=="" or opt.key=="") and not opt.noserver): print ("burnrate_collect_data.py -s <server> -k <key>") sys.exit(2) if opt.inputfile != "" and not opt.noserver: if opt.verbose: print "Sending existing data to grok..." with open(opt.inputfile, "rb") as inputFile: grok = GrokSession(server=opt.server, apikey=opt.key) with grok.connect() as sock: csvreader = csv.reader(inputFile) for row in csvreader: metricName = row[0] data = float(row[1]) ts = row[2] sock.sendall("%s %s %s\n" % (metricName, (data*int(opt.scale)), int(float(ts)))) else: if not os.path.isfile(opt.outputfile): open(opt.outputfile, "w").close() if not opt.noserver: sendMetricsToGrok(opt) else: writeMetricsToFile(opt)
def sendMetricsToHTMIT(opt): """Collects data for burnrate metrics, writes it to a csv file and sends it to HTMIT Collects the following metrics (toggled with CL flags): - Total hourly burnrate - Regional hourly burnrate (use -b) - Total number running instances - Regional number running instances (use -r) - Total number stopped instances - Regional number stopped instances (use -p) - Total number all instances - Regional number all instances (use -t) """ session = GrokSession(server=opt.server, apikey=opt.key) regionalData = getDataByRegions() ts = time.mktime(datetime.datetime.utcnow().timetuple()) with open(opt.outputfile, "ab") as csvfile: csvwriter = csv.writer(csvfile) with session.connect() as sock: # Regional burn rate calculation/send if opt.regionalBurnrates: if opt.verbose: print "Calculating / sending regional hourly burn rates" for region in regionalData.items(): sock.sendall( "%s %s %d\n" % (opt.prefix + "." + region[0] + ".burnrate", (region[1]["burnrate"] * int(opt.scale)), ts)) csvwriter.writerow([ (opt.prefix + "." + region[0] + ".burnrate"), str(region[1]["burnrate"]), str(ts) ]) # Total burn rate calculation/send if opt.verbose: print "Calculating / sending total hourly burn rate" burnrate = sum(region[1]["burnrate"] for region in regionalData.items()) sock.sendall("%s %s %d\n" % (opt.prefix + ".total.burnrate", (burnrate * int(opt.scale)), ts)) csvwriter.writerow( [opt.prefix + ".total.burnrate", str(burnrate), str(ts)]) # Regional running instances calculate/send if opt.regionalRunning: if opt.verbose: print "Calculating / sending regional running instances" for region in regionalData.items(): sock.sendall( "%s %s %d\n" % (opt.prefix + "." + region[0] + ".runningInstances", (region[1]["numberRunningInstances"] * int(opt.scale)), ts)) csvwriter.writerow([ (opt.prefix + "." + region[0] + ".runningInstances"), str(region[1]["numberRunningInstances"]), str(ts) ]) # Total running instances calculate/send if opt.verbose: print "Calculating / sending total running instances" numRunning = sum(region[1]["numberRunningInstances"] for region in regionalData.items()) sock.sendall("%s %s %d\n" % (opt.prefix + ".total.runningInstances", (numRunning * int(opt.scale)), ts)) csvwriter.writerow([ opt.prefix + ".total.runningInstances", str(numRunning), str(ts) ]) # Regional stopped instances calculate/send if opt.regionalStopped: if opt.verbose: print "Calculating / sending regional stopped instances" for region in regionalData.items(): sock.sendall( "%s %s %d\n" % (opt.prefix + "." + region[0] + ".stoppedInstances", (region[1]["numberStoppedInstances"] * int(opt.scale)), ts)) csvwriter.writerow([ (opt.prefix + "." + region[0] + ".stoppedInstances"), str(region[1]["numberStoppedInstances"]), str(ts) ]) # Total stopped instances calculate/send if opt.verbose: print "Calculating / sending total stopped instances" numStopped = sum(region[1]["numberStoppedInstances"] for region in regionalData.items()) sock.sendall("%s %s %d\n" % (opt.prefix + ".total.stoppedInstances", (numStopped * int(opt.scale)), ts)) csvwriter.writerow([ opt.prefix + ".total.stoppedInstances", str(numStopped), str(ts) ]) # Regional all instances calculate/send if opt.regionalAll: if opt.verbose: print "Calculating / sending regional all instances" for region in regionalData.items(): sock.sendall( "%s %s %d\n" % (opt.prefix + "." + region[0] + ".allInstances", (region[1]["numberAllInstances"] * int(opt.scale)), ts)) csvwriter.writerow([ (opt.prefix + "." + region[0] + ".AllInstances"), str(region[1]["numberAllInstances"]), str(ts) ]) # Total all instances calculate/send if opt.verbose: print "Calculating / sending total all instances" numAll = sum(region[1]["numberAllInstances"] for region in regionalData.items()) sock.sendall("%s %s %d\n" % (opt.prefix + ".total.allInstances", (numAll * int(opt.scale)), ts)) csvwriter.writerow( [opt.prefix + ".total.allInstances", str(numAll), str(ts)]) if opt.verbose: print "Done!"
default=1) opt, arg = parser.parse_args(sys.argv[1:]) path = os.path.dirname(os.path.abspath(__file__)) if ((opt.server == "" or opt.key == "") and not opt.noserver): print("burnrate_collect_data.py -s <server> -k <key>") sys.exit(2) if opt.inputfile != "" and not opt.noserver: if opt.verbose: print "Sending existing data to HTM-IT..." with open(opt.inputfile, "rb") as inputFile: session = GrokSession(server=opt.server, apikey=opt.key) with session.connect() as sock: csvreader = csv.reader(inputFile) for row in csvreader: metricName = row[0] data = float(row[1]) ts = row[2] sock.sendall("%s %s %s\n" % (metricName, (data * int(opt.scale)), int(float(ts)))) else: if not os.path.isfile(opt.outputfile): open(opt.outputfile, "w").close() if not opt.noserver: sendMetricsToHTMIT(opt) else:
opt, arg = parser.parse_args(sys.argv[1:]) path = os.path.dirname(os.path.abspath(__file__)) if ((opt.server=="" or opt.key=="") and not opt.noserver): print ("burnrate_collect_data.py -s <server> -k <key>") sys.exit(2) if opt.inputfile != "" and not opt.noserver: if opt.verbose: print "Sending existing data to HTM-IT..." with open(opt.inputfile, "rb") as inputFile: session = GrokSession(server=opt.server, apikey=opt.key) with session.connect() as sock: csvreader = csv.reader(inputFile) for row in csvreader: metricName = row[0] data = float(row[1]) ts = row[2] sock.sendall("%s %s %s\n" % (metricName, (data*int(opt.scale)), int(float(ts)))) else: if not os.path.isfile(opt.outputfile): open(opt.outputfile, "w").close() if not opt.noserver: sendMetricsToHTMIT(opt) else: writeMetricsToFile(opt)