def send(self,
metric: str,
value,
host: Optional[str] = None,
port: Optional[int] = None,
tags: Optional[Dict[str, str]] = None,
prefix: str = '',
protocol: str = 'tcp'):
"""
Send data to a Graphite instance.
:param metric: Metric name.
:param value: Value to be sent.
:param host: Graphite host (default: default configured ``host``).
:param port: Graphite port (default: default configured ``port``).
:param tags: Map of tags for the metric.
:param prefix: Metric prefix name (default: empty string).
:param protocol: Communication protocol - possible values: 'tcp', 'udp' (default: 'tcp').
"""
import graphyte
host = host or self.host
port = port or self.port
tags = tags or {}
protocol = protocol.lower()
graphyte.init(host, port=port, prefix=prefix, protocol=protocol)
graphyte.send(metric, value, tags=tags)
def send_statistics(self):
graphite_url = self.config['graphite']['url']
prefix = self.config['graphite']['base_prefix'].format(
self.cluster_label)
graphyte.init(graphite_url, prefix=prefix)
metric_path = ''
service_component_info = self.data.get('ServiceComponentInfo')
if service_component_info:
metric_path = '{service}.{component}Metrics'.format(
service=service_component_info.get('service_name'),
component=service_component_info.get('component_name'))
metrics = self.data.get('metrics')
if not metrics:
return
self.recursive_parse_metric(metric_path, metrics)
else:
items = self.data.get('items')
if items:
for item in items:
host_info = item.get('HostRoles')
host_name = host_info.get('host_name') \
.split('.')[0].split('-')[0]
metric_path = '{host_name}.{component}Metrics'.format(
host_name=host_name,
component=host_info.get('component_name'))
metrics = item.get('metrics')
if not metrics:
return
self.recursive_parse_metric(metric_path, metrics)
else:
raise NotImplementedError('Incorrect format')
def produce(self, **kwargs):
# print(kwargs)
# {'payload': {'result': 'success', 'request_type': 'GET', 'name': 'get_banners', 'response_time': 129.57279199999937, 'response_length': 15}, 'test_name': 'BANNERS.LOAD_TEST'}
payload = kwargs.get("payload")
test_name = kwargs.get("test_name")
metric_response_time = "%s.%s.%s.%s.response_time" % (
self.sanitize_string(test_name), payload["request_type"],
self.sanitize_string(payload["name"]), payload["result"])
# print(metric_response_time+ "=====>" +str(payload["response_time"]))
metric_response_length = "%s.%s.%s.%s.response_length" % (
self.sanitize_string(test_name), payload["request_type"],
self.sanitize_string(payload["name"]), payload["result"])
metric_resquests_count = "%s.%s.%s.%s" % (
self.sanitize_string(test_name), payload["request_type"],
self.sanitize_string(payload["name"]), payload["result"])
# send metrics
#graphyte.init(self.graphite_host, prefix='locust.loadtest', protocol='tcp')
graphyte.init(self.graphite_host,
prefix='locust.loadtest',
protocol='udp')
print(payload["response_time"])
graphyte.send(metric_response_time, payload["response_time"])
#graphyte.send(metric_response_length, payload["response_length"])
graphyte.send(metric_resquests_count, 1)
def send_to_graphite(host, path, metric, prefix):
graphyte.init(host, prefix=prefix)
log.info("sending graphite data path: {}, metric: {}".format(path, metric))
try:
graphyte.send(path, metric)
except Exception as e:
log.error("{}".format(e))
def on_start(self):
print("on_start")
#print("Connecting to redis")
#r = redis.Redis(host='localhost',port=6379)
#vals = r.spop("locust_users",2)
#print(vals)
graphyte.init('perf-test-influxdb')
def getStat(section, metrics):
date_time = datetime.utcnow()
now = str(date_time).replace(' ', 'T')[:-10]
beforeAnyTime = str(date_time - timedelta(seconds=int(section))).replace(' ', 'T')[:-10]
token = getToken()
for metric in metrics:
headers = {'Authorization': 'Token ' + token, 'Content-Type': 'application/json'}
payload = {'service': 'CDN', 'from': beforeAnyTime, 'to': now, 'group_by': 'resource', 'granularity': '1m',
'metrics': metric}
request = requests.get("https://api.gcdn.co/statistics/series?query_string", headers=headers, params=payload)
graphyte.init(graphite_host, port=graphite_port, prefix=metric_prefix)
try:
response = json.loads(request.text)
if response == {}:
pass
else:
totalValues = response['resource']
for i in resources.keys():
try:
metric = list(totalValues[i]['metrics'].keys())[0]
values = totalValues[i]['metrics'][metric]
prefix = i + '_' + resources[i].replace('.', '_') + '.' + metric
#print(prefix)
for v in values:
graphyte.send(prefix, v[1], v[0])
#print(prefix, v[1], v[0])
except:
pass
except:
pass
def main():
orgs = get_orgs()
report_console(orgs)
if len(sys.argv) > 1:
graphyte.init(sys.argv[1])
report_graphite(orgs)
def log_against_graphite(args):
if not args.password:
print("--password must be given for get_data")
return
result = {}
graphyte.init(args.graphite, port=2003, protocol="udp")
ip, name = autodetect_ess()
ess = ESS(name, args.password)
while True:
time.sleep(10 if not args.once else 0)
home = ess.get_state("home")
common = ess.get_state("common")
for key in home:
for key2 in home[key]:
try:
graphyte.send("home." + key + "." + key2,
float(home[key][key2]))
except:
pass
for key in common:
for key2 in common[key]:
try:
graphyte.send("common." + key + "." + key2,
float(common[key][key2]))
except:
pass
logger.info(datetime.datetime.now())
if args.once:
break
def test_send_socket(self):
graphyte.init('127.0.0.1', protocol='udp')
graphyte.send('foo', 42, timestamp=12345)
graphyte.send('bar', 43.5, timestamp=12346)
self.server.handle_request()
self.server.handle_request()
self.assertEqual(TestHandler.pop_message(), b'foo 42 12345\n')
self.assertEqual(TestHandler.pop_message(), b'bar 43.5 12346\n')
def main():
# Initialize graphite
graphyte.init(graphite_server_address,
graphite_server_port,
prefix=metric_root)
# Run the collectors
array_stats(array_name, array_url, api_token)
def __init__(self, metric, uri):
"""
Parameters:
metric (str): metric type to form key
uri (str): the uri for the graphit service
"""
prefix = '.'.join([HOSTNAME, metric]) # generates prefix for key
graphyte.init(uri, prefix=prefix)
def send_to_graphite(self, host: str, prefix: str = 'air.state'):
graphyte.init(host, prefix="%s.%s" % (prefix, self._station))
for p in self:
ts = p.last_update_seconds
graphyte.send(metric=p.chemicalFormula + ".pdk",
value=p.norma,
timestamp=ts)
graphyte.send(metric=p.chemicalFormula + ".value",
value=p.value,
timestamp=ts)
def main():
# init elasticsearch
es = Elasticsearch([{'host': 'elasticsearch', 'port': 9200}])
es.indices.create(index=ES_INDEX, ignore=400)
# init graphyte
graphyte.init('graphite', port=2003, prefix='system.sync')
# send data
while True:
send_to_es(es)
send_to_graphyte()
sleep(5)
def main(args):
graphyte.init("localhost", prefix="test")
with open("/tmp/docker-kafka/kafka-metrics.json") as file:
while True:
line = file.readline()
if line not in ("", "\n", "\r"):
try:
dict = json.loads(line)
extract_data(dict)
except (Exception):
pass
else:
time.sleep(1)
print("no more metrics")
def main():
(options, args) = parse_options()
os.system('ip link set eth0 qlen 1000')
os.system(
'tc qdisc add dev eth0 root tbf rate 512kbit latency 60ms burst 1540')
link = options.url
file_name = "tmp"
graphyte.init(options.graphite_host, prefix='http_dl_' + options.url)
with open(file_name, "wb") as f:
with requests.get(link, stream=True) as response:
total_length = int(response.headers.get('content-length'))
if total_length is None:
f.write(response.content)
else:
for data in response.iter_content(chunk_size=4096):
graphyte.send('bps', 1, time.time() // 1)
f.write(data)
def parseMetrics(response, host, data):
graphyte.init(graphiteHost, port=graphitePort, prefix=graphitePrefix)
allMetrics = json.loads(response.text)
worker = host.replace('.', '_')
for metrics in allMetrics:
record = {"{#WORKER}": host}
if 'CHANNEL' in metrics:
metricPrefix = worker + '.' + metrics
sendChannel(metricPrefix, channel=allMetrics[metrics])
record['{#CHANNEL}'] = metrics
elif 'SINK' in metrics:
metricPrefix = worker + '.' + metrics
sendSink(metricPrefix, sink=allMetrics[metrics])
record['{#SINK}'] = metrics
elif 'SOURCE' in metrics:
metricPrefix = worker + '.' + metrics
sendSource(metricPrefix, source=allMetrics[metrics])
record['{#SOURCE}'] = metrics
data.append(record)
#!/usr/bin/env python3
import requests
import urllib.parse
import re
import time
# local config
import config as cfg
import graphyte
graphyte.init(cfg.graphite['g_host'], prefix=cfg.graphite['prefix'], interval=20 )
while 1:
# request page
vars = (requests.get(cfg.hub['url'] + '/cgi/cgi_basicMyDevice.js').content.decode()).split('var')
rate = vars[2].split('\n')
for rate in rate:
rate = urllib.parse.unquote(rate.replace('\'', ''))
rate_items = rate.split(',')
if ( re.search('timestamp', rate_items[0])):
rate_mac = (rate_items[2].replace('mac:', '')).upper()
tx = (rate_items[3].replace('tx:', ''))
rx = (rate_items[4].replace('rx:', ''))
rx = rx.replace('}','')
graphyte.send(rate_mac + '.' + 'tx',int(tx))
graphyte.send(rate_mac + '.' + 'rx',int(rx))
time.sleep(10)
distavg = 0
count = 0
medianDistance = 0
mDistance = [0] * 100
#SENSORLOCATION
PIERHEIGHTINFEET = 82.5 / 12
SENSORHEIGHTINFEET = 91.0 / 12
SENSORABOVEPIERINFEET = SENSORHEIGHTINFEET - PIERHEIGHTINFEET
MAXFLOWTIME = (60 * 60)
GRAPHITE = "localhost"
myws = "KTXOLIVI"
mywslocation = "PancamoPointPier"
graphyte.init(GRAPHITE, prefix='wu.KTXOLIVI.PI2')
SERIAL_PORT = "/dev/serial0" # or "COM4" or whatever
serialport = serial.Serial(SERIAL_PORT, 9600)
# This function measures a distance
def read_me007ys(ser, timeout=1.0):
ts = time.time()
buf = bytearray(3)
idx = 0
while True:
# Option 1, we time out while waiting to get valid data
if time.time() - ts > timeout:
#!/usr/bin/python3
import board
import busio
import RPi.GPIO as GPIO
import adafruit_bme680
import time
import graphyte
i2c = busio.I2C(board.SCL, board.SDA)
sensor = adafruit_bme680.Adafruit_BME680_I2C(i2c)
GPIO.setup(24, GPIO.OUT)
fridgestate = False
HOST = "192.168.0.181"
graphyte.init(HOST, prefix='dryage', interval=10)
GPIO.output(24, 0)
temppos = 0
temps = [2.6, 2.6, 2.6, 2.6, 2.6]
onetwentytemps = [2.6] * 10
onetwentytemppos = 0
def sleeptime(sltime):
print("Sleep Thirty")
for x in range(0, sltime):
time.sleep(1)
graphyte.send('fridgestate', int(fridgestate))
print("{}...".format(x), end='')
while 1:
curtemp = sensor.temperature
prevavgtemp = sum(temps) / len(temps)
help='logfile. default: /tmp/solidfire-graphite-collector.log')
args = parser.parse_args()
to_graphite = True
# Logger module configuration
if args.logfile:
LOG = logging.getLogger('solidfire_graphite_collector.py')
logging.basicConfig(filename=args.logfile, level=logging.DEBUG, format='%(asctime)s %(message)s')
LOG.warning("Starting Collector script as a daemon. No console output possible.")
# Initialize graphyte sender
if args.graphite == "debug":
LOG.warning("Starting collector in debug mode. All the metrics will be shipped to logfile")
to_graphite = False
else:
graphyte.init(args.graphite, port=args.port, prefix=args.metricroot)
LOG.info("Metrics Collection for array: {0}".format(args.solidfire))
try:
sfe = ElementFactory.create(args.solidfire, args.username, args.password)
sfe.timeout(15)
cluster_name = sfe.get_cluster_info().to_json()['clusterInfo']['name']
send_cluster_stats(sfe, cluster_name)
send_cluster_faults(sfe, cluster_name)
send_cluster_capacity(sfe, cluster_name)
send_node_stats(sfe, cluster_name + '.node')
send_volume_stats(sfe, cluster_name)
send_drive_stats(sfe, cluster_name)
except solidfire.common.ApiServerError as e:
LOG.warning("ApiServerError: {0}".format(str(e)))
except Exception as e:
def send_data(data, host, port, prefix):
graphyte.init(host, prefix=prefix)
for metric, value in format_for_graphite(data).items():
print(f"Sending metric: {metric} with value: {value}")
graphyte.send(metric, value)
return True
def send_to_graphite(graphite_host, prefix, intervals, start_time):
graphyte.init(graphite_host, prefix='iperf_'+prefix)
for interval in intervals:
time = start_time + interval['sum']['start']
graphyte.send('bps', interval['sum']['bits_per_second'], time)
import graphyte
from datetime import datetime
from datetime import timedelta
graphyte.init('graphite', prefix='system.sync')
#graphyte.send('test.new', 2000)
for i in range(0,1000):
ts = (datetime.now() - timedelta(minutes=i)).timestamp()
print((datetime.now() - timedelta(minutes=i)))
i = i + 10
graphyte.send('test.new', i, ts)
def write_graphite(prefix, timestamp, metricvalue, metricname,
host): ## Add metric path / prefix
## Add Exception Handler
graphyte.init(host, prefix=prefix)
#print(metricname, int(metricvalue))
graphyte.send(metricname, int(metricvalue), timestamp=timestamp)
skyline_app = 'flux'
skyline_app_graphite_namespace = 'skyline.%s%s.populate_metric_worker' % (
parent_skyline_app, SERVER_METRIC_PATH)
# LOCAL_DEBUG = False
LOCAL_DEBUG = True
GRAPHITE_METRICS_PREFIX = None
CARBON_HOST = settings.FLUX_CARBON_HOST
CARBON_PORT = settings.FLUX_CARBON_PORT
FLUX_CARBON_PICKLE_PORT = settings.FLUX_CARBON_PICKLE_PORT
if settings.FLUX_SEND_TO_CARBON:
try:
graphyte.init(CARBON_HOST, port=CARBON_PORT, prefix=None, timeout=5)
logger.info(
'populate_metric_worker :: succeeded to graphyte.init with host: %s, port: %s, prefix: %s'
%
(str(CARBON_HOST), str(CARBON_PORT), str(GRAPHITE_METRICS_PREFIX)))
except:
logger.error(traceback.format_exc())
logger.error(
'error :: populate_metric_worker :: failed to run graphyte.init with host: %s, port: %s, prefix: %s'
%
(str(CARBON_HOST), str(CARBON_PORT), str(GRAPHITE_METRICS_PREFIX)))
class PopulateMetricWorker(Process):
"""
The worker grabs metrics from the queue, surfaces the data from the remote
def send_to_graphite(host, target, metric, time):
target = target.replace('.', '_')
graphyte.init(host, prefix=target)
graphyte.send('result', metric, time)
def main():
# Parse commandline arguments
parser = argparse.ArgumentParser()
parser.add_argument(
'-s',
'--solidfire',
default=os.environ.get('SOLIDFIRE', None),
help=
'hostname of SolidFire array from which metrics should be collected')
parser.add_argument('-u',
'--username',
default=os.environ.get('SOLIDFIREUSER', None),
help='username for SolidFire array. default admin')
parser.add_argument('-p',
'--password',
default=os.environ.get('SOLIDFIREPASSWORD', None),
help='password for SolidFire array. default password')
parser.add_argument(
'-g',
'--graphite',
default='graphite',
help=
'hostname of Graphite server to send to. default localhost. "debug" sends metrics to logfile'
)
parser.add_argument(
'-t',
'--port',
type=int,
default=2003,
help=
'port to send message to. default 2003. if the --graphite is set to debug can be omitted'
)
parser.add_argument(
'-m',
'--metricroot',
default='netapp.solidfire.cluster',
help='graphite metric root. default netapp.solidfire.cluster')
parser.add_argument('-l',
'--log',
default=True,
help='logfile. default: True')
parser.add_argument('-f',
'--logfile',
type=argparse.FileType('w'),
help='logfile. Specifiy a Log file vs stdrr or stdout')
args = parser.parse_args()
global to_graphite
to_graphite = True
# Logger module configuration
if bool(args.log):
LOG = logging.getLogger('solidfire_graphite_collector.py')
if args.logfile:
logging.basicConfig(filename=args.logfile,
level=logging.DEBUG,
format='%(asctime)s %(message)s')
else:
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(message)s')
LOG.warning(
"Starting Collector script as a daemon. No console output possible."
)
# Initialize graphyte sender
if args.graphite == "debug":
LOG.warning(
"Starting collector in debug mode. All the metrics will be shipped to logfile"
)
to_graphite = False
else:
graphyte.init(args.graphite, port=args.port, prefix=args.metricroot)
while (1):
LOG.info("Metrics Collection for array: {0}".format(args.solidfire))
try:
sfe = ElementFactory.create(args.solidfire, args.username,
args.password)
sfe.timeout(15)
cluster_name = sfe.get_cluster_info().to_json(
)['clusterInfo']['name']
send_cluster_stats(sfe, cluster_name)
send_cluster_faults(sfe, cluster_name)
send_cluster_capacity(sfe, cluster_name)
send_node_stats(sfe, cluster_name + '.node')
send_volume_stats(sfe, cluster_name)
send_drive_stats(sfe, cluster_name)
except solidfire.common.ApiServerError as e:
LOG.warning("ApiServerError: {0}".format(str(e)))
break
except Exception as e:
LOG.warning("General Exception: {0}".format(str(e)))
break
os.system("graphite")
time.sleep(60)
sfe = None
my_logger.setLevel(logging.DEBUG)
handler = logging.handlers.SysLogHandler(
address=(DOCKER_LOGHOST, int(DOCKER_LOGHOST_PORT)))
my_logger.addHandler(handler)
else:
print('NO SYSLOG ACTIVATED', file=sys.stderr)
# activate graphite
if DOCKER_G_HOST is not None:
if DOCKER_G_PREFIX is None:
DOCKER_G_PREFIX = 'smarthub'
print('SENDING BW METRICS TO: ' + DOCKER_G_HOST + ' prefix:' +
DOCKER_G_PREFIX,
file=sys.stderr)
graphyte.init(DOCKER_G_HOST, prefix=DOCKER_G_PREFIX, interval=20)
else:
print('NO GRAPHITEACTIVATED', file=sys.stderr)
# body of login request
login_body = ('O=helpdesk.htm&usr=admin&pws=' +
hashlib.md5(DOCKER_PASS.encode('utf-8')).hexdigest())
# max amount of events to check
max_event_count = 50
# login function
def login():
login_ts = int(time.time())
print('LOGGING IN AT: ' + str(login_ts) + " " + login_body,
def start(self):
graphyte.init(str(self.graphite_host), prefix=str(self.prefix))
import time
import graphyte
import sys
hostname = socket.gethostname()
config = load_config('config.yaml')
if config is None:
sys.exit('Config is empty, please supply a whitelist')
whitelist = config.get('whitelist')
if whitelist is None or whitelist == []:
sys.exit('Whitelist is empty, exiting')
carbon = config.get('carbon', {})
graphyte.init(
carbon.get('hostname', 'localhost'),
prefix=f'gm.{hostname}',
)
def get_memory():
"""Get a per process summary of all memory used by all processes in the whitelist"""
procs = {}
for program, rss in [(x.name(), x.memory_info().rss)
for x in psutil.process_iter()
if x.name() in whitelist]:
if program in procs:
procs[program] += rss
else:
procs[program] = rss
return procs
logger.info(f'Appliance name = {appliance_name}')
logger.info(f'Instance name = {instance_name}')
logger.info(f'Instance PID = {os.getpid()}')
logger.info(sys.version)
monitor.configure('processor', instance_name, config.mercure['bookkeeper'])
monitor.send_event(monitor.h_events.BOOT, monitor.severity.INFO,
f'PID = {os.getpid()}')
if len(config.mercure['graphite_ip']) > 0:
logger.info(
f'Sending events to graphite server: {config.mercure["graphite_ip"]}'
)
graphite_prefix = 'mercure.' + appliance_name + '.processor.' + instance_name
graphyte.init(config.mercure['graphite_ip'],
config.mercure['graphite_port'],
prefix=graphite_prefix)
logger.info(f'Processing folder: {config.mercure["processing_folder"]}')
processor_lockfile = Path(config.mercure['processing_folder'] + '/HALT')
# Start the timer that will periodically trigger the scan of the incoming folder
global main_loop
main_loop = helper.RepeatedTimer(
config.mercure['dispatcher_scan_interval'], run_processor,
exit_processor, {})
main_loop.start()
helper.g_log('events.boot', 1)
# Start the asyncio event loop for asynchronous function calls
