def send(self, metric: str, value, host: Optional[str] = None, port: Optional[int] = None, tags: Optional[Dict[str, str]] = None, prefix: str = '', protocol: str = 'tcp'): """ Send data to a Graphite instance. :param metric: Metric name. :param value: Value to be sent. :param host: Graphite host (default: default configured ``host``). :param port: Graphite port (default: default configured ``port``). :param tags: Map of tags for the metric. :param prefix: Metric prefix name (default: empty string). :param protocol: Communication protocol - possible values: 'tcp', 'udp' (default: 'tcp'). """ import graphyte host = host or self.host port = port or self.port tags = tags or {} protocol = protocol.lower() graphyte.init(host, port=port, prefix=prefix, protocol=protocol) graphyte.send(metric, value, tags=tags)
def log_against_graphite(args): if not args.password: print("--password must be given for get_data") return result = {} graphyte.init(args.graphite, port=2003, protocol="udp") ip, name = autodetect_ess() ess = ESS(name, args.password) while True: time.sleep(10 if not args.once else 0) home = ess.get_state("home") common = ess.get_state("common") for key in home: for key2 in home[key]: try: graphyte.send("home." + key + "." + key2, float(home[key][key2])) except: pass for key in common: for key2 in common[key]: try: graphyte.send("common." + key + "." + key2, float(common[key][key2])) except: pass logger.info(datetime.datetime.now()) if args.once: break
def send_to_graphite(host, path, metric, prefix): graphyte.init(host, prefix=prefix) log.info("sending graphite data path: {}, metric: {}".format(path, metric)) try: graphyte.send(path, metric) except Exception as e: log.error("{}".format(e))
def update(self, filename, time, value): if value is None: return filename = munge_filename(filename) graphyte.send(f"{filename}", value, time)
def getStat(section, metrics): date_time = datetime.utcnow() now = str(date_time).replace(' ', 'T')[:-10] beforeAnyTime = str(date_time - timedelta(seconds=int(section))).replace(' ', 'T')[:-10] token = getToken() for metric in metrics: headers = {'Authorization': 'Token ' + token, 'Content-Type': 'application/json'} payload = {'service': 'CDN', 'from': beforeAnyTime, 'to': now, 'group_by': 'resource', 'granularity': '1m', 'metrics': metric} request = requests.get("https://api.gcdn.co/statistics/series?query_string", headers=headers, params=payload) graphyte.init(graphite_host, port=graphite_port, prefix=metric_prefix) try: response = json.loads(request.text) if response == {}: pass else: totalValues = response['resource'] for i in resources.keys(): try: metric = list(totalValues[i]['metrics'].keys())[0] values = totalValues[i]['metrics'][metric] prefix = i + '_' + resources[i].replace('.', '_') + '.' + metric #print(prefix) for v in values: graphyte.send(prefix, v[1], v[0]) #print(prefix, v[1], v[0]) except: pass except: pass
def sendMessage(self): ws = create_connection("wss://echo.websocket.org", sslopt={"cert_reqs": ssl.CERT_NONE}) user_id = six.text_type(uuid.uuid4()) start_at = time.time() json_val = { 'id':user_id, 'salutation': "Hello", 'name': "James", 'start_at': start_at } msg = json.dumps(json_val) ws.send(msg) print(f"Sent {msg}") res = ws.recv() data = json.loads(res) end_at = time.time() response_time = int((end_at - data['start_at']) * 1000) print(f"Received {res}") print(f"Got back id {data['id']}") request_success.fire( request_type='WebSocket Response', name='ws', response_time=response_time, response_length=len(res), ) graphyte.send('websocket.response_time',response_time)
def send_metric(id, ts, name, values): short_name = "" value = "" if "ActiveControllerCount" in name: short_name = "ActiveControllerCount" value = values["Value"] elif "IsrExpandsPerSec" in name: short_name = "IsrExpandsPerSec" value = values["Count"] elif "IsrShrinksPerSec" in name: short_name = "IsrShrinksPerSec" value = values["Count"] elif "OfflinePartitionsCount" in name: short_name = "OfflinePartitionsCount" value = values["Value"] elif "UncleanLeaderElectionsPerSec" in name: short_name = "UncleanLeaderElectionsPerSec" value = values["Count"] elif "UnderReplicatedPartitions" in name: short_name = "UnderReplicatedPartitions" value = values["Value"] if value != "": path = id + "." + short_name graphyte.send(path, int(value), timestamp=ts)
def produce(self, **kwargs): # print(kwargs) # {'payload': {'result': 'success', 'request_type': 'GET', 'name': 'get_banners', 'response_time': 129.57279199999937, 'response_length': 15}, 'test_name': 'BANNERS.LOAD_TEST'} payload = kwargs.get("payload") test_name = kwargs.get("test_name") metric_response_time = "%s.%s.%s.%s.response_time" % ( self.sanitize_string(test_name), payload["request_type"], self.sanitize_string(payload["name"]), payload["result"]) # print(metric_response_time+ "=====>" +str(payload["response_time"])) metric_response_length = "%s.%s.%s.%s.response_length" % ( self.sanitize_string(test_name), payload["request_type"], self.sanitize_string(payload["name"]), payload["result"]) metric_resquests_count = "%s.%s.%s.%s" % ( self.sanitize_string(test_name), payload["request_type"], self.sanitize_string(payload["name"]), payload["result"]) # send metrics #graphyte.init(self.graphite_host, prefix='locust.loadtest', protocol='tcp') graphyte.init(self.graphite_host, prefix='locust.loadtest', protocol='udp') print(payload["response_time"]) graphyte.send(metric_response_time, payload["response_time"]) #graphyte.send(metric_response_length, payload["response_length"]) graphyte.send(metric_resquests_count, 1)
def send_vol_efficiency(sfe, prefix): """ Send per-volume efficiency info (dedupe & compression aka CxD only) Can be used to identify low-efficiency volumes (e.g. < 1.5x) """ results = sfe.list_accounts().to_json() for account in results['accounts']: av = sfe.list_volumes(accounts=[account['accountID']]).to_json() if len(av['volumes']) > 0: for volume in av['volumes']: vol_eff_d = sfe.get_volume_efficiency( volume['volumeID']).to_json() vol_eff = round( (vol_eff_d['deduplication'] * vol_eff_d['compression']), 2) if to_graphite: graphyte.send( prefix + '.volumeID.' + str(volume['volumeID']) + '.volumeEfficiency', float(vol_eff)) else: print(prefix + '.volumeID.' + str(volume['volumeID']) + '.volumeEfficiency ' + str(vol_eff)) LOG.warning(prefix + '.volumeID.' + str(volume['volumeID']) + '.volumeEfficiency ' + str(vol_eff)) else: LOG.warning(prefix + ': account ID ' + str(account['accountID']) + ' has no volumes')
def send_volume_histogram_stats(sfe, prefix): """ Send volume QoS histogram stats. Requires API v11 or above Note: as of August 2020, this API method is not well documented so stuff may not mean what we think it means. """ hmetrics = [ 'belowMinIopsPercentages', 'minToMaxIopsPercentages', 'minToMaxIopsPercentages', 'readBlockSizes', 'throttlePercentages', 'writeBlockSizes' ] qosh = sfe.invoke_sfapi("ListVolumeQoSHistograms", parameters=None) for i in range(len(qosh['qosHistograms'])): for metric in hmetrics: for key, value in ( qosh['qosHistograms'][i]['histograms'][metric]).items(): if to_graphite: graphyte.send( prefix + '.volumeID.' + str(qosh['qosHistograms'][i]['volumeID']) + '.' + metric + '.' + key, int(value)) else: LOG.warning(prefix + '.volumeID.' + str(qosh['qosHistograms'][i]['volumeID']) + '.' + metric + '.' + key + ' ' + str(value))
def send_volume_stats(sf_element_factory, prefix): """ send a subset of ListVolumeStatsByVolume results to graphite. Note: Calls ListVolumes to get volume names for use in metric path. """ metrics_list = [ 'volumeSize', 'zeroBlocks', 'nonZeroBlocks', 'volumeUtilization', 'actualIOPS', 'averageIOPSize', 'throttle', 'burstIOPSCredit', 'clientQueueDepth', 'latencyUSec', 'totalLatencyUSec', 'writeBytes', 'writeOps', 'writeLatencyUSec', 'unalignedWrites', 'readBytes', 'readOps', 'readLatencyUSec', 'unalignedReads' ] volume_list = sf_element_factory.list_volumes().to_json()['volumes'] volinfo_by_id = list_to_dict(volume_list, key="volumeID") volstats = sf_element_factory.list_volume_stats_by_volume().to_json( )['volumeStats'] for vs_dict in volstats: vol_name = volinfo_by_id[vs_dict['volumeID']]['name'] vol_accountID = volinfo_by_id[vs_dict['volumeID']]['accountID'] vol_accountName = sf_element_factory.get_account_by_id( vol_accountID).to_json()['account']['username'] # if account has a dot, trim it, cause it will conflict with graphite metrics structure. vol_accountName = vol_accountName.replace('.', '') for key in metrics_list: graphyte.send(prefix + '.accountID.' + str(vol_accountName) + \ '.volume.' + vol_name + '.' + key, to_num(vs_dict[key]))
def send_cluster_stats(sfe, prefix): """ send a subset of GetClusterStats API call results to graphite. """ metrics = [ 'clientQueueDepth', 'clusterUtilization', 'readOpsLastSample', 'readBytesLastSample', 'writeOpsLastSample', 'writeBytesLastSample', 'actualIOPS', 'latencyUSec', 'normalizedIOPS', 'readBytes', 'readLatencyUSec', 'readOps', 'unalignedReads', 'unalignedWrites', 'writeLatencyUSec', 'writeOps', 'writeBytes' ] cluster_stats_dict = sfe.get_cluster_stats().to_json()['clusterStats'] clusterUtilizationDec = float(cluster_stats_dict['clusterUtilization']) clusterUtilizationScaled = clusterUtilizationDec if to_graphite: graphyte.send(prefix + '.clusterUtilizationScaled', to_num(clusterUtilizationScaled)) else: LOG.warning(prefix + '.clusterUtilizationScaled ' + str(clusterUtilizationScaled)) for key in metrics: if to_graphite: graphyte.send(prefix + '.' + key, to_num(cluster_stats_dict[key])) else: LOG.warning(prefix + '.' + key + ' ' + str(cluster_stats_dict[key]))
def send_volume_stats(sf_element_factory, prefix): """ send a subset of ListVolumeStatsByVolume results to graphite. Note: Calls ListViolumes to get volume names for use in metric path. """ metrics_list = ['volumeSize', 'zeroBlocks', 'nonZeroBlocks', 'volumeUtilization', 'actualIOPS', 'averageIOPSize', 'throttle', 'burstIOPSCredit', 'clientQueueDepth', 'latencyUSec', 'writeBytes', 'writeOps', 'writeLatencyUSec', 'unalignedWrites', 'readBytes', 'readOps', 'readLatencyUSec', 'unalignedReads', 'readBytesLastSample', 'readOpsLastSample', 'writeBytesLastSample', 'writeOpsLastSample'] volume_list = sf_element_factory.list_volumes(include_virtual_volumes=False).to_json()['volumes'] volinfo_by_id = list_to_dict(volume_list, key="volumeID") volstats = sf_element_factory.list_volume_stats_by_volume(include_virtual_volumes=False).to_json()['volumeStats'] for vs_dict in volstats: vol_name = volinfo_by_id[vs_dict['volumeID']]['name'] vol_accountID = volinfo_by_id[vs_dict['volumeID']]['accountID'] vol_accountName = sf_element_factory.get_account_by_id(vol_accountID).to_json()['account']['username'] for key in metrics_list: if to_graphite: graphyte.send(prefix + '.accountID.' + str(vol_accountName) + \ '.volume.' + vol_name + '.' + key, to_num(vs_dict[key])) else: LOG.warning('accountID ' + str(vol_accountName) + \ ' volume ' + vol_name + ' ' + key + ' ' + str(vs_dict[key]))
def test_send_socket(self): graphyte.init('127.0.0.1', protocol='udp') graphyte.send('foo', 42, timestamp=12345) graphyte.send('bar', 43.5, timestamp=12346) self.server.handle_request() self.server.handle_request() self.assertEqual(TestHandler.pop_message(), b'foo 42 12345\n') self.assertEqual(TestHandler.pop_message(), b'bar 43.5 12346\n')
def send_cluster_capacity(sf_element_factory, prefix): """ send a subset of GetClusterCapacity API call results and derived metrics to graphite. """ metrics = ['activeBlockSpace', 'activeSessions', 'averageIOPS', 'clusterRecentIOSize', 'currentIOPS', 'maxIOPS', 'maxOverProvisionableSpace', 'maxProvisionedSpace', 'maxUsedMetadataSpace', 'maxUsedSpace', 'nonZeroBlocks', 'peakActiveSessions', 'peakIOPS', 'provisionedSpace', 'snapshotNonZeroBlocks', 'timestamp', 'totalOps', 'uniqueBlocks', 'uniqueBlocksUsedSpace', 'usedMetadataSpace', 'usedMetadataSpaceInSnapshots', 'usedSpace', 'zeroBlocks'] result = sf_element_factory.get_cluster_capacity().to_json()['clusterCapacity'] for key in metrics: if to_graphite: graphyte.send(prefix + '.' + key, to_num(result[key])) else: LOG.warning(key + ' ' + str(result[key])) # Calculate & send derived metrics non_zero_blocks = to_num(result['nonZeroBlocks']) zero_blocks = to_num(result['zeroBlocks']) unique_blocks = to_num(result['uniqueBlocks']) unique_blocks_used_space = to_num(result['uniqueBlocksUsedSpace']) if non_zero_blocks != 0: thin_factor = float((non_zero_blocks + zero_blocks)) / float(non_zero_blocks) else: thin_factor = 1 if to_graphite: graphyte.send(prefix + '.thin_factor', thin_factor) else: LOG.warning(key + ' ' + str(result[key])) if unique_blocks != 0: dedupe_factor = float(non_zero_blocks) / float(unique_blocks) else: dedupe_factor = 1 if to_graphite: graphyte.send(prefix + '.dedupe_factor', dedupe_factor) else: LOG.warning('dedupe_factor ' + str(dedupe_factor)) if unique_blocks_used_space != 0: #compression_factor = (unique_blocks * 4096.0) / unique_blocks_used_space compression_factor = (unique_blocks * 4096.0) / (unique_blocks_used_space * .93) else: compression_factor = 1 if to_graphite: graphyte.send(prefix + '.compression_factor', compression_factor) else: LOG.warning('compression_factor ' + str(compression_factor)) efficiency_factor = thin_factor * dedupe_factor * compression_factor if to_graphite: graphyte.send(prefix + '.efficiency_factor', efficiency_factor) else: LOG.warning('efficiency_factor ' + str(efficiency_factor))
def send_elem_version(sfe, prefix): """ Send the highest API version supported by current Element API """ result = sfe.get_cluster_version_info().to_json() sf_version = result['clusterAPIVersion'] if to_graphite: graphyte.send(prefix + '.version', float(sf_version)) else: LOG.warning(prefix + '.version ' + str(sf_version))
def send(self, bucket, value): """ Parameters: bucket: the value should be the lower bound of each bucket value: numeric value """ # TODO: Test with graphite # TODO: confirm ideal variable types: # should be indifferent of str, unicode, int graphyte.send(bucket, value)
def send_to_graphite(self, host: str, prefix: str = 'air.state'): graphyte.init(host, prefix="%s.%s" % (prefix, self._station)) for p in self: ts = p.last_update_seconds graphyte.send(metric=p.chemicalFormula + ".pdk", value=p.norma, timestamp=ts) graphyte.send(metric=p.chemicalFormula + ".value", value=p.value, timestamp=ts)
def put_graphite(self, docs): for doc in docs: keys = list(doc) hostname = doc['node_fqdn'].split('.')[0] metric_path = hostname + '.openelastic.' + doc[ 'metric'] + '.' + str(doc[keys[1]]) + '_' + doc['agg'] metric_value = (doc[doc['metric']]) # print(metric_path + " " + str(metric_value)) graphyte.send(metric_path, float(metric_value))
def recursive_parse_metric(self, metric_path, data): if isinstance(data, dict): for k, v in data.items(): name = '{}.{}'.format(metric_path, k) self.recursive_parse_metric(name, v) if isinstance(data, list): for value, timestamp in data: if not (self.test_start_timestamp and timestamp < self.test_start_timestamp): delta = timestamp - self.test_start_timestamp relative_timestamp = BASE_TIMESTAMP_TO_UPLOAD + delta graphyte.send(metric_path, value, relative_timestamp)
def send_cluster_stats(sf_element_factory, prefix): """ send a subset of GetClusterStats API call results to graphite. """ metrics = ['clientQueueDepth', 'clusterUtilization'] monotonic = ['readOps', 'readBytes', 'writeOps', 'writeBytes'] cluster_stats_dict = sf_element_factory.get_cluster_stats().to_json( )['clusterStats'] for key in metrics: graphyte.send(prefix + '.' + key, to_num(cluster_stats_dict[key])) for key in monotonic: graphyte.send(prefix + '.' + key, to_num(cluster_stats_dict[key]))
def send_drive_stats(sf_element_factory, prefix): """ calculates summary statistics about drives by status and type at both cluster and node levels and submits them to graphite. Calls ListDrives and ListAllNodes """ # Cluster level stats drive_list = sf_element_factory.list_drives().to_json()['drives'] for status in ['active', 'available', 'erasing', 'failed', 'removing']: value = count_if(drive_list, 'status', status) graphyte.send(prefix + '.drives.status.' + status, value) for type in ['volume', 'block', 'unknown']: value = count_if(drive_list, 'type', type) graphyte.send(prefix + '.drives.type.' + type, value) # Node level stats node_list = sf_element_factory.list_all_nodes().to_json()['nodes'] nodeinfo_by_id = list_to_dict(node_list, key="nodeID") for node in nodeinfo_by_id: node_name = nodeinfo_by_id[node]['name'] for status in ['active', 'available', 'erasing', 'failed', 'removing']: value = count_ifs(drive_list, 'status', status, 'nodeID', node) graphyte.send( prefix + '.node.' + node_name + '.drives.status.' + status, value) for drive_type in ['volume', 'block', 'unknown']: value = count_ifs(drive_list, 'type', drive_type, 'nodeID', node) graphyte.send( prefix + '.node.' + node_name + '.drives.type.' + drive_type, value)
def send_to(metric, metric_value, timestamp=None): '''(str, float) -> str|None it sends to graphite or print to standard output the passed metrics. >>> send_to(volume_space.vol_ppcmeta_221.volumes, 718848) ''' if timestamp: timestamp = timestamp / 1000 if to_graphite is True: graphyte.send(metric, metric_value, timestamp) else: print('{}.{} {} {}'.format(metric_root, metric, metric_value, timestamp))
def send_stats(self): graphyte.send('pulled_cnt', self.pulled_cnt) graphyte.send('queue_size', len(self.queue)) graphyte.send('added_cnt', len(self.added_urls)) for ip in self.ip_stats: for metric in self.ip_stats[ip]: graphyte.send("{}.{}".format(ip, metric), self.ip_stats[ip][metric]) self.last_stat = time.time() log.info('GRAPHITE: stats send to graphite')
def main(): (options, args) = parse_options() os.system('ip link set eth0 qlen 1000') os.system( 'tc qdisc add dev eth0 root tbf rate 512kbit latency 60ms burst 1540') link = options.url file_name = "tmp" graphyte.init(options.graphite_host, prefix='http_dl_' + options.url) with open(file_name, "wb") as f: with requests.get(link, stream=True) as response: total_length = int(response.headers.get('content-length')) if total_length is None: f.write(response.content) else: for data in response.iter_content(chunk_size=4096): graphyte.send('bps', 1, time.time() // 1) f.write(data)
def calculate_flow(data): current_time = int(time.time()) max_time = current_time + 9 * HOUR result = 0 for forecast in data["list"]: if forecast["dt"] > max_time: break shift = int((forecast["dt"] - time.time()) / HOUR) // 3 * 3 current = 0 for tp, mult in ("rain", 1), ("snow", 3): if tp in forecast: print(tp, forecast[tp].get("3h", 0), mult) graphyte.send("{}h.{}".format(shift, tp), forecast[tp].get("3h", 0)) current += forecast[tp].get("3h", 0) * mult result = max(result, current) return result / FORECAST_PERIOD_HOURS
def send_cluster_faults(sf_element_factory, prefix): """ send active cluster fault counts by: warning, error, critical exclude best practices, and only include current faults """ fault_list = sf_element_factory.list_cluster_faults(False,"current").to_json()['faults'] group = {'critical':0, 'error':0, 'warning':0 } for d in fault_list: if d['severity'] not in group: group[ d['severity'] ] = 1 else: group[ d['severity'] ] += 1 if to_graphite: for key in group: graphyte.send(prefix + '.fault.' + key, to_num(group[key])) else: for key in group: LOG.warning('fault.' + key, str(to_num(group[key])))
def send_conn_stats(sfe, prefix): """ calculates iSCSI connection stats at both cluster and node levels and submits them to Graphite. Calls ListConnections """ result = sfe.list_iscsisessions().to_json()['sessions'] tgts = [] accts = [] for i in range(len(result)): tgts.append(result[i]['targetIP'].split(':')[0]) accts.append(result[i]['initiatorIP'].split(':')[0]) if to_graphite: graphyte.send(prefix + '.iscsiActiveSessionCount', len(result)) graphyte.send(prefix + '.iscsiTargetCount', len(set(tgts))) else: LOG.warning(prefix + '.iscsiActiveSessionCount ' + str(len(result))) LOG.warning(prefix + '.iscsiTargetCount ' + str(len(set(tgts))))
def send_node_stats(sf_element_factory, prefix): """ send a subset of ListNodeStats API call results to graphite. Note: Calls ListAllNodes to get node name to use in metric path. """ metrics_list = ['cpu', 'usedMemory', 'networkUtilizationStorage', 'networkUtilizationCluster', 'cBytesOut', 'cBytesIn', 'sBytesOut', 'sBytesIn', 'mBytesOut', 'mBytesIn', 'readOps', 'writeOps'] node_list = sf_element_factory.list_all_nodes().to_json()['nodes'] nodeinfo_by_id = list_to_dict(node_list, key="nodeID") nodestats = sf_element_factory.list_node_stats().to_json()['nodeStats']['nodes'] for ns_dict in nodestats: node_name = nodeinfo_by_id[ns_dict['nodeID']]['name'] for key in metrics_list: if to_graphite: graphyte.send(prefix + '.' + node_name + '.' + key, to_num(ns_dict[key])) else: LOG.warning(node_name + ' ' + key + ' ' + str(ns_dict[key]))
def publish_wifi_signal_quality(): SSID = get_ssid() try: shell_cmd = 'iwconfig {} | grep Link'.format('wlan0') proc = Popen(shell_cmd, shell=True, stdout=PIPE, stderr=PIPE) output, err = proc.communicate() msg = output.decode('utf-8').strip() # like: # Link Quality=41/70 Signal level=-69 dBm quality_str = msg.split('Link Quality=')[1].split( '/70')[0].strip() #hurl quality = int(quality_str) graphyte.send('wifi.signal_quality', quality, tags={'SSID': SSID}) except CalledProcessError: print("couldn't get signal quality") pass
def update(self, filename, time, value): filename = munge_filename(filename) graphyte.send('{}'.format(filename), value, time)