def libratoPOST(url, data, auth=None): url = "https://metrics-api.librato.com/v1" + url debug(url) response = requests.put(url, data=data, auth=auth) if response.status_code != 204: return False return True
def get_key_from_configservice(env="DEV", api_key="", key="", default_value=None): global USE_DB if not USE_DB: url = "http://config.vs.intelius.com:8080/configuration/1.0/get?env=%s&key=%s" % (env, key) if len(api_key) > 0: url += "&callerid=%s" % api_key debug(url) try: data = requests.get(url) datajson = data.json() if data.status_code == 200 and "value" in datajson: return datajson["value"] elif "value" not in datajson: return default_value except Exception as e: USE_DB = True if USE_DB: # try to read directly from DB (config service in-proc) db_host = os.environ.get('INSIGHTS_CONFIGDB_HOST') if db_host is None: print("UNABLE TO RETRIEVE DATA FROM THE CONFIGURATION SERVICE") else: db_user = os.environ.get('INSIGHTS_CONFIGDB_USER') db_pass = os.environ.get('INSIGHTS_CONFIGDB_PASS') db = MySQLdb.connect(host=db_host, user=db_user, passwd=db_pass) cur = db.cursor() cur.execute("SELECT Value FROM Inome.Configuration WHERE Env = '%s' AND `Key` = '%s'" % (env, key)) for row in cur.fetchall(): return row[0] return default_value
def libratoGET(url, keyname=None, offset=0, auth=None): # If no keyname passed in, use the substring of url starting at index 1 if keyname is None: keyname = url[1:] fullurl = "https://metrics-api.librato.com/v1" + url + "?length=" + str(DEFAULT_LIBRATO_LENGTH) + "&offset=" + str(offset) debug(fullurl) data = requests.get(fullurl, auth=auth) returnArray = [] if data.status_code == 200: datajson = data.json() debug("Length = " + str(datajson["query"]["length"]) + ", found = " + str(datajson["query"]["found"]) + ", total=" + str(datajson["query"]["total"]) + ", offset=" + str(datajson["query"]["offset"])) returnArray.extend(datajson[keyname]) if (datajson["query"]["length"] + datajson["query"]["offset"]) < datajson["query"]["found"]: returnArray.extend(libratoGET(url, keyname=keyname, offset=(datajson["query"]["offset"] + DEFAULT_LIBRATO_LENGTH), auth=auth)) return returnArray return None
def addToSecurityGroup(pingdom_security_groups, ip): # length has to be at least 1 if len(pingdom_security_groups) == 0: return False # group to attempt to add this ip to pingdom_group = pingdom_security_groups[0] try: debug("Attempting to add " + ip + " to " + pingdom_group.name) conn.authorize_security_group(group_id=pingdom_group.id, ip_protocol="tcp", from_port=80, to_port=80, cidr_ip="%s/32" % ip) except boto.exception.EC2ResponseError: # remove the group we just tried and try again on a different one pingdom_security_groups.pop(0) if not addToSecurityGroup(pingdom_security_groups, ip): return False return True
def publish_metrics(self, accessProperties=None, verifyonly=False): metrics = MetricsPublisher.get_metrics(self) source_name = self.instance_name self.post_metrics(source_name, metrics, accessProperties, verifyonly) localhostname = getLocalHostname(True) clusterstate = self.getClusterStateForHost(localhostname) debug(clusterstate) hosts = clusterstate["leaders"] for host in hosts: metrics_to_publish = self.populateMetrics(localhostname, host) self.post_metrics(self.instance_name + "." + host["core"], metrics_to_publish, accessProperties, verifyonly) hosts = clusterstate["replicas"] for host in hosts: metrics_to_publish = self.populateMetrics(localhostname, host, True) self.post_metrics(self.instance_name + "." + host["core"], metrics_to_publish, accessProperties, verifyonly)
def getClusterStateForHost(self, localhostname): cluster_state_for_host = {"leaders": [], "replicas": []} clusterstate = requests.get("http://localhost:8080/solr/zookeeper?detail=true&path=/clusterstate.json").json() cloudstate = json.loads(clusterstate["znode"]["data"]) for index in cloudstate: print index for shard in cloudstate[index]["shards"]: for replica in cloudstate[index]["shards"][shard]["replicas"]: replica_details = cloudstate[index]["shards"][shard]["replicas"][replica] if localhostname in replica_details["base_url"]: if "leader" in replica_details and replica_details["leader"] == "true": debug("Adding a leader: " + str(replica_details)) cluster_state_for_host["leaders"].append(replica_details) else: debug("Adding a replica: " + str(replica_details)) cluster_state_for_host["replicas"].append(replica_details) return cluster_state_for_host
def post_metrics(self, metrics_source, metrics, accessProperties, verifyonly): if accessProperties is None or "librato_username" not in accessProperties or "librato_api_key" not in accessProperties: print "ERROR - unable to publish metrics without proper authentication credentials" return False request = { "source": metrics_source, "measure_time": self.measure_time, "counters": [], "gauges": [] } for metric_name in metrics: metric = metrics[metric_name] if metric["type"] == MetricsPublisher.COUNTER: request["counters"].append({"name": metric_name, "value": metric["value"]}) if metric["type"] == MetricsPublisher.GAUGE: request["gauges"].append({"name": metric_name, "value": metric["value"]}) debug(request) url = "https://metrics-api.librato.com/v1/metrics" debug(url) if verifyonly is False: response = requests.post(url, data=json.dumps(request), headers={'content-type': 'application/json'}, auth=(accessProperties["librato_username"], accessProperties["librato_api_key"]) ) debug(response)
def populateMetrics(self, localhostname, host, replica=False): solr_stats_url = host["base_url"].replace(localhostname, "localhost") + "/" + host["core"] + "/admin/mbeans?cat=QUERYHANDLER&key=/select&stats=true&wt=json" debug(solr_stats_url) stats = requests.get(solr_stats_url).json() #requests_per_second = stats["solr-mbeans"][1]["/select"]["stats"]["avgRequestsPerSecond"] avg_time_per_request = stats["solr-mbeans"][1]["/select"]["stats"]["avgTimePerRequest"] median_time_per_request = stats["solr-mbeans"][1]["/select"]["stats"]["medianRequestTime"] pct95_time_per_request = stats["solr-mbeans"][1]["/select"]["stats"]["95thPcRequestTime"] pct75_time_per_request = stats["solr-mbeans"][1]["/select"]["stats"]["75thPcRequestTime"] min5_req_rate_per_second = stats["solr-mbeans"][1]["/select"]["stats"]["5minRateReqsPerSecond"] metrics = {} if not replica: metrics[MetricsPublisher.METRICS_SOLRCLOUD_LEADER_AVG_REQUEST_TIME] = MetricsPublisher.wrap_value_type(self, avg_time_per_request, MetricsPublisher.GAUGE) metrics[MetricsPublisher.METRICS_SOLRCLOUD_LEADER_MEDIAN_REQUEST_TIME] = MetricsPublisher.wrap_value_type(self, median_time_per_request, MetricsPublisher.GAUGE) metrics[MetricsPublisher.METRICS_SOLRCLOUD_LEADER_95PCT_REQUEST_TIME] = MetricsPublisher.wrap_value_type(self, pct95_time_per_request, MetricsPublisher.GAUGE) metrics[MetricsPublisher.METRICS_SOLRCLOUD_LEADER_75PCT_REQUEST_TIME] = MetricsPublisher.wrap_value_type(self, pct75_time_per_request, MetricsPublisher.GAUGE) metrics[MetricsPublisher.METRICS_SOLRCLOUD_LEADER_5MIN_REQ_RATE] = MetricsPublisher.wrap_value_type(self, min5_req_rate_per_second, MetricsPublisher.GAUGE) else: metrics[MetricsPublisher.METRICS_SOLRCLOUD_REPLICA_AVG_REQUEST_TIME] = MetricsPublisher.wrap_value_type(self, avg_time_per_request, MetricsPublisher.GAUGE) metrics[MetricsPublisher.METRICS_SOLRCLOUD_REPLICA_MEDIAN_REQUEST_TIME] = MetricsPublisher.wrap_value_type(self, median_time_per_request, MetricsPublisher.GAUGE) metrics[MetricsPublisher.METRICS_SOLRCLOUD_REPLICA_95PCT_REQUEST_TIME] = MetricsPublisher.wrap_value_type(self, pct95_time_per_request, MetricsPublisher.GAUGE) metrics[MetricsPublisher.METRICS_SOLRCLOUD_REPLICA_75PCT_REQUEST_TIME] = MetricsPublisher.wrap_value_type(self, pct75_time_per_request, MetricsPublisher.GAUGE) metrics[MetricsPublisher.METRICS_SOLRCLOUD_REPLICA_5MIN_REQ_RATE] = MetricsPublisher.wrap_value_type(self, min5_req_rate_per_second, MetricsPublisher.GAUGE) return metrics
def get_metrics(self, metrics_to_publish={}): metrics_to_publish = MetricsPublisher.get_metrics(self, metrics_to_publish) data_mount_count = 0 log_mount_count = 0 for disk in psutil.disk_partitions(all=True): debug("inodemetrics.disk = " + str(disk)) if "data" in disk.mountpoint: data_mount_count += 1 debug("Marking disk mount %s as data drive" % disk.mountpoint) if "logs" in disk.mountpoint: log_mount_count += 1 debug("Marking disk mount %s as log drive" % disk.mountpoint) metrics_to_publish[MetricsPublisher.METRICS_INODE_DATA_MOUNTS_COUNT] = MetricsPublisher.wrap_value_type(self, data_mount_count, MetricsPublisher.GAUGE) metrics_to_publish[MetricsPublisher.METRICS_INODE_LOG_MOUNTS_COUNT] = MetricsPublisher.wrap_value_type(self, log_mount_count, MetricsPublisher.GAUGE) return metrics_to_publish
def post_metrics(self, metrics_source, metrics, accessProperties, verifyonly): if accessProperties is None or "librato_username" not in accessProperties or "librato_api_key" not in accessProperties: print "ERROR - unable to publish metrics without proper authentication credentials" return False request = { "source": metrics_source, "measure_time": self.measure_time, "counters": [], "gauges": [] } for metric_name in metrics: metric = metrics[metric_name] if metric["type"] == MetricsPublisher.COUNTER: request["counters"].append({ "name": metric_name, "value": metric["value"] }) if metric["type"] == MetricsPublisher.GAUGE: request["gauges"].append({ "name": metric_name, "value": metric["value"] }) debug(request) url = "https://metrics-api.librato.com/v1/metrics" debug(url) if verifyonly is False: response = requests.post( url, data=json.dumps(request), headers={'content-type': 'application/json'}, auth=(accessProperties["librato_username"], accessProperties["librato_api_key"])) debug(response)
def get_metrics(self, metrics_to_publish={}): metrics_to_publish = MetricsPublisher.get_metrics( self, metrics_to_publish) data_mount_count = 0 log_mount_count = 0 for disk in psutil.disk_partitions(all=True): debug("inodemetrics.disk = " + str(disk)) if "data" in disk.mountpoint: data_mount_count += 1 debug("Marking disk mount %s as data drive" % disk.mountpoint) if "logs" in disk.mountpoint: log_mount_count += 1 debug("Marking disk mount %s as log drive" % disk.mountpoint) metrics_to_publish[ MetricsPublisher. METRICS_INODE_DATA_MOUNTS_COUNT] = MetricsPublisher.wrap_value_type( self, data_mount_count, MetricsPublisher.GAUGE) metrics_to_publish[ MetricsPublisher. METRICS_INODE_LOG_MOUNTS_COUNT] = MetricsPublisher.wrap_value_type( self, log_mount_count, MetricsPublisher.GAUGE) return metrics_to_publish
new_lists_table = "%s\nCustomer Id\tCustomer Name\tEmail\tList Id\tList name\tDate Created\tNumber of Records\n\n" % new_lists_title new_lists_table += "-" * len(new_lists_table) new_lists_table += "\n" new_customers_table_html = "<table><tr><th colspan=6>%s</th></tr><tr><th>Customer Id</th><th>Customer Name</th><th>Email</th><th>Date Created</th><th>Terms Acceptance</th><th>Email Verified</th></tr>" % new_customers_title new_lists_table_html = "<table><tr><th colspan=7>%s</th></tr><tr><th>Customer Id</th><th>Customer Name</th><th>Email</th><th>List Id</th><th>List name</th><th>Date Created</th><th>Number of Records</th></tr>" % new_lists_title # fill in the Message record and send it for customeridint, customer in users_and_lists.iteritems(): params = dict() params['customerId'] = "%s" % customeridint if "lists" in customer: for customerlistid, customerlist in customer["lists"].iteritems(): params['customerListId'] = "%s" % customerlist["listid"] try: debug(requester) listdetails = None try: listdetails = requester.request('getListInfo', params) except AttributeError as e: print("Unable to retrieve list with params %s" % str(params)) continue except UnicodeDecodeError as e: print( "Bad response received from the server for params %s" % str(params)) continue new_lists_table += "%s\t%s\t%s\t%s\t%s\t%s\t%s\n" % (
debug(url) response = requests.put(url, data=data, auth=auth) if response.status_code != 204: return False return True def getLibratoSourcesToUpdate(auth): sources = libratoGET("/sources", auth=auth) sourcesToUpdate = [] for source in sources: if source["display_name"] is None: sourcesToUpdate.append(source["name"]) return sourcesToUpdate accessProperties = getAccessPropertiesFromConfigService() debug(accessProperties) totalSourcesUpdated = 0 totalErrorsEncountered = 0 sourcesToUpdate = getLibratoSourcesToUpdate(auth=(accessProperties["librato_username"], accessProperties["librato_api_key"])) debug("Librato sources needing a display_name value: " + str(sourcesToUpdate)) conn = boto.ec2.connect_to_region(accessProperties["aws_regions"], aws_access_key_id=accessProperties["aws_access_key"], aws_secret_access_key=accessProperties["aws_secret_access_key"]) instances = conn.get_all_reservations(filters={"instance-state-name": "running"}) for reservation in instances: for instance in reservation.instances: if instance.id in sourcesToUpdate: debug("Need to update display_name of " + instance.id + " to " + instance.tags['Name']) if not libratoPOST("/sources/" + instance.id, data={"display_name": instance.tags["Name"]}): print("ERROR updating source " + instance.id) totalErrorsEncountered += 1
new_lists_table = "%s\nCustomer Id\tCustomer Name\tEmail\tList Id\tList name\tDate Created\tNumber of Records\n\n" % new_lists_title new_lists_table += "-" * len(new_lists_table) new_lists_table += "\n" new_customers_table_html = "<table><tr><th colspan=6>%s</th></tr><tr><th>Customer Id</th><th>Customer Name</th><th>Email</th><th>Date Created</th><th>Terms Acceptance</th><th>Email Verified</th></tr>" % new_customers_title new_lists_table_html = "<table><tr><th colspan=7>%s</th></tr><tr><th>Customer Id</th><th>Customer Name</th><th>Email</th><th>List Id</th><th>List name</th><th>Date Created</th><th>Number of Records</th></tr>" % new_lists_title # fill in the Message record and send it for customeridint, customer in users_and_lists.iteritems(): params = dict() params['customerId'] = "%s" % customeridint if "lists" in customer: for customerlistid, customerlist in customer["lists"].iteritems(): params['customerListId'] = "%s" % customerlist["listid"] try: debug(requester) listdetails = None try: listdetails = requester.request('getListInfo', params) except AttributeError as e: print("Unable to retrieve list with params %s" % str(params)) continue except UnicodeDecodeError as e: print("Bad response received from the server for params %s" % str(params)) continue new_lists_table += "%s\t%s\t%s\t%s\t%s\t%s\t%s\n" % (customer["id"], customer["name"], customer["email"], customerlist["listid"], customerlist["listname"], customerlist["listdatecreated"], listdetails["numRecords"]) new_lists_table_html += "<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>" % \ (customer["id"], customer["name"], customer["email"], customerlist["listid"], customerlist["listname"], customerlist["listdatecreated"], listdetails["numRecords"]) except UnicodeDecodeError as e:
import argparse from librato.metrics_publisher import MetricsPublisher from librato.inode_metrics_publisher import INodeMetricsPublisher from librato.solrcloud_metrics_publisher import SOLRCloudMetricsPublisher from utils.locallogging import debug from utils.ec2instancedetails import getInstanceId, getInstanceName from utils.accesskeys import getAccessPropertiesFromConfigService # Parse all arguments, display help if required parser = argparse.ArgumentParser(description="inome's metrics publisher for librato. Gathers all metrics from the specific metrics collector and publishes them all to librato") parser.add_argument("--inode-metrics", dest="inodemetrics", action="store_true", help="Computes and publishes all metrics for one of inome's 'inodes'") parser.add_argument("--solrcloud-metrics", dest="solrcloudmetrics", action="store_true", help="Computes and publishes all metrics for one of inome's solrcloud nodes") parser.add_argument("--verify", dest="verify", action="store_true", help="Prints out metrics but does not send to Librato - use prior to installing as a cron to ensure proper values are being returned") parser.add_argument("--dev-mode", dest="devmode", action="store_true", help="Whether to run in 'development' mode or not (hard-coded instance values)") args = parser.parse_args() debug(args) # Retrieve all access information from the configuration service (http://config.vs.intelius.com:8080/configuration) accessProperties = getAccessPropertiesFromConfigService() if accessProperties is None: print("UNABLE TO CONTINUE - NOT ABLE TO RETRIEVE CONFIGURATION DETAILS") exit(1) # connect to ec2 and pull down the instance name based on the instance id we are running from conn = boto.ec2.connect_to_region(accessProperties["aws_regions"], aws_access_key_id=accessProperties["aws_access_key"], aws_secret_access_key=accessProperties["aws_secret_access_key"]) instanceid = getInstanceId(args.devmode) if instanceid is None: print("UNABLE TO CONTINUE - NOT ABLE TO IDENTIFY INSTANCE RUNNING") exit(1) instancename = "%s-%s" % (getInstanceName(ec2Connection=conn, instance_id=instanceid, devmode=args.devmode), instanceid)
# group to attempt to add this ip to pingdom_group = pingdom_security_groups[0] try: debug("Attempting to add " + ip + " to " + pingdom_group.name) conn.authorize_security_group(group_id=pingdom_group.id, ip_protocol="tcp", from_port=80, to_port=80, cidr_ip="%s/32" % ip) except boto.exception.EC2ResponseError: # remove the group we just tried and try again on a different one pingdom_security_groups.pop(0) if not addToSecurityGroup(pingdom_security_groups, ip): return False return True accessProperties = getAccessPropertiesFromConfigService() debug(accessProperties) toEncode = bytes(accessProperties["pingdom_username"] + ":" + accessProperties["pingdom_password"], "UTF-8") pingdomAuthHeader = base64.b64encode(toEncode) pingdomAuthHeader = str(pingdomAuthHeader)[2:][:-1] headers = {"Authorization": "Basic " + pingdomAuthHeader, "App-Key": accessProperties["pingdom_api_key"]} debug(headers) probes = requests.get("https://api.pingdom.com/api/2.0/probes", headers=headers).json() debug(probes) conn = boto.ec2.connect_to_region(accessProperties["aws_regions"], aws_access_key_id=accessProperties["aws_access_key"], aws_secret_access_key=accessProperties["aws_secret_access_key"]) all_security_groups = conn.get_all_security_groups() pingdom_security_groups = [] rule_cidr_map = {} for security_group in all_security_groups: if ("pingdom" in security_group.name):