def libratoPOST(url, data, auth=None):
url = "https://metrics-api.librato.com/v1" + url
debug(url)
response = requests.put(url, data=data, auth=auth)
if response.status_code != 204:
return False
return True
def get_key_from_configservice(env="DEV", api_key="", key="", default_value=None):
global USE_DB
if not USE_DB:
url = "http://config.vs.intelius.com:8080/configuration/1.0/get?env=%s&key=%s" % (env, key)
if len(api_key) > 0:
url += "&callerid=%s" % api_key
debug(url)
try:
data = requests.get(url)
datajson = data.json()
if data.status_code == 200 and "value" in datajson:
return datajson["value"]
elif "value" not in datajson:
return default_value
except Exception as e:
USE_DB = True
if USE_DB:
# try to read directly from DB (config service in-proc)
db_host = os.environ.get('INSIGHTS_CONFIGDB_HOST')
if db_host is None:
print("UNABLE TO RETRIEVE DATA FROM THE CONFIGURATION SERVICE")
else:
db_user = os.environ.get('INSIGHTS_CONFIGDB_USER')
db_pass = os.environ.get('INSIGHTS_CONFIGDB_PASS')
db = MySQLdb.connect(host=db_host, user=db_user, passwd=db_pass)
cur = db.cursor()
cur.execute("SELECT Value FROM Inome.Configuration WHERE Env = '%s' AND `Key` = '%s'" % (env, key))
for row in cur.fetchall():
return row[0]
return default_value
def libratoGET(url, keyname=None, offset=0, auth=None):
# If no keyname passed in, use the substring of url starting at index 1
if keyname is None:
keyname = url[1:]
fullurl = "https://metrics-api.librato.com/v1" + url + "?length=" + str(DEFAULT_LIBRATO_LENGTH) + "&offset=" + str(offset)
debug(fullurl)
data = requests.get(fullurl, auth=auth)
returnArray = []
if data.status_code == 200:
datajson = data.json()
debug("Length = " + str(datajson["query"]["length"]) + ", found = " + str(datajson["query"]["found"]) + ", total=" + str(datajson["query"]["total"]) + ", offset=" + str(datajson["query"]["offset"]))
returnArray.extend(datajson[keyname])
if (datajson["query"]["length"] + datajson["query"]["offset"]) < datajson["query"]["found"]:
returnArray.extend(libratoGET(url, keyname=keyname, offset=(datajson["query"]["offset"] + DEFAULT_LIBRATO_LENGTH), auth=auth))
return returnArray
return None
def addToSecurityGroup(pingdom_security_groups, ip):
# length has to be at least 1
if len(pingdom_security_groups) == 0:
return False
# group to attempt to add this ip to
pingdom_group = pingdom_security_groups[0]
try:
debug("Attempting to add " + ip + " to " + pingdom_group.name)
conn.authorize_security_group(group_id=pingdom_group.id, ip_protocol="tcp", from_port=80, to_port=80,
cidr_ip="%s/32" % ip)
except boto.exception.EC2ResponseError:
# remove the group we just tried and try again on a different one
pingdom_security_groups.pop(0)
if not addToSecurityGroup(pingdom_security_groups, ip):
return False
return True
def publish_metrics(self, accessProperties=None, verifyonly=False):
metrics = MetricsPublisher.get_metrics(self)
source_name = self.instance_name
self.post_metrics(source_name, metrics, accessProperties, verifyonly)
localhostname = getLocalHostname(True)
clusterstate = self.getClusterStateForHost(localhostname)
debug(clusterstate)
hosts = clusterstate["leaders"]
for host in hosts:
metrics_to_publish = self.populateMetrics(localhostname, host)
self.post_metrics(self.instance_name + "." + host["core"], metrics_to_publish, accessProperties, verifyonly)
hosts = clusterstate["replicas"]
for host in hosts:
metrics_to_publish = self.populateMetrics(localhostname, host, True)
self.post_metrics(self.instance_name + "." + host["core"], metrics_to_publish, accessProperties, verifyonly)
def getClusterStateForHost(self, localhostname):
cluster_state_for_host = {"leaders": [], "replicas": []}
clusterstate = requests.get("http://localhost:8080/solr/zookeeper?detail=true&path=/clusterstate.json").json()
cloudstate = json.loads(clusterstate["znode"]["data"])
for index in cloudstate:
print index
for shard in cloudstate[index]["shards"]:
for replica in cloudstate[index]["shards"][shard]["replicas"]:
replica_details = cloudstate[index]["shards"][shard]["replicas"][replica]
if localhostname in replica_details["base_url"]:
if "leader" in replica_details and replica_details["leader"] == "true":
debug("Adding a leader: " + str(replica_details))
cluster_state_for_host["leaders"].append(replica_details)
else:
debug("Adding a replica: " + str(replica_details))
cluster_state_for_host["replicas"].append(replica_details)
return cluster_state_for_host
def post_metrics(self, metrics_source, metrics, accessProperties, verifyonly):
if accessProperties is None or "librato_username" not in accessProperties or "librato_api_key" not in accessProperties:
print "ERROR - unable to publish metrics without proper authentication credentials"
return False
request = {
"source": metrics_source,
"measure_time": self.measure_time,
"counters": [],
"gauges": []
}
for metric_name in metrics:
metric = metrics[metric_name]
if metric["type"] == MetricsPublisher.COUNTER:
request["counters"].append({"name": metric_name, "value": metric["value"]})
if metric["type"] == MetricsPublisher.GAUGE:
request["gauges"].append({"name": metric_name, "value": metric["value"]})
debug(request)
url = "https://metrics-api.librato.com/v1/metrics"
debug(url)
if verifyonly is False:
response = requests.post(url, data=json.dumps(request),
headers={'content-type': 'application/json'},
auth=(accessProperties["librato_username"], accessProperties["librato_api_key"])
)
debug(response)
def populateMetrics(self, localhostname, host, replica=False):
solr_stats_url = host["base_url"].replace(localhostname, "localhost") + "/" + host["core"] + "/admin/mbeans?cat=QUERYHANDLER&key=/select&stats=true&wt=json"
debug(solr_stats_url)
stats = requests.get(solr_stats_url).json()
#requests_per_second = stats["solr-mbeans"][1]["/select"]["stats"]["avgRequestsPerSecond"]
avg_time_per_request = stats["solr-mbeans"][1]["/select"]["stats"]["avgTimePerRequest"]
median_time_per_request = stats["solr-mbeans"][1]["/select"]["stats"]["medianRequestTime"]
pct95_time_per_request = stats["solr-mbeans"][1]["/select"]["stats"]["95thPcRequestTime"]
pct75_time_per_request = stats["solr-mbeans"][1]["/select"]["stats"]["75thPcRequestTime"]
min5_req_rate_per_second = stats["solr-mbeans"][1]["/select"]["stats"]["5minRateReqsPerSecond"]
metrics = {}
if not replica:
metrics[MetricsPublisher.METRICS_SOLRCLOUD_LEADER_AVG_REQUEST_TIME] = MetricsPublisher.wrap_value_type(self, avg_time_per_request, MetricsPublisher.GAUGE)
metrics[MetricsPublisher.METRICS_SOLRCLOUD_LEADER_MEDIAN_REQUEST_TIME] = MetricsPublisher.wrap_value_type(self, median_time_per_request, MetricsPublisher.GAUGE)
metrics[MetricsPublisher.METRICS_SOLRCLOUD_LEADER_95PCT_REQUEST_TIME] = MetricsPublisher.wrap_value_type(self, pct95_time_per_request, MetricsPublisher.GAUGE)
metrics[MetricsPublisher.METRICS_SOLRCLOUD_LEADER_75PCT_REQUEST_TIME] = MetricsPublisher.wrap_value_type(self, pct75_time_per_request, MetricsPublisher.GAUGE)
metrics[MetricsPublisher.METRICS_SOLRCLOUD_LEADER_5MIN_REQ_RATE] = MetricsPublisher.wrap_value_type(self, min5_req_rate_per_second, MetricsPublisher.GAUGE)
else:
metrics[MetricsPublisher.METRICS_SOLRCLOUD_REPLICA_AVG_REQUEST_TIME] = MetricsPublisher.wrap_value_type(self, avg_time_per_request, MetricsPublisher.GAUGE)
metrics[MetricsPublisher.METRICS_SOLRCLOUD_REPLICA_MEDIAN_REQUEST_TIME] = MetricsPublisher.wrap_value_type(self, median_time_per_request, MetricsPublisher.GAUGE)
metrics[MetricsPublisher.METRICS_SOLRCLOUD_REPLICA_95PCT_REQUEST_TIME] = MetricsPublisher.wrap_value_type(self, pct95_time_per_request, MetricsPublisher.GAUGE)
metrics[MetricsPublisher.METRICS_SOLRCLOUD_REPLICA_75PCT_REQUEST_TIME] = MetricsPublisher.wrap_value_type(self, pct75_time_per_request, MetricsPublisher.GAUGE)
metrics[MetricsPublisher.METRICS_SOLRCLOUD_REPLICA_5MIN_REQ_RATE] = MetricsPublisher.wrap_value_type(self, min5_req_rate_per_second, MetricsPublisher.GAUGE)
return metrics
def get_metrics(self, metrics_to_publish={}):
metrics_to_publish = MetricsPublisher.get_metrics(self, metrics_to_publish)
data_mount_count = 0
log_mount_count = 0
for disk in psutil.disk_partitions(all=True):
debug("inodemetrics.disk = " + str(disk))
if "data" in disk.mountpoint:
data_mount_count += 1
debug("Marking disk mount %s as data drive" % disk.mountpoint)
if "logs" in disk.mountpoint:
log_mount_count += 1
debug("Marking disk mount %s as log drive" % disk.mountpoint)
metrics_to_publish[MetricsPublisher.METRICS_INODE_DATA_MOUNTS_COUNT] = MetricsPublisher.wrap_value_type(self, data_mount_count, MetricsPublisher.GAUGE)
metrics_to_publish[MetricsPublisher.METRICS_INODE_LOG_MOUNTS_COUNT] = MetricsPublisher.wrap_value_type(self, log_mount_count, MetricsPublisher.GAUGE)
return metrics_to_publish
def post_metrics(self, metrics_source, metrics, accessProperties,
verifyonly):
if accessProperties is None or "librato_username" not in accessProperties or "librato_api_key" not in accessProperties:
print "ERROR - unable to publish metrics without proper authentication credentials"
return False
request = {
"source": metrics_source,
"measure_time": self.measure_time,
"counters": [],
"gauges": []
}
for metric_name in metrics:
metric = metrics[metric_name]
if metric["type"] == MetricsPublisher.COUNTER:
request["counters"].append({
"name": metric_name,
"value": metric["value"]
})
if metric["type"] == MetricsPublisher.GAUGE:
request["gauges"].append({
"name": metric_name,
"value": metric["value"]
})
debug(request)
url = "https://metrics-api.librato.com/v1/metrics"
debug(url)
if verifyonly is False:
response = requests.post(
url,
data=json.dumps(request),
headers={'content-type': 'application/json'},
auth=(accessProperties["librato_username"],
accessProperties["librato_api_key"]))
debug(response)
def get_metrics(self, metrics_to_publish={}):
metrics_to_publish = MetricsPublisher.get_metrics(
self, metrics_to_publish)
data_mount_count = 0
log_mount_count = 0
for disk in psutil.disk_partitions(all=True):
debug("inodemetrics.disk = " + str(disk))
if "data" in disk.mountpoint:
data_mount_count += 1
debug("Marking disk mount %s as data drive" % disk.mountpoint)
if "logs" in disk.mountpoint:
log_mount_count += 1
debug("Marking disk mount %s as log drive" % disk.mountpoint)
metrics_to_publish[
MetricsPublisher.
METRICS_INODE_DATA_MOUNTS_COUNT] = MetricsPublisher.wrap_value_type(
self, data_mount_count, MetricsPublisher.GAUGE)
metrics_to_publish[
MetricsPublisher.
METRICS_INODE_LOG_MOUNTS_COUNT] = MetricsPublisher.wrap_value_type(
self, log_mount_count, MetricsPublisher.GAUGE)
return metrics_to_publish
new_lists_table = "%s\nCustomer Id\tCustomer Name\tEmail\tList Id\tList name\tDate Created\tNumber of Records\n\n" % new_lists_title
new_lists_table += "-" * len(new_lists_table)
new_lists_table += "\n"
new_customers_table_html = "<table><tr><th colspan=6>%s</th></tr><tr><th>Customer Id</th><th>Customer Name</th><th>Email</th><th>Date Created</th><th>Terms Acceptance</th><th>Email Verified</th></tr>" % new_customers_title
new_lists_table_html = "<table><tr><th colspan=7>%s</th></tr><tr><th>Customer Id</th><th>Customer Name</th><th>Email</th><th>List Id</th><th>List name</th><th>Date Created</th><th>Number of Records</th></tr>" % new_lists_title
# fill in the Message record and send it
for customeridint, customer in users_and_lists.iteritems():
params = dict()
params['customerId'] = "%s" % customeridint
if "lists" in customer:
for customerlistid, customerlist in customer["lists"].iteritems():
params['customerListId'] = "%s" % customerlist["listid"]
try:
debug(requester)
listdetails = None
try:
listdetails = requester.request('getListInfo', params)
except AttributeError as e:
print("Unable to retrieve list with params %s" %
str(params))
continue
except UnicodeDecodeError as e:
print(
"Bad response received from the server for params %s"
% str(params))
continue
new_lists_table += "%s\t%s\t%s\t%s\t%s\t%s\t%s\n" % (
debug(url)
response = requests.put(url, data=data, auth=auth)
if response.status_code != 204:
return False
return True
def getLibratoSourcesToUpdate(auth):
sources = libratoGET("/sources", auth=auth)
sourcesToUpdate = []
for source in sources:
if source["display_name"] is None:
sourcesToUpdate.append(source["name"])
return sourcesToUpdate
accessProperties = getAccessPropertiesFromConfigService()
debug(accessProperties)
totalSourcesUpdated = 0
totalErrorsEncountered = 0
sourcesToUpdate = getLibratoSourcesToUpdate(auth=(accessProperties["librato_username"], accessProperties["librato_api_key"]))
debug("Librato sources needing a display_name value: " + str(sourcesToUpdate))
conn = boto.ec2.connect_to_region(accessProperties["aws_regions"], aws_access_key_id=accessProperties["aws_access_key"], aws_secret_access_key=accessProperties["aws_secret_access_key"])
instances = conn.get_all_reservations(filters={"instance-state-name": "running"})
for reservation in instances:
for instance in reservation.instances:
if instance.id in sourcesToUpdate:
debug("Need to update display_name of " + instance.id + " to " + instance.tags['Name'])
if not libratoPOST("/sources/" + instance.id, data={"display_name": instance.tags["Name"]}):
print("ERROR updating source " + instance.id)
totalErrorsEncountered += 1
new_lists_table = "%s\nCustomer Id\tCustomer Name\tEmail\tList Id\tList name\tDate Created\tNumber of Records\n\n" % new_lists_title
new_lists_table += "-" * len(new_lists_table)
new_lists_table += "\n"
new_customers_table_html = "<table><tr><th colspan=6>%s</th></tr><tr><th>Customer Id</th><th>Customer Name</th><th>Email</th><th>Date Created</th><th>Terms Acceptance</th><th>Email Verified</th></tr>" % new_customers_title
new_lists_table_html = "<table><tr><th colspan=7>%s</th></tr><tr><th>Customer Id</th><th>Customer Name</th><th>Email</th><th>List Id</th><th>List name</th><th>Date Created</th><th>Number of Records</th></tr>" % new_lists_title
# fill in the Message record and send it
for customeridint, customer in users_and_lists.iteritems():
params = dict()
params['customerId'] = "%s" % customeridint
if "lists" in customer:
for customerlistid, customerlist in customer["lists"].iteritems():
params['customerListId'] = "%s" % customerlist["listid"]
try:
debug(requester)
listdetails = None
try:
listdetails = requester.request('getListInfo', params)
except AttributeError as e:
print("Unable to retrieve list with params %s" % str(params))
continue
except UnicodeDecodeError as e:
print("Bad response received from the server for params %s" % str(params))
continue
new_lists_table += "%s\t%s\t%s\t%s\t%s\t%s\t%s\n" % (customer["id"], customer["name"], customer["email"], customerlist["listid"], customerlist["listname"], customerlist["listdatecreated"], listdetails["numRecords"])
new_lists_table_html += "<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>" % \
(customer["id"], customer["name"], customer["email"], customerlist["listid"], customerlist["listname"], customerlist["listdatecreated"], listdetails["numRecords"])
except UnicodeDecodeError as e:
import argparse
from librato.metrics_publisher import MetricsPublisher
from librato.inode_metrics_publisher import INodeMetricsPublisher
from librato.solrcloud_metrics_publisher import SOLRCloudMetricsPublisher
from utils.locallogging import debug
from utils.ec2instancedetails import getInstanceId, getInstanceName
from utils.accesskeys import getAccessPropertiesFromConfigService
# Parse all arguments, display help if required
parser = argparse.ArgumentParser(description="inome's metrics publisher for librato. Gathers all metrics from the specific metrics collector and publishes them all to librato")
parser.add_argument("--inode-metrics", dest="inodemetrics", action="store_true", help="Computes and publishes all metrics for one of inome's 'inodes'")
parser.add_argument("--solrcloud-metrics", dest="solrcloudmetrics", action="store_true", help="Computes and publishes all metrics for one of inome's solrcloud nodes")
parser.add_argument("--verify", dest="verify", action="store_true", help="Prints out metrics but does not send to Librato - use prior to installing as a cron to ensure proper values are being returned")
parser.add_argument("--dev-mode", dest="devmode", action="store_true", help="Whether to run in 'development' mode or not (hard-coded instance values)")
args = parser.parse_args()
debug(args)
# Retrieve all access information from the configuration service (http://config.vs.intelius.com:8080/configuration)
accessProperties = getAccessPropertiesFromConfigService()
if accessProperties is None:
print("UNABLE TO CONTINUE - NOT ABLE TO RETRIEVE CONFIGURATION DETAILS")
exit(1)
# connect to ec2 and pull down the instance name based on the instance id we are running from
conn = boto.ec2.connect_to_region(accessProperties["aws_regions"], aws_access_key_id=accessProperties["aws_access_key"], aws_secret_access_key=accessProperties["aws_secret_access_key"])
instanceid = getInstanceId(args.devmode)
if instanceid is None:
print("UNABLE TO CONTINUE - NOT ABLE TO IDENTIFY INSTANCE RUNNING")
exit(1)
instancename = "%s-%s" % (getInstanceName(ec2Connection=conn, instance_id=instanceid, devmode=args.devmode), instanceid)
# group to attempt to add this ip to
pingdom_group = pingdom_security_groups[0]
try:
debug("Attempting to add " + ip + " to " + pingdom_group.name)
conn.authorize_security_group(group_id=pingdom_group.id, ip_protocol="tcp", from_port=80, to_port=80,
cidr_ip="%s/32" % ip)
except boto.exception.EC2ResponseError:
# remove the group we just tried and try again on a different one
pingdom_security_groups.pop(0)
if not addToSecurityGroup(pingdom_security_groups, ip):
return False
return True
accessProperties = getAccessPropertiesFromConfigService()
debug(accessProperties)
toEncode = bytes(accessProperties["pingdom_username"] + ":" + accessProperties["pingdom_password"], "UTF-8")
pingdomAuthHeader = base64.b64encode(toEncode)
pingdomAuthHeader = str(pingdomAuthHeader)[2:][:-1]
headers = {"Authorization": "Basic " + pingdomAuthHeader, "App-Key": accessProperties["pingdom_api_key"]}
debug(headers)
probes = requests.get("https://api.pingdom.com/api/2.0/probes", headers=headers).json()
debug(probes)
conn = boto.ec2.connect_to_region(accessProperties["aws_regions"], aws_access_key_id=accessProperties["aws_access_key"], aws_secret_access_key=accessProperties["aws_secret_access_key"])
all_security_groups = conn.get_all_security_groups()
pingdom_security_groups = []
rule_cidr_map = {}
for security_group in all_security_groups:
if ("pingdom" in security_group.name):