def get_basic_stats(endpoints):
graphs = ["volume", "latency", "errors", "data"]
rs = []
for endpoint in endpoints:
params = {
"params": {
"appId": endpoint['appId'],
"filters": {
"filterName": "0",
"filterValue": "0"
},
"graph": "latency",
"duration": 60
}
}
if 'domain' in endpoint:
params['params']['filters']['domain'] = endpoint['domain']
if 'service' in endpoint:
params['params']['filters']['service'] = endpoint['service']
if 'endpoint' in endpoint:
params['params']['filters']['endpoint'] = endpoint['endpoint']
for graph in graphs:
params["params"]["graph"] = graph
r = grequests.post(
url="https://developers.crittercism.com/v1/apm/graph",
headers={
"Authorization": "Bearer " + config['token'],
"Content-Type": "application/json",
},
data=json.dumps(params))
rs.append(r)
my_stats["rest-api-requests"] += len(rs)
print("{me}: Sending {count} graph requests to Crittercism".format(
me=me, count=len(rs)),
file=sys.stderr)
responses = grequests.map(rs, size=MAX_CONCURRENT_API_CALLS)
for response in responses:
(ok, data) = cccommon.check_apm_api_response(response)
if not ok:
my_stats["rest-api-failures"] += 1
url = response.request.url
print("{me}: {error} for {url}".format(me=me, error=data, url=url),
file=sys.stderr)
continue
my_stats["rest-api-responses"] += 1
start = dateutil.parser.parse(data['data']['start'])
end = dateutil.parser.parse(data['data']['end'])
interval = data['data']['interval']
appId = data['params']['appId']
mastertags = basic_tags_for_app(appId)
metricprefix = "crittercism.service."
filters = data['params']['filters']
if 'domain' in filters:
mastertags += " domain={domain}".format(
domain=cccommon.clean_for_opentsdb(filters['domain']))
if 'service' in filters:
mastertags += " service={service}".format(
service=filters['service'])
if 'endpoint' in filters:
metricprefix = "crittercism.endpoint."
mastertags += " endpoint={endpoint}".format(
endpoint=cccommon.clean_for_opentsdb(filters['endpoint']))
for series in data["data"]["series"]:
print_points(start, interval, metricprefix + series["name"],
mastertags, series["points"])
def get_error_detail(endpoints):
rs = []
for endpoint in endpoints:
params = {
"params": {
"appId": endpoint['appId'],
"filters": {
"filterName": "0",
"filterValue": "0",
},
"duration": 60
}
}
if 'domain' in endpoint:
params['params']['filters']['domain'] = endpoint['domain']
if 'service' in endpoint:
params['params']['filters']['service'] = endpoint['service']
if 'endpoint' in endpoint:
params['params']['filters']['endpoint'] = endpoint['endpoint']
r = grequests.post(
url="https://developers.crittercism.com/v1/apm/errordetail",
headers={
"Authorization": "Bearer " + config['token'],
"Content-Type": "application/json",
},
data=json.dumps(params))
rs.append(r)
my_stats["rest-api-requests"] += len(rs)
print("{me}: Sending {count} error detail requests to Crittercism".format(
me=me, count=len(rs)),
file=sys.stderr)
responses = grequests.map(rs, size=MAX_CONCURRENT_API_CALLS)
for response in responses:
(ok, data) = cccommon.check_apm_api_response(response)
if not ok:
my_stats["rest-api-failures"] += 1
url = response.request.url
print("{me}: {error} for {url}".format(me=me, error=data, url=url),
file=sys.stderr)
continue
my_stats["rest-api-responses"] += 1
start = dateutil.parser.parse(data['data']['start'])
end = dateutil.parser.parse(data['data']['end'])
interval = data['data']['interval']
appId = data['params']['appId']
mastertags = basic_tags_for_app(appId)
metricprefix = "crittercism.service."
filters = data['params']['filters']
if 'domain' in filters:
mastertags += " domain={domain}".format(
domain=cccommon.clean_for_opentsdb(filters['domain']))
if 'service' in filters:
mastertags += " service={service}".format(
service=filters['service'])
if 'endpoint' in filters:
metricprefix = "crittercism.endpoint."
mastertags += " endpoint={endpoint}".format(
endpoint=cccommon.clean_for_opentsdb(filters['endpoint']))
metric = metricprefix + "errors.type"
serieslookup = {}
for prefix, table in data['data']['errorTables'].iteritems():
for code, value in table['mapping'].iteritems():
key = prefix + ":" + code
mapping = cccommon.clean_for_opentsdb(value)
if key.startswith('h:'):
mapping = "{code}-{mapping}".format(code=code,
mapping=mapping)
serieslookup[key] = mapping
for series in data['data']['series']:
print_points(
start, interval, metric, mastertags +
" type={type}".format(type=serieslookup[series['name']]),
series['points'])
def get_latency_detail(endpoints):
rs = []
for endpoint in endpoints:
params = {
"params": {
"appId": endpoint['appId'],
"filters": {
"filterName": "0",
"filterValue": "0"
},
"graph": "latency",
"duration": 60
}
}
if 'domain' in endpoint:
params['params']['filters']['domain'] = endpoint['domain']
if 'service' in endpoint:
params['params']['filters']['service'] = endpoint['service']
if 'endpoint' in endpoint:
params['params']['filters']['endpoint'] = endpoint['endpoint']
r = grequests.post(
url="https://developers.crittercism.com/v1/apm/graphdetail",
headers={
"Authorization": "Bearer " + config['token'],
"Content-Type": "application/json",
},
data=json.dumps(params))
rs.append(r)
if len(rs) == 0:
# nothing to do
return
my_stats["rest-api-requests"] += len(rs)
print(
"{me}: Sending {count} latency detail requests to Crittercism".format(
me=me, count=len(rs)),
file=sys.stderr)
responses = grequests.map(rs, size=MAX_CONCURRENT_API_CALLS)
for response in responses:
(ok, data) = cccommon.check_apm_api_response(response)
if not ok:
my_stats["rest-api-failures"] += 1
url = response.request.url
print("{me}: {error} for {url}".format(me=me, error=data, url=url),
file=sys.stderr)
continue
my_stats["rest-api-responses"] += 1
start = dateutil.parser.parse(data['data']['start'])
end = dateutil.parser.parse(data['data']['end'])
interval = data['data']['interval']
masterseriespoints = data['data']['series'][0]['points']
deviceseries = data['data']['series'][1]['subseries']
carrierseries = data['data']['series'][2]['subseries']
appId = data['params']['appId']
mastertags = basic_tags_for_app(appId)
metricprefix = "crittercism.service."
filters = data['params']['filters']
if 'domain' in filters:
mastertags += " domain={domain}".format(
domain=cccommon.clean_for_opentsdb(filters['domain']))
if 'service' in filters:
mastertags += " service={service}".format(
service=filters['service'])
if 'endpoint' in filters:
metricprefix = "crittercism.endpoint."
mastertags += " endpoint={endpoint}".format(
endpoint=cccommon.clean_for_opentsdb(filters['endpoint']))
deviceseries.sort(key=lambda series: sum(series['volume']),
reverse=True)
carrierseries.sort(key=lambda series: sum(series['volume']),
reverse=True)
for series in deviceseries[:LATENCY_DETAILS_TRACKED_DEVICES]:
clean_device_name = cccommon.clean_for_opentsdb(series['name'])
if not clean_device_name or clean_device_name == "":
clean_device_name = cccommon.clean_for_opentsdb(
series['label'])
num_latency_points = sum([1 for x in series['volume'] if x > 0])
avg_latency = sum(series['points']) / num_latency_points
total_volume = sum(series['volume'])
print_point(
metricprefix + "latency.device",
int(time.mktime(start.timetuple())), avg_latency, mastertags +
" device={device}".format(device=clean_device_name))
print_point(
metricprefix + "volume.device",
int(time.mktime(start.timetuple())), total_volume, mastertags +
" device={device}".format(device=clean_device_name))
for series in carrierseries[:LATENCY_DETAILS_TRACKED_CARRIERS]:
clean_carrier_name = cccommon.clean_for_opentsdb(series['name'])
if clean_carrier_name == "":
clean_carrier_name = "Unknown"
num_latency_points = sum([1 for x in series['volume'] if x > 0])
avg_latency = sum(series['points']) / num_latency_points
total_volume = sum(series['volume'])
print_point(
metricprefix + "latency.carrier",
int(time.mktime(start.timetuple())), avg_latency, mastertags +
" carrier={carrier}".format(carrier=clean_carrier_name))
print_point(
metricprefix + "volume.carrier",
int(time.mktime(start.timetuple())), total_volume, mastertags +
" carrier={carrier}".format(carrier=clean_carrier_name))
def get_latency_detail(endpoints):
rs = []
for endpoint in endpoints:
params = {
"params": {
"appId": endpoint['appId'],
"filters": {
"filterName": "0",
"filterValue": "0"
},
"graph": "latency",
"duration": 60
}
}
if 'domain' in endpoint:
params['params']['filters']['domain'] = endpoint['domain']
if 'service' in endpoint:
params['params']['filters']['service'] = endpoint['service']
if 'endpoint' in endpoint:
params['params']['filters']['endpoint'] = endpoint['endpoint']
r = grequests.post(
url="https://developers.crittercism.com/v1/apm/graphdetail",
headers={
"Authorization": "Bearer " + config['token'],
"Content-Type": "application/json",
},
data=json.dumps(params)
)
rs.append(r)
if len(rs) == 0:
# nothing to do
return
my_stats["rest-api-requests"] += len(rs)
print("{me}: Sending {count} latency detail requests to Crittercism".format(me=me, count=len(rs)), file=sys.stderr)
responses = grequests.map(rs, size=MAX_CONCURRENT_API_CALLS)
for response in responses:
(ok, data) = cccommon.check_apm_api_response(response)
if not ok:
my_stats["rest-api-failures"] += 1
url = response.request.url
print("{me}: {error} for {url}".format(me=me, error=data, url=url), file=sys.stderr)
continue
my_stats["rest-api-responses"] += 1
start = dateutil.parser.parse(data['data']['start'])
end = dateutil.parser.parse(data['data']['end'])
interval = data['data']['interval']
masterseriespoints = data['data']['series'][0]['points'];
deviceseries = data['data']['series'][1]['subseries'];
carrierseries = data['data']['series'][2]['subseries'];
appId = data['params']['appId']
mastertags = basic_tags_for_app(appId)
metricprefix = "crittercism.service."
filters = data['params']['filters']
if 'domain' in filters:
mastertags += " domain={domain}".format(domain=cccommon.clean_for_opentsdb(filters['domain']))
if 'service' in filters:
mastertags += " service={service}".format(service=filters['service'])
if 'endpoint' in filters:
metricprefix = "crittercism.endpoint."
mastertags += " endpoint={endpoint}".format(endpoint=cccommon.clean_for_opentsdb(filters['endpoint']))
deviceseries.sort(key=lambda series: sum(series['volume']), reverse=True)
carrierseries.sort(key=lambda series: sum(series['volume']), reverse=True)
for series in deviceseries[:LATENCY_DETAILS_TRACKED_DEVICES]:
clean_device_name = cccommon.clean_for_opentsdb(series['name'])
if not clean_device_name or clean_device_name == "":
clean_device_name = cccommon.clean_for_opentsdb(series['label'])
num_latency_points = sum([1 for x in series['volume'] if x > 0])
avg_latency = sum(series['points']) / num_latency_points
total_volume = sum(series['volume'])
print_point(metricprefix + "latency.device", int(time.mktime(start.timetuple())), avg_latency, mastertags + " device={device}".format(device=clean_device_name))
print_point(metricprefix + "volume.device", int(time.mktime(start.timetuple())), total_volume, mastertags + " device={device}".format(device=clean_device_name))
for series in carrierseries[:LATENCY_DETAILS_TRACKED_CARRIERS]:
clean_carrier_name = cccommon.clean_for_opentsdb(series['name'])
if clean_carrier_name == "":
clean_carrier_name = "Unknown"
num_latency_points = sum([1 for x in series['volume'] if x > 0])
avg_latency = sum(series['points']) / num_latency_points
total_volume = sum(series['volume'])
print_point(metricprefix + "latency.carrier", int(time.mktime(start.timetuple())), avg_latency, mastertags + " carrier={carrier}".format(carrier=clean_carrier_name))
print_point(metricprefix + "volume.carrier", int(time.mktime(start.timetuple())), total_volume, mastertags + " carrier={carrier}".format(carrier=clean_carrier_name))
def get_basic_stats(endpoints):
graphs = ["volume", "latency", "errors", "data"]
rs = []
for endpoint in endpoints:
params = {
"params": {
"appId": endpoint['appId'],
"filters": {
"filterName": "0",
"filterValue": "0"
},
"graph": "latency",
"duration": 60
}
}
if 'domain' in endpoint:
params['params']['filters']['domain'] = endpoint['domain']
if 'service' in endpoint:
params['params']['filters']['service'] = endpoint['service']
if 'endpoint' in endpoint:
params['params']['filters']['endpoint'] = endpoint['endpoint']
for graph in graphs:
params["params"]["graph"] = graph
r = grequests.post(
url="https://developers.crittercism.com/v1/apm/graph",
headers={
"Authorization": "Bearer " + config['token'],
"Content-Type": "application/json",
},
data=json.dumps(params)
)
rs.append(r)
my_stats["rest-api-requests"] += len(rs)
print("{me}: Sending {count} graph requests to Crittercism".format(me=me, count=len(rs)), file=sys.stderr)
responses = grequests.map(rs, size=MAX_CONCURRENT_API_CALLS)
for response in responses:
(ok, data) = cccommon.check_apm_api_response(response)
if not ok:
my_stats["rest-api-failures"] += 1
url = response.request.url
print("{me}: {error} for {url}".format(me=me, error=data, url=url), file=sys.stderr)
continue
my_stats["rest-api-responses"] += 1
start = dateutil.parser.parse(data['data']['start'])
end = dateutil.parser.parse(data['data']['end'])
interval = data['data']['interval']
appId = data['params']['appId']
mastertags = basic_tags_for_app(appId)
metricprefix = "crittercism.service."
filters = data['params']['filters']
if 'domain' in filters:
mastertags += " domain={domain}".format(domain=cccommon.clean_for_opentsdb(filters['domain']))
if 'service' in filters:
mastertags += " service={service}".format(service=filters['service'])
if 'endpoint' in filters:
metricprefix = "crittercism.endpoint."
mastertags += " endpoint={endpoint}".format(endpoint=cccommon.clean_for_opentsdb(filters['endpoint']))
for series in data["data"]["series"]:
print_points(start, interval, metricprefix + series["name"], mastertags, series["points"])
def get_error_detail(endpoints):
rs = []
for endpoint in endpoints:
params = {
"params": {
"appId": endpoint['appId'],
"filters": {
"filterName": "0",
"filterValue": "0",
},
"duration": 60
}
}
if 'domain' in endpoint:
params['params']['filters']['domain'] = endpoint['domain']
if 'service' in endpoint:
params['params']['filters']['service'] = endpoint['service']
if 'endpoint' in endpoint:
params['params']['filters']['endpoint'] = endpoint['endpoint']
r = grequests.post(
url="https://developers.crittercism.com/v1/apm/errordetail",
headers={
"Authorization": "Bearer " + config['token'],
"Content-Type": "application/json",
},
data=json.dumps(params)
)
rs.append(r)
my_stats["rest-api-requests"] += len(rs)
print("{me}: Sending {count} error detail requests to Crittercism".format(me=me, count=len(rs)), file=sys.stderr)
responses = grequests.map(rs, size=MAX_CONCURRENT_API_CALLS)
for response in responses:
(ok, data) = cccommon.check_apm_api_response(response)
if not ok:
my_stats["rest-api-failures"] += 1
url = response.request.url
print("{me}: {error} for {url}".format(me=me, error=data, url=url), file=sys.stderr)
continue
my_stats["rest-api-responses"] += 1
start = dateutil.parser.parse(data['data']['start'])
end = dateutil.parser.parse(data['data']['end'])
interval = data['data']['interval']
appId = data['params']['appId']
mastertags = basic_tags_for_app(appId)
metricprefix = "crittercism.service."
filters = data['params']['filters']
if 'domain' in filters:
mastertags += " domain={domain}".format(domain=cccommon.clean_for_opentsdb(filters['domain']))
if 'service' in filters:
mastertags += " service={service}".format(service=filters['service'])
if 'endpoint' in filters:
metricprefix = "crittercism.endpoint."
mastertags += " endpoint={endpoint}".format(endpoint=cccommon.clean_for_opentsdb(filters['endpoint']))
metric = metricprefix + "errors.type"
serieslookup = {}
for prefix, table in data['data']['errorTables'].iteritems():
for code, value in table['mapping'].iteritems():
key = prefix + ":" + code
mapping = cccommon.clean_for_opentsdb(value)
if key.startswith('h:'):
mapping = "{code}-{mapping}".format(code=code, mapping=mapping)
serieslookup[key] = mapping
for series in data['data']['series']:
print_points(start, interval, metric, mastertags + " type={type}".format(type=serieslookup[series['name']]), series['points'])