def get_state(self, duration):
'''Run any necessary calculations on the data collected from the logs
and return a list of metric objects.'''
self.duration = duration
metrics = []
# loop all the collected timers
for time_name in self.times:
values = self.times[time_name]['values']
unit = self.times[time_name]['unit']
# add counter
if self.duration > 0:
metrics.append(
MetricObject(time_name + '.count',
len(values) / self.duration))
# calculate statistical values
metrics.append(
MetricObject(time_name + '.mean',
stats_helper.find_mean(values), unit))
metrics.append(
MetricObject(time_name + '.median',
stats_helper.find_median(values), unit))
metrics += [
MetricObject(
'%s.%sth_percentile' % (time_name, percentile),
stats_helper.find_percentile(values, int(percentile)),
unit) for percentile in self.percentiles
]
return metrics
def get_state(self, duration):
'''Run any necessary calculations on the data collected from the logs
and return a list of metric objects.'''
metrics = []
if duration > 0:
metrics += [
MetricObject(counter, self.counts[counter] / duration)
for counter in self.counts
]
for time_name in self.times:
values = self.times[time_name]['values']
unit = self.times[time_name]['unit']
metrics.append(
MetricObject(time_name + '.mean',
stats_helper.find_mean(values), unit))
metrics.append(
MetricObject(time_name + '.median',
stats_helper.find_median(values), unit))
metrics += [
MetricObject(
'%s.%sth_percentile' % (time_name, percentile),
stats_helper.find_percentile(values, int(percentile)),
unit) for percentile in self.percentiles
]
return metrics
def get_state(self, duration):
result = []
for tag, data in self.monitored_tags.iteritems():
result.append(MetricObject(data['name']+'_success', data['success'], 'Tag '+data['name']+': successes'))
result.append(MetricObject(data['name']+'_error', data['error'], 'Tag '+data['name']+': errors'))
return result
def get_state(self, duration):
result = []
for action, count in self.actions.iteritems():
result.append(MetricObject("action_"+action, count, "Action: "+action.upper()))
for protocol, count in self.protocols.iteritems():
result.append(MetricObject("proto_"+protocol, count, "Protocol: "+protocol.upper()))
return result
def get_state(self, duration):
'''Run any necessary calculations on the data collected from the logs
and return a list of metric objects.'''
self.duration = duration
# Return a list of metrics objects
return [
MetricObject('count', self.line_count, 'lines on log since last run', type='int'),
MetricObject('rate', (float(self.line_count) / float(self.duration)), 'lines per sec', type='float', slope='both'),
]
def test_get_metric_object(self):
should_be = MetricObject('int', 1, type='int32')
metric_object = self.json_logster.get_metric_object('int', 1)
self.assertEquals(should_be.name, metric_object.name)
self.assertEquals(should_be.value, metric_object.value)
self.assertEquals(should_be.type, metric_object.type)
should_be = MetricObject('bool', 'False', type='string')
metric_object = self.json_logster.get_metric_object('bool', False)
self.assertEquals(should_be.name, metric_object.name)
self.assertEquals(should_be.value, metric_object.value)
self.assertEquals(should_be.type, metric_object.type)
def get_state(self, duration):
'''Run any necessary calculations on the data collected from the logs
and return a list of metric objects.'''
self.duration = duration
# Return a list of metrics objects
return [
MetricObject("api." + SHORT_HOSTNAME + ".http_1xx", (self.http_1xx / self.duration), "Responses per sec"),
MetricObject("api." + SHORT_HOSTNAME + ".http_2xx", (self.http_2xx / self.duration), "Responses per sec"),
MetricObject("api." + SHORT_HOSTNAME + ".http_3xx", (self.http_3xx / self.duration), "Responses per sec"),
MetricObject("api." + SHORT_HOSTNAME + ".http_4xx", (self.http_4xx / self.duration), "Responses per sec"),
MetricObject("api." + SHORT_HOSTNAME + ".http_5xx", (self.http_5xx / self.duration), "Responses per sec"),
]
def get_state(self, duration):
'''Run any necessary calculations on the data collected from the logs
and return a list of metric objects.'''
self.duration = duration / 10.0
# Return a list of metrics objects
return [
MetricObject("notice", (self.notice / self.duration), "Logs per 10 sec"),
MetricObject("warn", (self.warn / self.duration), "Logs per 10 sec"),
MetricObject("error", (self.error / self.duration), "Logs per 10 sec"),
MetricObject("crit", (self.crit / self.duration), "Logs per 10 sec"),
MetricObject("other", (self.other / self.duration), "Logs per 10 sec"),
]
def get_state(self, duration):
'''Run any necessary calculations on the data collected from the logs
and return a list of metric objects.'''
self.duration = float(duration)
# Return a list of metrics objects
result = []
for key, occ in self.error_type.iteritems():
if self.raw_data:
metric = MetricObject(key, occ, "Responses")
else:
metric = MetricObject(key, (occ / self.duration), "Responses per sec")
result.append(metric)
return result
def read_nmon_line(self, line, nmonSection, nmonHeader, dt, path):
values = line.strip().split(',')
values.remove
if (len(values) != len(nmonHeader)):
logging.getLogger('logster').warn(
"length of values:{} and headers:{} does not mismatch".format(
len(values), len(nmonHeader)))
limit = len(values)
if (len(values) > len(nmonHeader)):
limit = len(nmonHeader)
logging.getLogger('logster').warn(
"length of values:{} > headers:{} - line: {}".format(
len(values), len(nmonHeader), line))
logging.getLogger('logster').warn(
"Printing only {} values\n".format(limit))
for i in range(2, limit):
name = "%s.%s.%s" % (path, nmonSection, nmonHeader[i])
timestamp = time.mktime(dt.timetuple())
if self.tz != 'local':
tz = pytz.timezone(self.tz)
timestamp = time.mktime(dt.astimezone(tz).timetuple())
m = MetricObject(name, values[i], timestamp=timestamp)
self.metrics += [m]
def get_state(self, duration):
'''Run any necessary calculations on the data collected from the logs
and return a list of metric objects.'''
self.duration = float(duration)
metrics = [MetricObject(level, (getattr(self, level) / self.duration)) for level in self.levels]
return metrics
def setUp(self):
self.metric = MetricObject("ERROR", 1, None)
self.metric.timestamp = strftime("%Y%m%dT%H:%M:00Z", gmtime(self.metric.timestamp))
self.cw = CloudWatch("key", "secretkey", self.metric)
self.cw.get_instance_id("myserverID").set_params().get_signed_url()
def get_state(self, duration):
self.duration = duration
return [
MetricObject(s.prop, (self.metrics[s.prop] / self.duration),
"Responses/sec") for s in status
]
def get_state(self, duration):
# Compile a list of alert counts
result = []
for level, count in self.levels.iteritems():
result.append(MetricObject("level_"+level, count, "Error count of level "+level))
return result
def get_state(self, duration):
'''Run any necessary calculations on the data collected from the logs
and return a list of metric objects.'''
self.duration = duration
# Return a list of metrics objects
return_array = [
MetricObject("http_1xx", (self.http_1xx / self.duration),
"Responses per sec"),
MetricObject("http_2xx", (self.http_2xx / self.duration),
"Responses per sec"),
MetricObject("http_3xx", (self.http_3xx / self.duration),
"Responses per sec"),
MetricObject("http_4xx", (self.http_4xx / self.duration),
"Responses per sec"),
MetricObject("http_5xx", (self.http_5xx / self.duration),
"Responses per sec"),
MetricObject("size", (self.size_transferred / self.duration),
"Size per sec")
]
for squid_code in self.squid_codes:
return_array.append(
MetricObject("squid_" + squid_code,
(self.squid_codes[squid_code] / self.duration),
"Squid code per sec"))
return return_array
def get_state(self, _duration):
return list(map(
lambda memory_measure: MetricObject(
'prod.system.%(memory_measure)s' % {'memory_measure': memory_measure},
getattr(self, memory_measure) if hasattr(self, memory_measure) else None,
'%(memory_measure)s in MB' % {'memory_measure': memory_measure},
),
MemoryParser.MEMORY_MEASURES
))
def get_state(self, duration):
'''Run any necessary calculations on the data collected from the logs
and return a list of metric objects.'''
self.duration = duration
metrics = [
MetricObject(('api.' + SHORT_HOSTNAME + '.' + level),
(getattr(self, level) / self.duration))
for level in self.levels
]
return metrics
def get_state(self, duration):
'''Run any necessary calculations on the data collected from the logs
and return a list of metric objects.'''
metrics = []
for status, count in self.combined_status.items():
metrics.append(
MetricObject('combined.status.{status}'.format(status=status),
count, 'Responses'))
for httpver, count in self.combined_httpver.items():
metrics.append(
MetricObject(
'combined.httpver.{httpver}'.format(httpver=httpver),
count, 'Responses'))
for firstsegment, statuses in self.status_stats.items():
for status, count in statuses.items():
metric_name = 'raw.{firstsegment}.status.{status}'.format(
firstsegment=firstsegment, status=status)
metrics.append(MetricObject(metric_name, count, 'Responses'))
return metrics
def get_state(self, duration):
'''Run any necessary calculations on the data collected from the logs
and return a list of metric objects.'''
self.duration = float(duration)
self.latencies.sort()
# Return a list of metrics objects
requests = [
MetricObject("requests.http_1xx", self.http_1xx, "Requests"),
MetricObject("requests.http_2xx", self.http_2xx, "Requests"),
MetricObject("requests.http_3xx", self.http_3xx, "Requests"),
MetricObject("requests.http_4xx", self.http_4xx, "Requests"),
MetricObject("requests.http_5xx", self.http_5xx, "Requests"),
MetricObject("latency.p50", self.percentile(0.5), "Miliseconds"),
MetricObject("latency.p90", self.percentile(0.9), "Miliseconds"),
MetricObject("latency.p99", self.percentile(0.99), "Miliseconds"),
MetricObject("latency.p999", self.percentile(0.999),
"Miliseconds"),
MetricObject("latency.max", self.max, "Miliseconds")
]
return requests
def get_metric_object(self, metric_name, metric_value):
'''
Default key_to_metric_object method.
Given a metric name and value, this returns
a MetricObject filled in. This default method
will attempt to infer MetricObject.type from the metric_value
type, but nothing more. If you need to set any particular
MetricObject parameters (such as slope, title, etc.),
you should override this method.
'''
metric_type = self.infer_metric_type(metric_value)
# make sure the metric value is properly a string.
if metric_type == 'string':
metric_value = str(metric_value)
return MetricObject(metric_name, metric_value, type=metric_type)
def get_state(self, duration):
metric_objects = []
for metric in self.metrics:
hostname = metric
total_time_taken = self.metrics[metric]['total_time_taken']
count = self.metrics[metric]['count']
total_time_taken_milliseconds = self._microseconds_to_milliseconds(
total_time_taken)
#Drop the decimals, not needed for our metrics
avg_time_taken_milliseconds = int(
(total_time_taken_milliseconds / count))
metric_objects.append(
MetricObject(hostname, avg_time_taken_milliseconds,
"apache_avg_resp_time"))
return metric_objects
def get_state(self, duration):
'''Run any necessary calculations on the data collected from the logs
and return a list of metric objects.'''
self.duration = duration
totalTxns = self.numSent + self.numBounced + self.numDeferred
pctDeferred = 0.0
pctSent = 0.0
pctBounced = 0.0
avgDelay = 0
mailTxnsSec = 0
mailSentSec = 0
#mind divide by zero situations
if (totalTxns > 0):
pctDeferred = (float(self.numDeferred) / totalTxns) * 100
pctSent = (float(self.numSent) / totalTxns) * 100
pctBounced = (float(self.numBounced) / totalTxns) * 100
if (self.numSent > 0):
avgDelay = self.totalDelay / self.numSent
if (self.duration > 0):
mailTxnsSec = totalTxns / self.duration
mailSentSec = self.numSent / self.duration
# Return a list of metrics objects
return [
MetricObject("numSent", self.numSent, "Total Sent"),
MetricObject("pctSent", pctSent, "Percentage Sent"),
MetricObject("numDeferred", self.numDeferred, "Total Deferred"),
MetricObject("pctDeferred", pctDeferred, "Percentage Deferred"),
MetricObject("numBounced", self.numBounced, "Total Bounced"),
MetricObject("pctBounced", pctBounced, "Percentage Bounced"),
MetricObject("mailTxnsSec", mailTxnsSec, "Transactions per sec"),
MetricObject("mailSentSec", mailSentSec, "Sends per sec"),
MetricObject("avgDelay", avgDelay, "Average Sending Delay"),
]
def get_state(self, duration):
'''Run any necessary calculations on the data collected from the logs
and return a list of metric objects.'''
self.duration = duration
metric_objects = []
for metric_name, metric_value in self.metrics.items():
if type(metric_value) == float:
metric_type = 'float'
elif type(metric_value) == int or type(metric_value) == long:
metric_type = 'int32'
else:
metric_type = 'string'
metric_value = str(metric_value)
metric_objects.append(MetricObject(metric_name, metric_value, type='int'))
return metric_objects
def get_state(self, duration):
self.write(self.header)
kmatches = self.keyword_match()
self.duration = duration
line_rate = float(self.line_count) / float(self.duration)
alarm_rate = float(self.alarm_count) / float(self.duration)
if self.alarm_count > self.alarmthreshold and self.emails:
self.write("Alarm count is {}".format(self.alarm_count))
self.notify(line_rate, kmatches)
return [
MetricObject('{}.alarm_rate'.format(self.name), (alarm_rate),
'lines per sec',
type='float',
slope='both'),
]
def parse_line(self, line):
'''This function should digest the contents of one line at a time, updating
object's state variables. Takes a single argument, the line to be parsed.'''
try:
# Apply regular expression to each line and extract interesting bits.
regMatch = self.reg.match(line)
if regMatch:
linebits = regMatch.groupdict()
test_path = linebits['test_path'].replace(".","").replace("/spec","spec").replace("/",".").replace(":",".")
run_time = float(linebits['run_time'])
graphite_path = "specjour." + linebits['computer_name'] + "." + test_path
self.tests.append(MetricObject(graphite_path, run_time))
else:
raise LogsterParsingException, "regmatch failed to match"
except Exception, e:
raise LogsterParsingException, "regmatch or contents failed with %s" % e
def get_state(self, duration):
'''Run any necessary calculations on the data collected from the logs
and return a list of metric objects.'''
self.duration = duration
# Return a list of metrics objects
return [
MetricObject("http_1xx", (self.http_1xx / self.duration),
"Responses per sec"),
MetricObject("http_2xx", (self.http_2xx / self.duration),
"Responses per sec"),
MetricObject("http_3xx", (self.http_3xx / self.duration),
"Responses per sec"),
MetricObject("http_4xx", (self.http_4xx / self.duration),
"Responses per sec"),
MetricObject("http_5xx", (self.http_5xx / self.duration),
"Responses per sec"),
MetricObject("request_rate", (self.request_count / self.duration),
"Request per sec"),
MetricObject("avg_request_time",
(self.request_time / self.request_count),
"Milliseconds per request"),
]
def get_state(self, duration):
'''Run any necessary calculations on the data collected from the logs
and return a list of metric objects.'''
self.duration = float(duration)
# Return a list of metrics objects
return [
MetricObject("total_requests",
(self.total_requests / self.duration),
"Requests per sec"),
MetricObject("response_time_avg",
(self.total_response_time / self.total_requests /
self.duration), "Average response time"),
MetricObject("http_1xx", (self.http_1xx / self.duration),
"Responses per sec"),
MetricObject("http_2xx", (self.http_2xx / self.duration),
"Responses per sec"),
MetricObject("http_3xx", (self.http_3xx / self.duration),
"Responses per sec"),
MetricObject("http_4xx", (self.http_4xx / self.duration),
"Responses per sec"),
MetricObject("http_5xx", (self.http_5xx / self.duration),
"Responses per sec"),
]
def get_state(self, duration):
self.duration = duration
metrics = []
for upstream, value in self.metrics['upstreams'].items():
metrics.append(
MetricObject(
self.upstream_metric(upstream, 'hits'),
value['hits'] / self.duration,
"upstream hits/sec"
)
)
metrics.append(
MetricObject(
self.upstream_metric(upstream, 'resp_mean_time_ms'),
value['ms'] / value['hits'],
"upstream mean resp time/hit"
)
)
metrics.append(
MetricObject(
self.upstream_metric(upstream, 'size'),
value['size'],
"total size"
)
)
metrics.append(
MetricObject(
self.upstream_metric(upstream, 'speed'),
value['size'] / self.duration,
"bytes/s"
)
)
metrics.append(
MetricObject(
self.upstream_metric(upstream, 'http_200.hits'),
value['http_200'] / self.duration,
"http 200/sec"
)
)
if value['http_200']:
metrics.append(
MetricObject(
self.upstream_metric(upstream, 'http_200.gzipped'),
value['http_200_gzipped'] / self.duration,
"http 200 gzipped/sec"
)
)
metrics.append(
MetricObject(
self.upstream_metric(upstream, 'http_200.resp_mean_time_ms'),
value['http_200_ms'] / value['http_200'],
"upstream mean resp time/hit"
)
)
if value['http_200_gzipped']:
metrics.append(
MetricObject(
self.upstream_metric(upstream, 'http_200.mean_gzip_ratio'),
value['gzip_ratio'] / value['http_200_gzipped'],
"Mean gzip ratio/gzipped hit"
)
)
return metrics
def test_full(self):
# init parser
p = PerformanceLogster('--mapping=tests/test.map')
samples = [
'{ "timestamp": "2014-10-10T06:30:32+00:00", "request": { "remote_addr": "192.168.56.1", "request": "GET /profile/me HTTP/1.1", "status": "200", "request_time": "0.406", "upstream_response_time": "0.406" } }',
'{ "timestamp": "2014-10-10T06:30:36+00:00", "request": { "remote_addr": "192.168.56.1", "request": "GET / HTTP/1.1", "status": "200", "request_time": "0.413", "upstream_response_time": "0.413" } }'
]
# parse samples
for sample in samples:
p.parse_line(sample)
expected = [
MetricObject('http_200.unknown.GET.upstreamTime.count', 1),
MetricObject('http_200.unknown.GET.upstreamTime.mean', 413.0,
'ms'),
MetricObject('http_200.unknown.GET.upstreamTime.median', 413.0,
'ms'),
MetricObject('http_200.unknown.GET.upstreamTime.90th_percentile',
413.0, 'ms'),
MetricObject('http_200.profile.GET.upstreamTime.count', 1),
MetricObject('http_200.profile.GET.upstreamTime.mean', 406.0,
'ms'),
MetricObject('http_200.profile.GET.upstreamTime.median', 406.0,
'ms'),
MetricObject('http_200.profile.GET.upstreamTime.90th_percentile',
406.0, 'ms'),
MetricObject('http_200.profile.GET.responseTime.count', 1),
MetricObject('http_200.profile.GET.responseTime.mean', 406.0,
'ms'),
MetricObject('http_200.profile.GET.responseTime.median', 406.0,
'ms'),
MetricObject('http_200.profile.GET.responseTime.90th_percentile',
406.0, 'ms'),
MetricObject('http_200.unknown.GET.responseTime.count', 1),
MetricObject('http_200.unknown.GET.responseTime.mean', 413.0,
'ms'),
MetricObject('http_200.unknown.GET.responseTime.median', 413.0,
'ms'),
MetricObject('http_200.unknown.GET.responseTime.90th_percentile',
413.0, 'ms'),
]
outputMetrics = p.get_state(1)
# validate number of metrics
self.assertEqual(len(expected), len(outputMetrics))
# compare all metrics objects
for i in range(0, len(expected) - 1):
self.assertEqual(expected[i].name, outputMetrics[i].name)
self.assertEqual(expected[i].value, outputMetrics[i].value)
self.assertEqual(expected[i].units, outputMetrics[i].units)