def get_state(self, duration): '''Run any necessary calculations on the data collected from the logs and return a list of metric objects.''' self.duration = duration metrics = [] # loop all the collected timers for time_name in self.times: values = self.times[time_name]['values'] unit = self.times[time_name]['unit'] # add counter if self.duration > 0: metrics.append( MetricObject(time_name + '.count', len(values) / self.duration)) # calculate statistical values metrics.append( MetricObject(time_name + '.mean', stats_helper.find_mean(values), unit)) metrics.append( MetricObject(time_name + '.median', stats_helper.find_median(values), unit)) metrics += [ MetricObject( '%s.%sth_percentile' % (time_name, percentile), stats_helper.find_percentile(values, int(percentile)), unit) for percentile in self.percentiles ] return metrics
def get_state(self, duration): '''Run any necessary calculations on the data collected from the logs and return a list of metric objects.''' metrics = [] if duration > 0: metrics += [ MetricObject(counter, self.counts[counter] / duration) for counter in self.counts ] for time_name in self.times: values = self.times[time_name]['values'] unit = self.times[time_name]['unit'] metrics.append( MetricObject(time_name + '.mean', stats_helper.find_mean(values), unit)) metrics.append( MetricObject(time_name + '.median', stats_helper.find_median(values), unit)) metrics += [ MetricObject( '%s.%sth_percentile' % (time_name, percentile), stats_helper.find_percentile(values, int(percentile)), unit) for percentile in self.percentiles ] return metrics
def get_state(self, duration): result = [] for tag, data in self.monitored_tags.iteritems(): result.append(MetricObject(data['name']+'_success', data['success'], 'Tag '+data['name']+': successes')) result.append(MetricObject(data['name']+'_error', data['error'], 'Tag '+data['name']+': errors')) return result
def get_state(self, duration): result = [] for action, count in self.actions.iteritems(): result.append(MetricObject("action_"+action, count, "Action: "+action.upper())) for protocol, count in self.protocols.iteritems(): result.append(MetricObject("proto_"+protocol, count, "Protocol: "+protocol.upper())) return result
def get_state(self, duration): '''Run any necessary calculations on the data collected from the logs and return a list of metric objects.''' self.duration = duration # Return a list of metrics objects return [ MetricObject('count', self.line_count, 'lines on log since last run', type='int'), MetricObject('rate', (float(self.line_count) / float(self.duration)), 'lines per sec', type='float', slope='both'), ]
def test_get_metric_object(self): should_be = MetricObject('int', 1, type='int32') metric_object = self.json_logster.get_metric_object('int', 1) self.assertEquals(should_be.name, metric_object.name) self.assertEquals(should_be.value, metric_object.value) self.assertEquals(should_be.type, metric_object.type) should_be = MetricObject('bool', 'False', type='string') metric_object = self.json_logster.get_metric_object('bool', False) self.assertEquals(should_be.name, metric_object.name) self.assertEquals(should_be.value, metric_object.value) self.assertEquals(should_be.type, metric_object.type)
def get_state(self, duration): '''Run any necessary calculations on the data collected from the logs and return a list of metric objects.''' self.duration = duration # Return a list of metrics objects return [ MetricObject("api." + SHORT_HOSTNAME + ".http_1xx", (self.http_1xx / self.duration), "Responses per sec"), MetricObject("api." + SHORT_HOSTNAME + ".http_2xx", (self.http_2xx / self.duration), "Responses per sec"), MetricObject("api." + SHORT_HOSTNAME + ".http_3xx", (self.http_3xx / self.duration), "Responses per sec"), MetricObject("api." + SHORT_HOSTNAME + ".http_4xx", (self.http_4xx / self.duration), "Responses per sec"), MetricObject("api." + SHORT_HOSTNAME + ".http_5xx", (self.http_5xx / self.duration), "Responses per sec"), ]
def get_state(self, duration): '''Run any necessary calculations on the data collected from the logs and return a list of metric objects.''' self.duration = duration / 10.0 # Return a list of metrics objects return [ MetricObject("notice", (self.notice / self.duration), "Logs per 10 sec"), MetricObject("warn", (self.warn / self.duration), "Logs per 10 sec"), MetricObject("error", (self.error / self.duration), "Logs per 10 sec"), MetricObject("crit", (self.crit / self.duration), "Logs per 10 sec"), MetricObject("other", (self.other / self.duration), "Logs per 10 sec"), ]
def get_state(self, duration): '''Run any necessary calculations on the data collected from the logs and return a list of metric objects.''' self.duration = float(duration) # Return a list of metrics objects result = [] for key, occ in self.error_type.iteritems(): if self.raw_data: metric = MetricObject(key, occ, "Responses") else: metric = MetricObject(key, (occ / self.duration), "Responses per sec") result.append(metric) return result
def read_nmon_line(self, line, nmonSection, nmonHeader, dt, path): values = line.strip().split(',') values.remove if (len(values) != len(nmonHeader)): logging.getLogger('logster').warn( "length of values:{} and headers:{} does not mismatch".format( len(values), len(nmonHeader))) limit = len(values) if (len(values) > len(nmonHeader)): limit = len(nmonHeader) logging.getLogger('logster').warn( "length of values:{} > headers:{} - line: {}".format( len(values), len(nmonHeader), line)) logging.getLogger('logster').warn( "Printing only {} values\n".format(limit)) for i in range(2, limit): name = "%s.%s.%s" % (path, nmonSection, nmonHeader[i]) timestamp = time.mktime(dt.timetuple()) if self.tz != 'local': tz = pytz.timezone(self.tz) timestamp = time.mktime(dt.astimezone(tz).timetuple()) m = MetricObject(name, values[i], timestamp=timestamp) self.metrics += [m]
def get_state(self, duration): '''Run any necessary calculations on the data collected from the logs and return a list of metric objects.''' self.duration = float(duration) metrics = [MetricObject(level, (getattr(self, level) / self.duration)) for level in self.levels] return metrics
def setUp(self): self.metric = MetricObject("ERROR", 1, None) self.metric.timestamp = strftime("%Y%m%dT%H:%M:00Z", gmtime(self.metric.timestamp)) self.cw = CloudWatch("key", "secretkey", self.metric) self.cw.get_instance_id("myserverID").set_params().get_signed_url()
def get_state(self, duration): self.duration = duration return [ MetricObject(s.prop, (self.metrics[s.prop] / self.duration), "Responses/sec") for s in status ]
def get_state(self, duration): # Compile a list of alert counts result = [] for level, count in self.levels.iteritems(): result.append(MetricObject("level_"+level, count, "Error count of level "+level)) return result
def get_state(self, duration): '''Run any necessary calculations on the data collected from the logs and return a list of metric objects.''' self.duration = duration # Return a list of metrics objects return_array = [ MetricObject("http_1xx", (self.http_1xx / self.duration), "Responses per sec"), MetricObject("http_2xx", (self.http_2xx / self.duration), "Responses per sec"), MetricObject("http_3xx", (self.http_3xx / self.duration), "Responses per sec"), MetricObject("http_4xx", (self.http_4xx / self.duration), "Responses per sec"), MetricObject("http_5xx", (self.http_5xx / self.duration), "Responses per sec"), MetricObject("size", (self.size_transferred / self.duration), "Size per sec") ] for squid_code in self.squid_codes: return_array.append( MetricObject("squid_" + squid_code, (self.squid_codes[squid_code] / self.duration), "Squid code per sec")) return return_array
def get_state(self, _duration): return list(map( lambda memory_measure: MetricObject( 'prod.system.%(memory_measure)s' % {'memory_measure': memory_measure}, getattr(self, memory_measure) if hasattr(self, memory_measure) else None, '%(memory_measure)s in MB' % {'memory_measure': memory_measure}, ), MemoryParser.MEMORY_MEASURES ))
def get_state(self, duration): '''Run any necessary calculations on the data collected from the logs and return a list of metric objects.''' self.duration = duration metrics = [ MetricObject(('api.' + SHORT_HOSTNAME + '.' + level), (getattr(self, level) / self.duration)) for level in self.levels ] return metrics
def get_state(self, duration): '''Run any necessary calculations on the data collected from the logs and return a list of metric objects.''' metrics = [] for status, count in self.combined_status.items(): metrics.append( MetricObject('combined.status.{status}'.format(status=status), count, 'Responses')) for httpver, count in self.combined_httpver.items(): metrics.append( MetricObject( 'combined.httpver.{httpver}'.format(httpver=httpver), count, 'Responses')) for firstsegment, statuses in self.status_stats.items(): for status, count in statuses.items(): metric_name = 'raw.{firstsegment}.status.{status}'.format( firstsegment=firstsegment, status=status) metrics.append(MetricObject(metric_name, count, 'Responses')) return metrics
def get_state(self, duration): '''Run any necessary calculations on the data collected from the logs and return a list of metric objects.''' self.duration = float(duration) self.latencies.sort() # Return a list of metrics objects requests = [ MetricObject("requests.http_1xx", self.http_1xx, "Requests"), MetricObject("requests.http_2xx", self.http_2xx, "Requests"), MetricObject("requests.http_3xx", self.http_3xx, "Requests"), MetricObject("requests.http_4xx", self.http_4xx, "Requests"), MetricObject("requests.http_5xx", self.http_5xx, "Requests"), MetricObject("latency.p50", self.percentile(0.5), "Miliseconds"), MetricObject("latency.p90", self.percentile(0.9), "Miliseconds"), MetricObject("latency.p99", self.percentile(0.99), "Miliseconds"), MetricObject("latency.p999", self.percentile(0.999), "Miliseconds"), MetricObject("latency.max", self.max, "Miliseconds") ] return requests
def get_metric_object(self, metric_name, metric_value): ''' Default key_to_metric_object method. Given a metric name and value, this returns a MetricObject filled in. This default method will attempt to infer MetricObject.type from the metric_value type, but nothing more. If you need to set any particular MetricObject parameters (such as slope, title, etc.), you should override this method. ''' metric_type = self.infer_metric_type(metric_value) # make sure the metric value is properly a string. if metric_type == 'string': metric_value = str(metric_value) return MetricObject(metric_name, metric_value, type=metric_type)
def get_state(self, duration): metric_objects = [] for metric in self.metrics: hostname = metric total_time_taken = self.metrics[metric]['total_time_taken'] count = self.metrics[metric]['count'] total_time_taken_milliseconds = self._microseconds_to_milliseconds( total_time_taken) #Drop the decimals, not needed for our metrics avg_time_taken_milliseconds = int( (total_time_taken_milliseconds / count)) metric_objects.append( MetricObject(hostname, avg_time_taken_milliseconds, "apache_avg_resp_time")) return metric_objects
def get_state(self, duration): '''Run any necessary calculations on the data collected from the logs and return a list of metric objects.''' self.duration = duration totalTxns = self.numSent + self.numBounced + self.numDeferred pctDeferred = 0.0 pctSent = 0.0 pctBounced = 0.0 avgDelay = 0 mailTxnsSec = 0 mailSentSec = 0 #mind divide by zero situations if (totalTxns > 0): pctDeferred = (float(self.numDeferred) / totalTxns) * 100 pctSent = (float(self.numSent) / totalTxns) * 100 pctBounced = (float(self.numBounced) / totalTxns) * 100 if (self.numSent > 0): avgDelay = self.totalDelay / self.numSent if (self.duration > 0): mailTxnsSec = totalTxns / self.duration mailSentSec = self.numSent / self.duration # Return a list of metrics objects return [ MetricObject("numSent", self.numSent, "Total Sent"), MetricObject("pctSent", pctSent, "Percentage Sent"), MetricObject("numDeferred", self.numDeferred, "Total Deferred"), MetricObject("pctDeferred", pctDeferred, "Percentage Deferred"), MetricObject("numBounced", self.numBounced, "Total Bounced"), MetricObject("pctBounced", pctBounced, "Percentage Bounced"), MetricObject("mailTxnsSec", mailTxnsSec, "Transactions per sec"), MetricObject("mailSentSec", mailSentSec, "Sends per sec"), MetricObject("avgDelay", avgDelay, "Average Sending Delay"), ]
def get_state(self, duration): '''Run any necessary calculations on the data collected from the logs and return a list of metric objects.''' self.duration = duration metric_objects = [] for metric_name, metric_value in self.metrics.items(): if type(metric_value) == float: metric_type = 'float' elif type(metric_value) == int or type(metric_value) == long: metric_type = 'int32' else: metric_type = 'string' metric_value = str(metric_value) metric_objects.append(MetricObject(metric_name, metric_value, type='int')) return metric_objects
def get_state(self, duration): self.write(self.header) kmatches = self.keyword_match() self.duration = duration line_rate = float(self.line_count) / float(self.duration) alarm_rate = float(self.alarm_count) / float(self.duration) if self.alarm_count > self.alarmthreshold and self.emails: self.write("Alarm count is {}".format(self.alarm_count)) self.notify(line_rate, kmatches) return [ MetricObject('{}.alarm_rate'.format(self.name), (alarm_rate), 'lines per sec', type='float', slope='both'), ]
def parse_line(self, line): '''This function should digest the contents of one line at a time, updating object's state variables. Takes a single argument, the line to be parsed.''' try: # Apply regular expression to each line and extract interesting bits. regMatch = self.reg.match(line) if regMatch: linebits = regMatch.groupdict() test_path = linebits['test_path'].replace(".","").replace("/spec","spec").replace("/",".").replace(":",".") run_time = float(linebits['run_time']) graphite_path = "specjour." + linebits['computer_name'] + "." + test_path self.tests.append(MetricObject(graphite_path, run_time)) else: raise LogsterParsingException, "regmatch failed to match" except Exception, e: raise LogsterParsingException, "regmatch or contents failed with %s" % e
def get_state(self, duration): '''Run any necessary calculations on the data collected from the logs and return a list of metric objects.''' self.duration = duration # Return a list of metrics objects return [ MetricObject("http_1xx", (self.http_1xx / self.duration), "Responses per sec"), MetricObject("http_2xx", (self.http_2xx / self.duration), "Responses per sec"), MetricObject("http_3xx", (self.http_3xx / self.duration), "Responses per sec"), MetricObject("http_4xx", (self.http_4xx / self.duration), "Responses per sec"), MetricObject("http_5xx", (self.http_5xx / self.duration), "Responses per sec"), MetricObject("request_rate", (self.request_count / self.duration), "Request per sec"), MetricObject("avg_request_time", (self.request_time / self.request_count), "Milliseconds per request"), ]
def get_state(self, duration): '''Run any necessary calculations on the data collected from the logs and return a list of metric objects.''' self.duration = float(duration) # Return a list of metrics objects return [ MetricObject("total_requests", (self.total_requests / self.duration), "Requests per sec"), MetricObject("response_time_avg", (self.total_response_time / self.total_requests / self.duration), "Average response time"), MetricObject("http_1xx", (self.http_1xx / self.duration), "Responses per sec"), MetricObject("http_2xx", (self.http_2xx / self.duration), "Responses per sec"), MetricObject("http_3xx", (self.http_3xx / self.duration), "Responses per sec"), MetricObject("http_4xx", (self.http_4xx / self.duration), "Responses per sec"), MetricObject("http_5xx", (self.http_5xx / self.duration), "Responses per sec"), ]
def get_state(self, duration): self.duration = duration metrics = [] for upstream, value in self.metrics['upstreams'].items(): metrics.append( MetricObject( self.upstream_metric(upstream, 'hits'), value['hits'] / self.duration, "upstream hits/sec" ) ) metrics.append( MetricObject( self.upstream_metric(upstream, 'resp_mean_time_ms'), value['ms'] / value['hits'], "upstream mean resp time/hit" ) ) metrics.append( MetricObject( self.upstream_metric(upstream, 'size'), value['size'], "total size" ) ) metrics.append( MetricObject( self.upstream_metric(upstream, 'speed'), value['size'] / self.duration, "bytes/s" ) ) metrics.append( MetricObject( self.upstream_metric(upstream, 'http_200.hits'), value['http_200'] / self.duration, "http 200/sec" ) ) if value['http_200']: metrics.append( MetricObject( self.upstream_metric(upstream, 'http_200.gzipped'), value['http_200_gzipped'] / self.duration, "http 200 gzipped/sec" ) ) metrics.append( MetricObject( self.upstream_metric(upstream, 'http_200.resp_mean_time_ms'), value['http_200_ms'] / value['http_200'], "upstream mean resp time/hit" ) ) if value['http_200_gzipped']: metrics.append( MetricObject( self.upstream_metric(upstream, 'http_200.mean_gzip_ratio'), value['gzip_ratio'] / value['http_200_gzipped'], "Mean gzip ratio/gzipped hit" ) ) return metrics
def test_full(self): # init parser p = PerformanceLogster('--mapping=tests/test.map') samples = [ '{ "timestamp": "2014-10-10T06:30:32+00:00", "request": { "remote_addr": "192.168.56.1", "request": "GET /profile/me HTTP/1.1", "status": "200", "request_time": "0.406", "upstream_response_time": "0.406" } }', '{ "timestamp": "2014-10-10T06:30:36+00:00", "request": { "remote_addr": "192.168.56.1", "request": "GET / HTTP/1.1", "status": "200", "request_time": "0.413", "upstream_response_time": "0.413" } }' ] # parse samples for sample in samples: p.parse_line(sample) expected = [ MetricObject('http_200.unknown.GET.upstreamTime.count', 1), MetricObject('http_200.unknown.GET.upstreamTime.mean', 413.0, 'ms'), MetricObject('http_200.unknown.GET.upstreamTime.median', 413.0, 'ms'), MetricObject('http_200.unknown.GET.upstreamTime.90th_percentile', 413.0, 'ms'), MetricObject('http_200.profile.GET.upstreamTime.count', 1), MetricObject('http_200.profile.GET.upstreamTime.mean', 406.0, 'ms'), MetricObject('http_200.profile.GET.upstreamTime.median', 406.0, 'ms'), MetricObject('http_200.profile.GET.upstreamTime.90th_percentile', 406.0, 'ms'), MetricObject('http_200.profile.GET.responseTime.count', 1), MetricObject('http_200.profile.GET.responseTime.mean', 406.0, 'ms'), MetricObject('http_200.profile.GET.responseTime.median', 406.0, 'ms'), MetricObject('http_200.profile.GET.responseTime.90th_percentile', 406.0, 'ms'), MetricObject('http_200.unknown.GET.responseTime.count', 1), MetricObject('http_200.unknown.GET.responseTime.mean', 413.0, 'ms'), MetricObject('http_200.unknown.GET.responseTime.median', 413.0, 'ms'), MetricObject('http_200.unknown.GET.responseTime.90th_percentile', 413.0, 'ms'), ] outputMetrics = p.get_state(1) # validate number of metrics self.assertEqual(len(expected), len(outputMetrics)) # compare all metrics objects for i in range(0, len(expected) - 1): self.assertEqual(expected[i].name, outputMetrics[i].name) self.assertEqual(expected[i].value, outputMetrics[i].value) self.assertEqual(expected[i].units, outputMetrics[i].units)