def prime_metrics(self):
for pair in self.metrics_prime_list:
try:
pmetric,val = pair.split(":")
m=gMetric("float", "%s%s" % (self.prefix,pmetric), "prime", self.notify_schedule,self.graphite_server,self.debug)
m.send(float(val),1)
self.total_metric_count += 1
except Exception, e:
print >> sys.stderr, "Failed to send prime metric %s (%s)" % (pair, e)
def notifybrand(self, brand, seconds):
if self.graphite_server:
from graphitelib import gMetric
else:
from gmetriclib import gMetric
try:
if not self.gmetric_brands.has_key(brand):
self.gmetric_brands[brand]=gMetric("float", "%sQPS_%s" % (self.prefix,brand), "qps", self.notify_schedule,self.graphite_server,self.debug)
self.gmetric_brands[brand].send(float(self.brand_counts[brand]/seconds), 1)
self.total_metric_count += 1
except Exception, e:
print "couldn't notify for brand %s (%s)" % (brand, e)
def notify(self, seconds):
if self.graphite_server:
from graphitelib import gMetric
from graphitelib import sendMetrics
else:
from gmetriclib import gMetric
self.notify_time_start=time.time()
#print time.strftime("%H:%M:%S")
if self.pt_requests > 0:
self.gmetric["TPT"].send(self.processing_time, 1)
#print "%.2f / %d" % (self.processing_time,self.pt_requests)
self.gmetric["APT"].send(self.processing_time/self.pt_requests, 1)
self.gmetric["MAX"].send(self.max_processing_time, 1)
self.gmetric["SLA"].send(self.pt_requests_exceeding_sla*100.0/self.pt_requests, 1)
self.gmetric["SLA_ct"].send(self.pt_requests_exceeding_sla, 1)
else:
self.gmetric["TPT"].send(0.0, 1)
self.gmetric["APT"].send(0.0, 1)
self.gmetric["MAX"].send(0.0, 1)
self.gmetric["SLA"].send(0.0, 1)
self.gmetric["SLA_ct"].send(0.0, 1)
if seconds > 0:
qps=float(self.requests/seconds)
else:
qps=0.0
self.gmetric["Q"].send(self.requests, 1)
self.gmetric["QPS"].send(qps, 1)
#print self.processing_time
self.total_metric_count += 7
#print "covered %d, requests %d" % (self.covered,self.requests)
if self.requests > 0:
coverage_per_query=self.covered*100.0/self.requests
else:
coverage_per_query=0.0
#print "served %d, possible %d" % (self.inventory_served,self.inventory_possible)
if self.inventory_possible > 0:
coverage_per_ad_requested=self.inventory_served*100.0/self.inventory_possible
else:
coverage_per_ad_requested=0.0
#self.gmetric_cpq.send(coverage_per_query, 1)
#self.gmetric_cpar.send(coverage_per_ad_requested, 1)
self.gmetric["code_version"].send("\"%s\"" % code_version, 0)
self.gmetric["ignore"].send(self.ignored_count, 1)
self.total_metric_count += 2
for brand in self.brand_counts.keys():
self.notifybrand(brand,seconds)
for rmetric in self.metrics_ratio_list:
tot=0
regex=re.compile("^%s" % rmetric)
for smetric in self.metric_sums.keys():
rmetric_name="%s_ratio" % smetric
if re.match(regex, smetric):
if self.requests != 0:
# we don't want to multiply by 100 for sum ratios
perc=float(self.metric_sums[smetric])/float(self.requests)
else:
perc=0.0
try:
self.gmetric[rmetric_name].send(perc,1)
except: #sketchy
self.gmetric[rmetric_name]=gMetric("float", "%s%s" % (self.prefix,rmetric_name), "percent", self.notify_schedule,self.graphite_server,self.debug)
self.gmetric[rmetric_name].send(perc,1)
self.total_metric_count += 1
for cmetric in self.metric_counts.keys():
if re.match(regex, cmetric):
tot=tot+self.metric_counts[cmetric]
#print "TOTAL %d" % tot
for cmetric in self.metric_counts.keys():
rmetric_name="%s_ratio" % cmetric
if re.match(regex, cmetric):
if tot!=0:
perc=float(self.metric_counts[cmetric])/float(tot) * 100
else:
perc=0.0
#print "%s %s %.2f" % (self.metric_counts[cmetric], cmetric, perc)
try:
self.gmetric[rmetric_name].send(perc,1)
except: #sketchy
self.gmetric[rmetric_name]=gMetric("float", "%s%s" % (self.prefix,rmetric_name), "percent", self.notify_schedule,self.graphite_server,self.debug)
self.gmetric[rmetric_name].send(perc,1)
self.total_metric_count += 1
# send smetrics
for smetric in self.metric_sums.keys():
#print "DEBUG: sending %.2f" % self.metric_sums[smetric]
try:
self.gmetric[smetric].send(self.metric_sums[smetric],1)
except: #sketchy
self.gmetric[smetric]=gMetric("float", "%s%s" % (self.prefix,smetric), "sum", self.notify_schedule,self.graphite_server,self.debug)
self.gmetric[smetric].send(self.metric_sums[smetric],1)
self.total_metric_count += 1
# send cmetrics
for cmetric in self.metric_counts.keys():
#print "DEBUG: sending %.2f" % self.metric_counts[cmetric]
try:
self.gmetric[cmetric].send(self.metric_counts[cmetric],1)
except: #sketchy
self.gmetric[cmetric]=gMetric("float", "%s%s" % (self.prefix,cmetric), "count", self.notify_schedule,self.graphite_server,self.debug)
self.gmetric[cmetric].send(self.metric_counts[cmetric],1)
self.total_metric_count += 1
# send emetrics/calcs
for emetric in self.metric_calcs.keys():
try:
cvalue=self.calculate(self.metric_calc_expr[emetric])
except Exception, e:
print Exception, e
cvalue=0
#print "DEBUG: emetric sending %.2f for %s" % (cvalue, emetric)
try:
self.gmetric[emetric].send(cvalue,1)
except Exception, e: #sketchy, create then send instead of pre-initializing
self.gmetric[emetric]=gMetric("float", "%s%s" % (self.prefix,emetric), "expression", self.notify_schedule,self.graphite_server,self.debug)
self.gmetric[emetric].send(cvalue,1)
def __init__(self, pidfile=None, daemonize=0, configfile=None, distinguisher=None, debug=0, quit=False, beginning=False, testconfig=False, graphite_server=None, use_graphite=False):
self.log=""
self.fd=None
self.graphite_server=graphite_server
if self.graphite_server:
self.use_graphite = True
else:
self.use_graphite = use_graphite
# initializing, will be populated later
self.plugin_list=[]
self.plugins=[]
self.plugin_dir=None
self.plugin_paths = ["/app/logwatcher/plugins", os.path.dirname(__file__)+"/plugins"]
self.gmetric_brands={}
self.regex={}
self.gmetric={}
# metrics that count matching lines
self.metric_counts={}
# metrics that sum values found
self.metric_sums={}
# metrics that are calculated from other metrics
self.metric_calcs={}
self.metric_calc_expr={}
# metrics that describe distributions
self.metric_dists={}
self.metric_dist_bucketsize={}
self.metric_dist_bucketcount={}
self.ignore_pattern=""
self.ignore=None
self.configfile=configfile
self.debug=debug
self.pidfile=pidfile
self.distinguisher=distinguisher
self.quit=quit
self.beginning=beginning
self.testconfig=testconfig
self.log_time=0
self.log_time_start=0
self.notify_time=0
self.notify_time_start=0
self.readConfig()
signal.signal(signal.SIGHUP, self.reReadConfig)
self.new_metric_count=0 # counts new-found dynamic metrics
self.total_metric_count=0 # counts metrics sent
self.prefix_root="LW_"
self.prefix=self.prefix_root
if self.distinguisher:
self.prefix="%s%s_" % (self.prefix, self.distinguisher)
self.daemonize=daemonize
if self.getPID() < 1:
if self.daemonize == 1:
procdaemonize()
if self.lockPID() == 0:
print "Pidfile found"
sys.exit(-1)
self.log_count=0 # how many different logs have we opened?
self.curr_pos=0
self.prev_pos=0
self.last_time=time.time()
if self.use_graphite and not self.graphite_server:
self.graphite_server = self.readGraphiteConf()
if not self.graphite_server:
print >> sys.stderr, "ERROR: Failed to set graphite server. Using gmetric."
else:
self.use_graphite = True
self.brand_counts={}
if self.graphite_server:
from graphitelib import gMetric
else:
from gmetriclib import gMetric
self.gmetric["Q"]=gMetric("float", "%sQueries" % self.prefix, "count", self.notify_schedule,self.graphite_server,self.debug)
self.gmetric["QPS"]=gMetric("float", "%sQPS" % self.prefix, "qps", self.notify_schedule,self.graphite_server,self.debug)
self.gmetric["APT"]=gMetric("float", "%sAvg_Processing_Time" % self.prefix, "seconds", self.notify_schedule,self.graphite_server,self.debug)
self.gmetric["MAX"]=gMetric("float", "%sMax_Processing_Time" % self.prefix, "seconds", self.notify_schedule,self.graphite_server,self.debug)
self.gmetric["TPT"]=gMetric("float", "%sTotal_Processing_Time" % self.prefix, "seconds", self.notify_schedule,self.graphite_server,self.debug)
self.gmetric["SLA"]=gMetric("float", "%sexceeding_SLA" % self.prefix, "percent", self.notify_schedule,self.graphite_server,self.debug)
self.gmetric["SLA_ct"]=gMetric("float", "%sexceeding_SLA_ct" % self.prefix, "percent", self.notify_schedule,self.graphite_server,self.debug)
self.gmetric["code_version"]=gMetric("string", "%sLW_Version" % self.prefix_root, "string", self.notify_schedule,self.graphite_server,self.debug)
self.gmetric["ignore"]=gMetric("float", "%signored" % self.prefix, "count", self.notify_schedule,self.graphite_server,self.debug)
self.gmetric["NOTIFY_TIME"]=gMetric("float", "%s%s" % (self.prefix_root,"LW_NotifyTime"), "seconds", self.notify_schedule,self.graphite_server,self.debug)
self.gmetric["LOG_TIME"]=gMetric("float", "%s%s" % (self.prefix_root,"LW_LogTime"), "seconds", self.notify_schedule,self.graphite_server,self.debug)
self.gmetric["NEW_METRICS"]=gMetric("float", "%s%s" % (self.prefix_root,"LW_NewMetrics"), "float", self.notify_schedule,self.graphite_server,self.debug)
self.gmetric["TOTAL_METRICS"]=gMetric("float", "%s%s" % (self.prefix_root,"LW_TotalMetrics"), "float", self.notify_schedule,self.graphite_server,self.debug)
# use this for sub-hourly and other odd log rotation
self.curr_inode=None
self.prime_metrics()
self.initialize_counters()
self.watch()
# first bucket
if last == 0:
dmetric_b="%s_%d-%d" % (dmetric, 0, current-1)
# last bucket
elif bucket == self.metric_dist_bucketcount[dmetric]-1:
dmetric_b="%s_%d-%s" % (dmetric, last, "inf")
# other buckets
else:
dmetric_b="%s_%d-%d" % (dmetric, last, current-1)
last=current
#print dmetric_b,self.metric_dists[dmetric][bucket]
#print "DEBUG: sending %.2f" % self.metric_counts[dmetric_b][bucket]
try:
self.gmetric[dmetric_b].send(self.metric_counts[dmetric_b],1)
except: #sketchy
self.gmetric[dmetric_b]=gMetric("float", "%s%s" % (self.prefix,dmetric_b), "count", self.notify_schedule,self.graphite_server,self.debug)
self.gmetric[dmetric_b].send(self.metric_dists[dmetric][bucket],1)
self.total_metric_count += 1
if self.requests != 0:
# we don't want to multiply by 100 for sum ratios
perc=float(self.metric_dists[dmetric][bucket])/float(self.requests) * 100
#perc=float(self.metric_counts[cmetric])/float(tot) * 100 # do we need to count matches (tot)?
else:
perc=0.0
try:
self.gmetric[dmetric_b+"_ratio"].send(perc,1)
except: #sketchy
self.gmetric[dmetric_b+"_ratio"]=gMetric("float", "%s%s_ratio" % (self.prefix,dmetric_b), "percent", self.notify_schedule,self.graphite_server,self.debug)
self.gmetric[dmetric_b+"_ratio"].send(perc,1)
self.total_metric_count += 1