def getStatsSummary(self):
if "PA_WFSUMMARY" in self.cache:
table = self.cache.cacheFetch('PA_WFSUMMARY',self._getWFSummary)
result = {}
result['workflow_count']=len(table)
result['total_unmerged']=0
result['total_merged']=0
time_per_event = []
for t in table:
result['total_unmerged']+=int(table[t]['UNMERGED_EVTS'])
result['total_merged']+=int(table[t]['MERGED_EVTS'])
time_per_event.append(float(table[t]['TIME_PER_EVENT']))
if len(time_per_event)>0:
result['avg_time_per_event']='%.2f'%(sum(time_per_event)/len(time_per_event))
result['min_time_per_event']='%.2f'%min(time_per_event)
result['max_time_per_event']='%.2f'%max(time_per_event)
else:
_logwarn("PRODMON_S: WFSummary table contains no workflows")
result['avg_time_per_event']='No workflows found'
result['min_time_per_event']='No workflows found'
result['max_time_per_event']='No workflows found'
return result
else:
self.cache.asyncCacheFill('PA_WFSUMMARY',self._getWFSummary)
return {
'workflow_count':'Data Unavailable',
'total_unmerged':'Data Unavailable',
'total_merged':'Data Unavailable',
'avg_time_per_event':'Data Unavailable',
'max_time_per_event':'Data Unavailable',
'min_time_per_event':'Data Unavailable'
}
def extension(modules, what, *args):
for m in modules:
ctor = getattr(m, what, None)
if ctor:
return ctor(*args)
_logwarn("extension '%s' not found" % what)
return None
def extension(modules, what, *args):
for m in modules:
ctor = getattr(m, what, None)
if ctor:
return ctor(*args)
_logwarn("extension '%s' not found" % what)
return None
def run(self):
while True:
self.lock.acquire()
stopme = self.stopme
self.lock.release()
if stopme:
break
rximg = re.compile(r"<img\s+(.*?)/?>", re.I)
for s in self.infosrc:
img = 0
imgrefs = {}
try:
data = urllib2.urlopen(s['html']).read().replace("\n", " ")
for imgtag in re.findall(rximg, data):
while True:
m = re.match(
"^([-A-Za-z0-9]+)=(\"[^\"]*\"|'[^']*'|\S+)\s*/?",
imgtag)
if not m:
break
arg = m.group(2)
if len(arg
) >= 2 and arg[0] == '"' and arg[-1] == '"':
arg = arg[1:-1]
elif len(arg) >= 2 and arg[0] == "'" and arg[
-1] == "'":
arg = arg[1:-1]
if m.group(1) == "src" and arg.find(
s['match']) >= 0:
imgrefs[s['images'][img]] = arg
img = img + 1
imgtag = imgtag[m.end():]
except Exception, e:
_logwarn("failed to fetch document '%s': %s" %
(s['html'], str(e)))
imgdata = {}
imgurl = None
for (name, url) in imgrefs.iteritems():
try:
imgurl = urlparse.urljoin(s['html'], url)
img = urllib2.urlopen(imgurl)
info = img.info()
imgdata[name] = {
'data': img.read(),
'type': info['Content-Type']
}
except Exception, e:
_logwarn("failed to fetch image %s from '%s': %s" %
(name, imgurl, str(e)))
imgdata[name] = None
self.lock.acquire()
for (name, info) in imgdata.iteritems():
self.images[name] = info
self.lock.release()
def _getResult(self,query):
"""
GetResult is a cachemaker function which actually gets data from the backend. HTTP errors are not actually handled here (logged and re-raised) since we don't want dead data to get cached - better to end up with HTTP 500 instead
"""
urlquery = query.getURL()
try:
url = "%s%s" % (ProdMonSource.url,urlquery)
data = urllib2.urlopen(url).read()
except urllib2.HTTPError,e:
_logwarn("PRODMON_S: HTTPError code=%s url=%s"%(e.code,url))
raise
def _queryData(self, db, label, validfor, q, *args):
cache = db['cache'].setdefault(label, {})
key = ":".join(str(i) for i in args)
now = time.time()
if key not in cache or now > cache[key]['valid']:
try:
if not db['db']: db['db'] = db['def']['connect']()
c = db['db'].cursor(); c.arraysize = 10000
cache[key] = {'valid': now + validfor, 'data': q(c, *args) }
except Exception, e:
_logwarn("failed to execute query %s for '%s' in %s: %s; disconnecting"
% (label, key, db['def']['label'], str(e)))
db['db'] = None
raise
def run(self):
while True:
self.lock.acquire()
stopme = self.stopme
self.lock.release()
if stopme:
break
rximg = re.compile(r"<img\s+(.*?)/?>", re.I)
for s in self.infosrc:
img = 0
imgrefs = {}
try:
data = urllib2.urlopen(s['html']).read().replace("\n", " ")
for imgtag in re.findall(rximg, data):
while True:
m = re.match("^([-A-Za-z0-9]+)=(\"[^\"]*\"|'[^']*'|\S+)\s*/?", imgtag)
if not m:
break
arg = m.group(2)
if len(arg) >= 2 and arg[0] == '"' and arg[-1] == '"':
arg = arg[1:-1]
elif len(arg) >= 2 and arg[0] == "'" and arg[-1] == "'":
arg = arg[1:-1]
if m.group(1) == "src" and arg.find(s['match']) >= 0:
imgrefs[s['images'][img]] = arg
img = img+1
imgtag = imgtag[m.end():]
except Exception, e:
_logwarn("failed to fetch document '%s': %s" % (s['html'], str(e)))
imgdata = {}
imgurl = None
for (name, url) in imgrefs.iteritems():
try:
imgurl = urlparse.urljoin(s['html'], url)
img = urllib2.urlopen(imgurl)
info = img.info()
imgdata[name] = {'data': img.read(), 'type': info['Content-Type']}
except Exception, e:
_logwarn("failed to fetch image %s from '%s': %s" % (name, imgurl, str(e)))
imgdata[name] = None
self.lock.acquire()
for (name, info) in imgdata.iteritems():
self.images[name] = info
self.lock.release()
def _queryData(self, db, label, validfor, q, *args):
cache = db['cache'].setdefault(label, {})
key = ":".join(str(i) for i in args)
now = time.time()
if key not in cache or now > cache[key]['valid']:
try:
if not db['db']: db['db'] = db['def']['connect']()
c = db['db'].cursor()
c.arraysize = 10000
cache[key] = {'valid': now + validfor, 'data': q(c, *args)}
except Exception, e:
_logwarn(
"failed to execute query %s for '%s' in %s: %s; disconnecting"
% (label, key, db['def']['label'], str(e)))
db['db'] = None
raise
def insert(self,key,lifetime,data):
"""
Insert a new key into the cache, with associated expiry lifetime, size and data. Triggers volume cleaning if the new cache size exceeds the maximum.
@param key: key for the new data - should be a string
@param lifetime: desired cache lifetime of this object in seconds
@param data: free data object. Preferably doesn't refer to other objects in the cache, or size calculations will become impossible.
"""
self.lock.acquire()
try:
try:
data = cPickle.dumps(data,cPickle.HIGHEST_PROTOCOL)
self._insert(key,lifetime,len(data),data)
except cPickle.PickleError:
_logwarn("CACHE: Pickling error with key=%s"%key)
pass
finally:
self.lock.release()
def legend(self):
legend = self._getLegend()
if len(legend)>=int(self.get('max_legend')):
_logwarn("PLOT: Legend too long (%s) - truncating"%len(legend))
legend = legend[:int(self.get('max_legend'))]
legend.append((int(self.get('max_legend')),'Some entries truncated',{'facecolor':'#000000'},0))
if len(legend)==0:
(fig, canvas, w, h) = self.canvas()
return self.save(fig,canvas)
namelen = max(map(lambda x: len(x[1]),legend))
i = len(legend)+1
dpif = float(self.get('dpi')) / 72
wcol = 10 + (2 + 0.6*namelen)*self.font.get_size()*dpif
ncol = max(1, math.floor(float(self.get('width')) / wcol))
wcol = float(self.get('width')) / ncol
nrow = int((i + ncol - 1) / ncol)
self.height = (nrow + 1) * self.font.get_size() * 1.4 * dpif
(fig, canvas, w, h) = self.canvas()
step = self.font.get_size() * 1.4 * dpif / h
x = 0
y = 0.5 * self.font.get_size() * dpif / h
xbox = 10./w
wbox = self.font.get_size() * dpif / w * 1.25
hbox = self.font.get_size() * dpif / h * 0.75
xtext = xbox + wbox*1.5
row = 0
for item in legend:
fig.text(x + xtext, 1 - y, item[1],
horizontalalignment='left',
verticalalignment='top',
fontproperties=self.font)
p = Rectangle((x + xbox, 1-y-hbox), wbox, hbox, linewidth=0.5, fill=True)
p.update(item[2])
fig._set_artist_props(p)
fig.patches.append(p)
y += step
row += 1
if row == nrow:
x += wcol/w
y = 0.5 * self.font.get_size() * dpif / h
row = 0
return self.save(fig, canvas)
def _databases(self, names):
self.lock.acquire()
try:
instances = {}
for namegroup in names:
if namegroup in self.db:
instances[namegroup] = self.db[namegroup]
else:
for (x, db) in self.db.iteritems():
if db['def']['class'] == namegroup:
instances[x] = db
valid = []
for i in instances.values():
(db, spec, ok) = (i['db'], i['def'], False)
if db:
try:
c = db.cursor()
c.execute("select sysdate from dual")
c.fetchall()
ok = True
except Exception, e:
_logwarn("stale database connection %s, error was %s" %
(spec['label'], str(e)))
i['db'] = None
if not ok:
try:
# _loginfo("(re)connecting to database %s" % spec['label'])
# db = spec['connect']()
# c = db.cursor()
# c.execute("select sysdate from dual")
# c.fetchall()
# i['db'] = db
ok = True
except Exception, e:
_logwarn(
"disabling database %s due to connection error %s"
% (spec['label'], str(e)))
if ok:
valid.append(i)
def retrieve(self,key,default=None):
#_loginfo("CACHE: Retrieve key=%s"%key)
"""
Attempts to retrieve the data associated with a key from the cache. Returns default (None) if not found.
@param key: cache key to retrieve data for
@param default: what to return if the key is not in the cache (None)
"""
#print "PC: Getting key [%s]" % (key)
result=default
self.lock.acquire()
try:
if self._exists(key):
try:
result = cPickle.loads(self.cache[key][2])
except cPickle.PickleError:
_logwarn("CACHE: Unpickling error with key=%s"%key)
pass
finally:
self.lock.release()
return result
def _databases(self, names):
self.lock.acquire()
try:
instances = {}
for namegroup in names:
if namegroup in self.db:
instances[namegroup] = self.db[namegroup]
else:
for (x, db) in self.db.iteritems():
if db['def']['class'] == namegroup:
instances[x] = db
valid = []
for i in instances.values():
(db, spec, ok) = (i['db'], i['def'], False)
if db:
try:
c = db.cursor()
c.execute("select sysdate from dual")
c.fetchall()
ok = True
except Exception, e:
_logwarn("stale database connection %s, error was %s"
% (spec['label'], str(e)))
i['db'] = None
if not ok:
try:
# _loginfo("(re)connecting to database %s" % spec['label'])
# db = spec['connect']()
# c = db.cursor()
# c.execute("select sysdate from dual")
# c.fetchall()
# i['db'] = db
ok = True
except Exception, e:
_logwarn("disabling database %s due to connection error %s"
% (spec['label'], str(e)))
if ok:
valid.append(i)
def _queryTimeSeries(self, db, label, validfor, q, *args):
cache = db['cache'].setdefault(label, {})
(bin, start, end) = args[0:3]
found = None
for key in cache:
if key[0] == bin and key[1] <= start and key[2] >= end:
if not found or found[1] < key[1] or found[2] > key[2]:
found = key
now = time.time()
key = (found or (bin, start, end))
if key not in cache or now > cache[key]['valid']:
try:
if not db['db']: db['db'] = db['def']['connect']()
c = db['db'].cursor(); c.arraysize = 10000
cache[key] = { 'valid': now + validfor, 'data': q(c, *args) }
except Exception, e:
_logwarn("failed to execute query %s for '%s' in %s: %s; disconnecting"
% (label, key, db['def']['label'], str(e)))
db['db'] = None
raise
def _getPlotFromQuery(self,query,resulttype):
"""
GetPlotFromQuery evaluates a query object, testing it for validity before using the cache access functions to return the data.
"""
if not (resulttype in ('plot','legend','map','plot+legend','thumbnail')): #or (query['plot']=='data' and resulttype=='table')):
_logwarn("PRODMON_S: Unknown or inconsistent result type: %s"%resulttype)
return None,None
if not query.isValid():
_logwarn("PRODMON_S: Invalid query.")
return None,None
plot = query['plot']
if not plot in ProdMonSource.params['plot']:
_logwarn("PRODMON_S: Unknown plot type: %s"%plot)
return None,None
plotdata = self.cache.lockedCacheFetch(query.cacheKey('PA_PLOT'),self._plotMaker,query)
#_loginfo("PRODMON_S: Result plotsize=%s legendsize=%s mapsize=%s plot+legendsize=%s thumbnailsize=%s" %(len(plotdata[0]),len(plotdata[1]),len(plotdata[2]),len(plotdata[3]),len(plotdata[4])))
if resulttype=='plot':
return ProdMonSource.types[query['type']],plotdata[0]
elif resulttype=='legend':
return ProdMonSource.types[query['type']],plotdata[1]
elif resulttype=='map':
return 'text/plain',plotdata[2]
elif resulttype=='plot+legend':
return ProdMonSource.types[query['type']],plotdata[3]
elif resulttype=='thumbnail':
return ProdMonSource.types[query['type']],plotdata[4]
else:
pass # Already validated that we have one of the above
def _verify(self,key,value):
timepattern = re.compile('^[0-9]{4}-[0-9]{1,2}-[0-9]{1,2} [0-9]{2}:[0-9]{2}:[0-9]{2}$')
verifier = {
'sortby': lambda x: x in ProdMonSource.params['sortby'],
'what': lambda x: x in ProdMonSource.params['what'],
'job_type': lambda x: x in ProdMonSource.params['job_type'],
'graph': lambda x: x in ProdMonSource.params['plot'],
'use_period': lambda x: x in ('true','false'),
'period': lambda x: x.isdigit() and int(x)>0 and int(x)<=8760,
'image_height': lambda x: x.isdigit() and int(x)>0 and int(x)<=2000,
'image_width': lambda x: x.isdigit() and int(x)>0 and int(x)<=10000,
'image_dpi': lambda x: x.isdigit() and int(x)>0 and int(x)<=600,
'starttime': lambda x: timepattern.match(x),
'endtime': lambda x: timepattern.match(x)
}
if key in verifier:
if verifier[key](value):
return value
else:
_logwarn("PRODMON_W: Verification failed for %s=%s"%(key,value))
return deepcopy(ProdMonWorkspace.ProdMonPlot.plotdef[key])
else:
return value
def _queryTimeSeries(self, db, label, validfor, q, *args):
cache = db['cache'].setdefault(label, {})
(bin, start, end) = args[0:3]
found = None
for key in cache:
if key[0] == bin and key[1] <= start and key[2] >= end:
if not found or found[1] < key[1] or found[2] > key[2]:
found = key
now = time.time()
key = (found or (bin, start, end))
if key not in cache or now > cache[key]['valid']:
try:
if not db['db']: db['db'] = db['def']['connect']()
c = db['db'].cursor()
c.arraysize = 10000
cache[key] = {'valid': now + validfor, 'data': q(c, *args)}
except Exception, e:
_logwarn(
"failed to execute query %s for '%s' in %s: %s; disconnecting"
% (label, key, db['def']['label'], str(e)))
db['db'] = None
raise
def plot(self,*path,**attrs):
"""Landing function for plotfairy requests.
Format in \*path should be what/sortby/plot/jobtype/datatype?starttime=XXX&endtime=...
:arg path: path arguments are mandatory and this function should fail and return None,None if they are not found.
:arg kwargs: keyword are all optional, defaults are provided in all cases.
This function formats the query into a ProdMonQuery object, then passes it to GetPlotFromQuery.
Defaults for missing keyword args are 48h plot, png image 800x600@72dpi (although these defaults are applied within ProdMonQuery)
Also handles summary plots with paths of the form summary/{site,wf}/{plot,legend,map}
An appropriate query object is created for these and then they are generated by the normal code path."""
if len(path) == 3:
if path[0] in ('summary') and path[1] in ('site','wf','team') and path[2] in ('plot','legend','map','plot+legend'): # Deprecated table output, now handled in state
resulttype=path[2]
if path[1] == 'site':
query = ProdMonQuery('evts_read_any','site','pie','Any',type='png',width='400',height='300',group='top5site',period='48',notitle='1',nolabels='1')
elif path[1] == 'wf':
query = ProdMonQuery('evts_read_any','wf','pie','Any',type='png',width='400',height='300',group='top5wf',period='48',notitle='1',nolabels='1')
elif path[1] == 'team':
query = ProdMonQuery('evts_read_any','prod_team','pie','Any',type='png',width='400',height='300',period='48',notitle='1',nolabels='1')
else:
_logwarn("PRODMON_S: Invalid summary requested: %s"%path)
return None,None # Type of plot we don't understand
return self._getPlotFromQuery(query,resulttype)
elif len(path) == 5:
query = ProdMonQuery(*path,**attrs)
resulttype=path[4]
return self._getPlotFromQuery(query,resulttype)
elif len(path) >= 2:
if path[0]=='autocomplete':
ac_type = path[1]
ac_text = '/'.join(path[2:])
if ac_type in ('wf','dts','site','prod_team','prod_agent','exit_code'):
return self._autoComplete(ac_type,ac_text)
else:
_logwarn("PRODMON_S: Invalid autocomplete requested: %s"%path)
return None,None
_logwarn("PRODMON_S: Bad path length: %s"%path)
return None,None # Fall through if the path length didn't match 3 or 5, or the summary format was inconsistent
def _getResult(self,query):
"""
GetResult is a cachemaker function which actually gets data from the backend. HTTP errors are not actually handled here (logged and re-raised) since we don't want dead data to get cached - better to end up with HTTP 500 instead
"""
urlquery = query.getURL()
try:
url = "%s%s" % (ProdMonSource.url,urlquery)
data = urllib2.urlopen(url).read()
except urllib2.HTTPError,e:
_logwarn("PRODMON_S: HTTPError code=%s url=%s"%(e.code,url))
raise
except urllib2.URLError,e:
_logwarn("PRODMON_S: URLError reason=%s url=%s"%(e.reason,url))
raise
if ("Exception" in data.split("\n",1)[0]):
_logwarn("PRODMON_S: Dashboard returned exception url=%s"%url)
raise IOError, "Dashboard returned error %s in query %s" % (data.split("\n",1)[0],self.urlquery)
result = ProdMonQueryResult(data)
return result,1200
def _getWFList(self):
"""
Get the list of all known workflows from the wf_summary table.
Updated _getWFList for use with _fetchCache
"""
query = ProdMonQuery('wf_summary','wf','data','Any')
qresult = self.cache.lockedCacheFetch(query.cacheKey('PA_QUERY',False),self._getResult,query)
result = []
for item in qresult:
result.append(copy(item['WORKFLOW'].encode('ascii')))
return result,1800
