def getStatsSummary(self): if "PA_WFSUMMARY" in self.cache: table = self.cache.cacheFetch('PA_WFSUMMARY',self._getWFSummary) result = {} result['workflow_count']=len(table) result['total_unmerged']=0 result['total_merged']=0 time_per_event = [] for t in table: result['total_unmerged']+=int(table[t]['UNMERGED_EVTS']) result['total_merged']+=int(table[t]['MERGED_EVTS']) time_per_event.append(float(table[t]['TIME_PER_EVENT'])) if len(time_per_event)>0: result['avg_time_per_event']='%.2f'%(sum(time_per_event)/len(time_per_event)) result['min_time_per_event']='%.2f'%min(time_per_event) result['max_time_per_event']='%.2f'%max(time_per_event) else: _logwarn("PRODMON_S: WFSummary table contains no workflows") result['avg_time_per_event']='No workflows found' result['min_time_per_event']='No workflows found' result['max_time_per_event']='No workflows found' return result else: self.cache.asyncCacheFill('PA_WFSUMMARY',self._getWFSummary) return { 'workflow_count':'Data Unavailable', 'total_unmerged':'Data Unavailable', 'total_merged':'Data Unavailable', 'avg_time_per_event':'Data Unavailable', 'max_time_per_event':'Data Unavailable', 'min_time_per_event':'Data Unavailable' }
def extension(modules, what, *args): for m in modules: ctor = getattr(m, what, None) if ctor: return ctor(*args) _logwarn("extension '%s' not found" % what) return None
def run(self): while True: self.lock.acquire() stopme = self.stopme self.lock.release() if stopme: break rximg = re.compile(r"<img\s+(.*?)/?>", re.I) for s in self.infosrc: img = 0 imgrefs = {} try: data = urllib2.urlopen(s['html']).read().replace("\n", " ") for imgtag in re.findall(rximg, data): while True: m = re.match( "^([-A-Za-z0-9]+)=(\"[^\"]*\"|'[^']*'|\S+)\s*/?", imgtag) if not m: break arg = m.group(2) if len(arg ) >= 2 and arg[0] == '"' and arg[-1] == '"': arg = arg[1:-1] elif len(arg) >= 2 and arg[0] == "'" and arg[ -1] == "'": arg = arg[1:-1] if m.group(1) == "src" and arg.find( s['match']) >= 0: imgrefs[s['images'][img]] = arg img = img + 1 imgtag = imgtag[m.end():] except Exception, e: _logwarn("failed to fetch document '%s': %s" % (s['html'], str(e))) imgdata = {} imgurl = None for (name, url) in imgrefs.iteritems(): try: imgurl = urlparse.urljoin(s['html'], url) img = urllib2.urlopen(imgurl) info = img.info() imgdata[name] = { 'data': img.read(), 'type': info['Content-Type'] } except Exception, e: _logwarn("failed to fetch image %s from '%s': %s" % (name, imgurl, str(e))) imgdata[name] = None self.lock.acquire() for (name, info) in imgdata.iteritems(): self.images[name] = info self.lock.release()
def _getResult(self,query): """ GetResult is a cachemaker function which actually gets data from the backend. HTTP errors are not actually handled here (logged and re-raised) since we don't want dead data to get cached - better to end up with HTTP 500 instead """ urlquery = query.getURL() try: url = "%s%s" % (ProdMonSource.url,urlquery) data = urllib2.urlopen(url).read() except urllib2.HTTPError,e: _logwarn("PRODMON_S: HTTPError code=%s url=%s"%(e.code,url)) raise
def _queryData(self, db, label, validfor, q, *args): cache = db['cache'].setdefault(label, {}) key = ":".join(str(i) for i in args) now = time.time() if key not in cache or now > cache[key]['valid']: try: if not db['db']: db['db'] = db['def']['connect']() c = db['db'].cursor(); c.arraysize = 10000 cache[key] = {'valid': now + validfor, 'data': q(c, *args) } except Exception, e: _logwarn("failed to execute query %s for '%s' in %s: %s; disconnecting" % (label, key, db['def']['label'], str(e))) db['db'] = None raise
def run(self): while True: self.lock.acquire() stopme = self.stopme self.lock.release() if stopme: break rximg = re.compile(r"<img\s+(.*?)/?>", re.I) for s in self.infosrc: img = 0 imgrefs = {} try: data = urllib2.urlopen(s['html']).read().replace("\n", " ") for imgtag in re.findall(rximg, data): while True: m = re.match("^([-A-Za-z0-9]+)=(\"[^\"]*\"|'[^']*'|\S+)\s*/?", imgtag) if not m: break arg = m.group(2) if len(arg) >= 2 and arg[0] == '"' and arg[-1] == '"': arg = arg[1:-1] elif len(arg) >= 2 and arg[0] == "'" and arg[-1] == "'": arg = arg[1:-1] if m.group(1) == "src" and arg.find(s['match']) >= 0: imgrefs[s['images'][img]] = arg img = img+1 imgtag = imgtag[m.end():] except Exception, e: _logwarn("failed to fetch document '%s': %s" % (s['html'], str(e))) imgdata = {} imgurl = None for (name, url) in imgrefs.iteritems(): try: imgurl = urlparse.urljoin(s['html'], url) img = urllib2.urlopen(imgurl) info = img.info() imgdata[name] = {'data': img.read(), 'type': info['Content-Type']} except Exception, e: _logwarn("failed to fetch image %s from '%s': %s" % (name, imgurl, str(e))) imgdata[name] = None self.lock.acquire() for (name, info) in imgdata.iteritems(): self.images[name] = info self.lock.release()
def _queryData(self, db, label, validfor, q, *args): cache = db['cache'].setdefault(label, {}) key = ":".join(str(i) for i in args) now = time.time() if key not in cache or now > cache[key]['valid']: try: if not db['db']: db['db'] = db['def']['connect']() c = db['db'].cursor() c.arraysize = 10000 cache[key] = {'valid': now + validfor, 'data': q(c, *args)} except Exception, e: _logwarn( "failed to execute query %s for '%s' in %s: %s; disconnecting" % (label, key, db['def']['label'], str(e))) db['db'] = None raise
def insert(self,key,lifetime,data): """ Insert a new key into the cache, with associated expiry lifetime, size and data. Triggers volume cleaning if the new cache size exceeds the maximum. @param key: key for the new data - should be a string @param lifetime: desired cache lifetime of this object in seconds @param data: free data object. Preferably doesn't refer to other objects in the cache, or size calculations will become impossible. """ self.lock.acquire() try: try: data = cPickle.dumps(data,cPickle.HIGHEST_PROTOCOL) self._insert(key,lifetime,len(data),data) except cPickle.PickleError: _logwarn("CACHE: Pickling error with key=%s"%key) pass finally: self.lock.release()
def legend(self): legend = self._getLegend() if len(legend)>=int(self.get('max_legend')): _logwarn("PLOT: Legend too long (%s) - truncating"%len(legend)) legend = legend[:int(self.get('max_legend'))] legend.append((int(self.get('max_legend')),'Some entries truncated',{'facecolor':'#000000'},0)) if len(legend)==0: (fig, canvas, w, h) = self.canvas() return self.save(fig,canvas) namelen = max(map(lambda x: len(x[1]),legend)) i = len(legend)+1 dpif = float(self.get('dpi')) / 72 wcol = 10 + (2 + 0.6*namelen)*self.font.get_size()*dpif ncol = max(1, math.floor(float(self.get('width')) / wcol)) wcol = float(self.get('width')) / ncol nrow = int((i + ncol - 1) / ncol) self.height = (nrow + 1) * self.font.get_size() * 1.4 * dpif (fig, canvas, w, h) = self.canvas() step = self.font.get_size() * 1.4 * dpif / h x = 0 y = 0.5 * self.font.get_size() * dpif / h xbox = 10./w wbox = self.font.get_size() * dpif / w * 1.25 hbox = self.font.get_size() * dpif / h * 0.75 xtext = xbox + wbox*1.5 row = 0 for item in legend: fig.text(x + xtext, 1 - y, item[1], horizontalalignment='left', verticalalignment='top', fontproperties=self.font) p = Rectangle((x + xbox, 1-y-hbox), wbox, hbox, linewidth=0.5, fill=True) p.update(item[2]) fig._set_artist_props(p) fig.patches.append(p) y += step row += 1 if row == nrow: x += wcol/w y = 0.5 * self.font.get_size() * dpif / h row = 0 return self.save(fig, canvas)
def _databases(self, names): self.lock.acquire() try: instances = {} for namegroup in names: if namegroup in self.db: instances[namegroup] = self.db[namegroup] else: for (x, db) in self.db.iteritems(): if db['def']['class'] == namegroup: instances[x] = db valid = [] for i in instances.values(): (db, spec, ok) = (i['db'], i['def'], False) if db: try: c = db.cursor() c.execute("select sysdate from dual") c.fetchall() ok = True except Exception, e: _logwarn("stale database connection %s, error was %s" % (spec['label'], str(e))) i['db'] = None if not ok: try: # _loginfo("(re)connecting to database %s" % spec['label']) # db = spec['connect']() # c = db.cursor() # c.execute("select sysdate from dual") # c.fetchall() # i['db'] = db ok = True except Exception, e: _logwarn( "disabling database %s due to connection error %s" % (spec['label'], str(e))) if ok: valid.append(i)
def retrieve(self,key,default=None): #_loginfo("CACHE: Retrieve key=%s"%key) """ Attempts to retrieve the data associated with a key from the cache. Returns default (None) if not found. @param key: cache key to retrieve data for @param default: what to return if the key is not in the cache (None) """ #print "PC: Getting key [%s]" % (key) result=default self.lock.acquire() try: if self._exists(key): try: result = cPickle.loads(self.cache[key][2]) except cPickle.PickleError: _logwarn("CACHE: Unpickling error with key=%s"%key) pass finally: self.lock.release() return result
def _databases(self, names): self.lock.acquire() try: instances = {} for namegroup in names: if namegroup in self.db: instances[namegroup] = self.db[namegroup] else: for (x, db) in self.db.iteritems(): if db['def']['class'] == namegroup: instances[x] = db valid = [] for i in instances.values(): (db, spec, ok) = (i['db'], i['def'], False) if db: try: c = db.cursor() c.execute("select sysdate from dual") c.fetchall() ok = True except Exception, e: _logwarn("stale database connection %s, error was %s" % (spec['label'], str(e))) i['db'] = None if not ok: try: # _loginfo("(re)connecting to database %s" % spec['label']) # db = spec['connect']() # c = db.cursor() # c.execute("select sysdate from dual") # c.fetchall() # i['db'] = db ok = True except Exception, e: _logwarn("disabling database %s due to connection error %s" % (spec['label'], str(e))) if ok: valid.append(i)
def _queryTimeSeries(self, db, label, validfor, q, *args): cache = db['cache'].setdefault(label, {}) (bin, start, end) = args[0:3] found = None for key in cache: if key[0] == bin and key[1] <= start and key[2] >= end: if not found or found[1] < key[1] or found[2] > key[2]: found = key now = time.time() key = (found or (bin, start, end)) if key not in cache or now > cache[key]['valid']: try: if not db['db']: db['db'] = db['def']['connect']() c = db['db'].cursor(); c.arraysize = 10000 cache[key] = { 'valid': now + validfor, 'data': q(c, *args) } except Exception, e: _logwarn("failed to execute query %s for '%s' in %s: %s; disconnecting" % (label, key, db['def']['label'], str(e))) db['db'] = None raise
def _getPlotFromQuery(self,query,resulttype): """ GetPlotFromQuery evaluates a query object, testing it for validity before using the cache access functions to return the data. """ if not (resulttype in ('plot','legend','map','plot+legend','thumbnail')): #or (query['plot']=='data' and resulttype=='table')): _logwarn("PRODMON_S: Unknown or inconsistent result type: %s"%resulttype) return None,None if not query.isValid(): _logwarn("PRODMON_S: Invalid query.") return None,None plot = query['plot'] if not plot in ProdMonSource.params['plot']: _logwarn("PRODMON_S: Unknown plot type: %s"%plot) return None,None plotdata = self.cache.lockedCacheFetch(query.cacheKey('PA_PLOT'),self._plotMaker,query) #_loginfo("PRODMON_S: Result plotsize=%s legendsize=%s mapsize=%s plot+legendsize=%s thumbnailsize=%s" %(len(plotdata[0]),len(plotdata[1]),len(plotdata[2]),len(plotdata[3]),len(plotdata[4]))) if resulttype=='plot': return ProdMonSource.types[query['type']],plotdata[0] elif resulttype=='legend': return ProdMonSource.types[query['type']],plotdata[1] elif resulttype=='map': return 'text/plain',plotdata[2] elif resulttype=='plot+legend': return ProdMonSource.types[query['type']],plotdata[3] elif resulttype=='thumbnail': return ProdMonSource.types[query['type']],plotdata[4] else: pass # Already validated that we have one of the above
def _verify(self,key,value): timepattern = re.compile('^[0-9]{4}-[0-9]{1,2}-[0-9]{1,2} [0-9]{2}:[0-9]{2}:[0-9]{2}$') verifier = { 'sortby': lambda x: x in ProdMonSource.params['sortby'], 'what': lambda x: x in ProdMonSource.params['what'], 'job_type': lambda x: x in ProdMonSource.params['job_type'], 'graph': lambda x: x in ProdMonSource.params['plot'], 'use_period': lambda x: x in ('true','false'), 'period': lambda x: x.isdigit() and int(x)>0 and int(x)<=8760, 'image_height': lambda x: x.isdigit() and int(x)>0 and int(x)<=2000, 'image_width': lambda x: x.isdigit() and int(x)>0 and int(x)<=10000, 'image_dpi': lambda x: x.isdigit() and int(x)>0 and int(x)<=600, 'starttime': lambda x: timepattern.match(x), 'endtime': lambda x: timepattern.match(x) } if key in verifier: if verifier[key](value): return value else: _logwarn("PRODMON_W: Verification failed for %s=%s"%(key,value)) return deepcopy(ProdMonWorkspace.ProdMonPlot.plotdef[key]) else: return value
def _queryTimeSeries(self, db, label, validfor, q, *args): cache = db['cache'].setdefault(label, {}) (bin, start, end) = args[0:3] found = None for key in cache: if key[0] == bin and key[1] <= start and key[2] >= end: if not found or found[1] < key[1] or found[2] > key[2]: found = key now = time.time() key = (found or (bin, start, end)) if key not in cache or now > cache[key]['valid']: try: if not db['db']: db['db'] = db['def']['connect']() c = db['db'].cursor() c.arraysize = 10000 cache[key] = {'valid': now + validfor, 'data': q(c, *args)} except Exception, e: _logwarn( "failed to execute query %s for '%s' in %s: %s; disconnecting" % (label, key, db['def']['label'], str(e))) db['db'] = None raise
def plot(self,*path,**attrs): """Landing function for plotfairy requests. Format in \*path should be what/sortby/plot/jobtype/datatype?starttime=XXX&endtime=... :arg path: path arguments are mandatory and this function should fail and return None,None if they are not found. :arg kwargs: keyword are all optional, defaults are provided in all cases. This function formats the query into a ProdMonQuery object, then passes it to GetPlotFromQuery. Defaults for missing keyword args are 48h plot, png image 800x600@72dpi (although these defaults are applied within ProdMonQuery) Also handles summary plots with paths of the form summary/{site,wf}/{plot,legend,map} An appropriate query object is created for these and then they are generated by the normal code path.""" if len(path) == 3: if path[0] in ('summary') and path[1] in ('site','wf','team') and path[2] in ('plot','legend','map','plot+legend'): # Deprecated table output, now handled in state resulttype=path[2] if path[1] == 'site': query = ProdMonQuery('evts_read_any','site','pie','Any',type='png',width='400',height='300',group='top5site',period='48',notitle='1',nolabels='1') elif path[1] == 'wf': query = ProdMonQuery('evts_read_any','wf','pie','Any',type='png',width='400',height='300',group='top5wf',period='48',notitle='1',nolabels='1') elif path[1] == 'team': query = ProdMonQuery('evts_read_any','prod_team','pie','Any',type='png',width='400',height='300',period='48',notitle='1',nolabels='1') else: _logwarn("PRODMON_S: Invalid summary requested: %s"%path) return None,None # Type of plot we don't understand return self._getPlotFromQuery(query,resulttype) elif len(path) == 5: query = ProdMonQuery(*path,**attrs) resulttype=path[4] return self._getPlotFromQuery(query,resulttype) elif len(path) >= 2: if path[0]=='autocomplete': ac_type = path[1] ac_text = '/'.join(path[2:]) if ac_type in ('wf','dts','site','prod_team','prod_agent','exit_code'): return self._autoComplete(ac_type,ac_text) else: _logwarn("PRODMON_S: Invalid autocomplete requested: %s"%path) return None,None _logwarn("PRODMON_S: Bad path length: %s"%path) return None,None # Fall through if the path length didn't match 3 or 5, or the summary format was inconsistent
def _getResult(self,query): """ GetResult is a cachemaker function which actually gets data from the backend. HTTP errors are not actually handled here (logged and re-raised) since we don't want dead data to get cached - better to end up with HTTP 500 instead """ urlquery = query.getURL() try: url = "%s%s" % (ProdMonSource.url,urlquery) data = urllib2.urlopen(url).read() except urllib2.HTTPError,e: _logwarn("PRODMON_S: HTTPError code=%s url=%s"%(e.code,url)) raise except urllib2.URLError,e: _logwarn("PRODMON_S: URLError reason=%s url=%s"%(e.reason,url)) raise if ("Exception" in data.split("\n",1)[0]): _logwarn("PRODMON_S: Dashboard returned exception url=%s"%url) raise IOError, "Dashboard returned error %s in query %s" % (data.split("\n",1)[0],self.urlquery) result = ProdMonQueryResult(data) return result,1200 def _getWFList(self): """ Get the list of all known workflows from the wf_summary table. Updated _getWFList for use with _fetchCache """ query = ProdMonQuery('wf_summary','wf','data','Any') qresult = self.cache.lockedCacheFetch(query.cacheKey('PA_QUERY',False),self._getResult,query) result = [] for item in qresult: result.append(copy(item['WORKFLOW'].encode('ascii'))) return result,1800