def normalizeSchema(schema, skip=None): """ Normalize the Db schema """ for i, c in enumerate(schema): nm = nmap.get(c) # if 'pandaid' in nm.lower() or 'time' in nm.lower(): continue if skip != None and nm.lower() in skip: continue schema[i] = nm return schema
def normalizeSchema(schema,skip=None): """ Normalize the Db schema """ for i,c in enumerate(schema): nm = nmap.get(c) # if 'pandaid' in nm.lower() or 'time' in nm.lower(): continue if skip != None and nm.lower() in skip: continue schema[i] = nm return schema
def normalizeDbSchema(db,schema,skip=['pandaid','metadata','jobparameters']): """ Normalize the Db schema """ description = db.describe('^%s$' % schema) params = {} ci = name2Index(description['header'],'column_name') for c in description['rows']: nm = nmap.get(c[ci]) c[ci] = nm # if 'pandaid' in nm.lower() or 'time' in nm.lower(): continue if nm.lower() in skip: continue params[nm] = None return (description,params)
def normalizeDbSchema(db,schema,skip=['pandaid','metadata','jobparameters']): """ Normalize the Db schema """ description = db.describe('^%s$' % schema) params = {} ci = name2Index(description['header'],'column_name') for c in description['rows']: nm = nmap.get(c[ci]) c[ci] = nm # if 'pandaid' in nm.lower() or 'time' in nm.lower(): continue if nm.lower() in skip: continue params[nm] = None return (description,params)
def doJson(self,table=None,column=None,db='pmt',doc=None): """ Describe the selected Panda table / column <br> <code>table = the Db table name regexp comma-separated patterns to check</code><br> <code>column = the Db table column name regexp comma-separated patterns to find</code><br> <code>doc = '*|all' - document all columns selected if available<br> = 'comma-separated list of the regexp' is to display the columns with the comments matched the regexp provided </code><br> """ title = 'Describe the Panda Db ' inp = False if table != None and table != 'None': title += "'%s*'" % table inp = True title +=" table" else: title +=" tables" if column != None and column !='None': title += " with '%s*' column " % column inp = True main = {} if inp: self.publishTitle(title) timer = Stopwatch.Stopwatch() main["buffer"] = {} main["buffer"]["db"] = db dbAccess = self._db.get(db) if dbAccess != None: main["buffer"]["method"] = "describe" main["buffer"]["params"] = (table,column,doc) main["buffer"]["type"] = False if not utils.isFilled(doc): tdes = dbAccess.describe(table,column) iclmn = utils.name2Index(tdes['header'],'COLUMN_NAME') rows = tdes['rows'] for h in rows: h[iclmn] = nmap.get(h[iclmn]) main["buffer"]["data"] = tdes else: docrows = [] if utils.isFilled(table): tdoc = dbAccess.pandatabs(table,comments=doc,select="table_name,'Table Description' as COLUMN_NAME,comments") iclmn = utils.name2Index(tdoc['header'],'COLUMN_NAME') docrows = tdoc['rows'] for h in docrows: h[iclmn] = nmap.get(h[iclmn]) tcol = dbAccess.pandacols(column,table,select='TABLE_NAME,COLUMN_NAME,COMMENTS', comments=doc) iclmn = utils.name2Index(tcol['header'],'COLUMN_NAME') for h in tcol['rows']: h[iclmn] = nmap.get(h[iclmn]) tcol['rows'] += docrows main["buffer"]["data"] = tcol self.publish( {'s-maxage':60000,'max-age': 6000}, role=pmRoles.cache()) else: self.publishTitle("Error: 'describe' - unknown Db name '%s' has been provided" % db) self.publish( {'s-maxage':0,'max-age': 0}, role=pmRoles.cache()) else: self.publishTitle("Error: 'describe' - no parameter has been provided. Define one at least. It should be either 'column' or 'table'") self.publish( {'s-maxage':0,'max-age': 0}, role=pmRoles.cache()) self.publish(main) self.publish( "%s/%s" % (self.server().fileScriptURL(),"taskBuffer/%s.js" % "describe"),role=pmRoles.script() )
def doJobs( self, pandaid=None, params=None, jobparam=None, table=None, hours=None, tstart=None, tend=None, field="modificationTime", days=None, format=None, jobtype=None, testsite=None, limit=1500, summary=None, minjob=1, dump=None, region=None, lfn=None, showcpu=None, ): ## Perform the query main = {} errorcolcheck = False selection = "*" vals = self.extraValuesFilled(self._alias) title = "PanDA Job(s)" if utils.isFilled(pandaid): hours = None days = None title += " %s " % pandaid if days != None: title += " for %.2f Day" % float(days) if days > 1: title += "s" if hours != None: if days == None: title += " for " title += " %d Hour" % int(hours) if hours > 1: title += "s" if vals != None: for k in vals.keys(): if vals[k] == "undefined": vals[k] = None elif ( vals[k] != None and k.lower() == "jobstatus" and ( vals[k].lower().find("prerun") >= 0 or vals[k].lower().find("finishing") >= 0 or vals[k].lower().find("unassigned") >= 0 or vals[k].lower().find("retried") >= 0 ) ): stvals = utils.parseArray(vals[k]) statvals = [] for v in stvals: if v.lower() == "prerun": statvals += ["defined", "assigned", "waiting", "activated", "pending", "sent", "starting"] elif v.lower() == "finishing": statvals += ["holding", "transferring"] elif v.lower() == "unassigned": vals["computingSite"] = "NULL" statvals += ["cancelled"] elif v.lower() == "retried": # See the slide 3 from Tadashi: https://indico.cern.ch/getFile.py/access?contribId=13&sessionId=11&resId=0&materialId=slides&confId=119171 vals["taskBufferErrorCode"] = [106, 107] statvals += ["failed"] else: statvals += [v] vals[k] = ",".join(statvals) cleanVal = dict((k, v) for k, v in vals.iteritems() if v is not None) vals = cleanVal if vals == None: vals = {} nav = "" certificate = None certusr = None try: cert = self.server().certificate() if cert != None: certusr = pmt.cleanUserID(cert) main["ceruser"] = certusr except: pass if vals.get("prodUserName") == "auto" and certusr != None: if cert != None: vals["prodUserName"] = certusr for v in vals: nav += "<b>%s</b>=%s " % (v, vals[v]) # nav += "<span id=linkjobid stule='display:inline'></span>" # nav +="<script>$(document).ready(function(){$('#linkjobid').html(utils().linkJobs(s.cloud,jobtype" # nav += jobtype if jobtype!=None else "undefined" # nav += ", s.jobStatus, " # nav += "%d" % hours if hours!=None else "undefined" # nav += ", s.computingSite,'Classic Page' ))});</script>"; if not utils.isFilled(summary): nav += "<b>%s</b>=%s " % ("limit", limit) self.publishNav(nav) if len(vals) == 0: vals = None if vals == None and jobparam == None: main["Description"] = self.description() else: if lfn != None and lfn != "undefined": lfnselect = "pandaid,lfn" lfnjobs = pmt.getJobLFN(select=lfnselect, table="new", lfn=lfn, pandaid=pandaid, type="*", limit=limit) if lfnjobs["rows"] > 0: pandaid = [ljobs[0] for ljobs in lfnjobs["rows"]] selection = {} if vals != None: selection.update(vals) if pandaid != None: selection.update({nmap.get("pandaid"): pandaid}) if region != None: selection.update({"region": region}) main["Selection"] = selection conditions = vals if conditions == None: conditions = {} cs = conditions.get("computingSite") if cs == None: site4region = self.getSite4Region(region) if site4region != None and len(site4region) > 0: conditions["computingSite"] = site4region else: pass # ignore the region parameter if utils.isValid(summary): jobtype = ( "production" if jobtype == None and jobparam.lower().find("taskid") >= 0 and jobparam.lower().find("jeditaskid") < 0 else jobtype ) jobs = pmt.getJobsAtt( pandaid, jobparam, conditions, None, hours, tstart, tend, field, days, format, jobtype, testsite, None, True, ) rows = jobs["rows"] indx = [] if len(rows) > 30: for i, r in enumerate(rows): try: if r[1] > minjob: indx.append(i) continue except: pass cleanrows = [rows[i] for i in indx] jobs["rows"] = cleanrows istatus = utils.name2Index(jobs["header"], "jobstatus") if istatus != None: statOrder = utils.name2Index(utils.jobStates) def byStatus(a, b): return statOrder[a[istatus]] - statOrder[b[istatus]] jobs["rows"] = sorted(jobs["rows"], cmp=byStatus) else: if jobparam == self._allparams and conditions.get(self._jobParametersName) == None: conditions[self._jobParametersName] = "any" jobs = pmt.getJobsAtt( pandaid, "PandaID," + jobparam, conditions, None, hours, tstart, tend, field, days, format, jobtype, testsite, limit, ) errorcolcheck = True if len(jobs) > 0: header = [nmap.get(h) for h in jobs["header"]] info = jobs["rows"] main["header"] = header # + self._extraFields timecols = [] errdict = None errorcol = not errorcolcheck for i, h in enumerate(header): if "time" in h.lower() and "cpuconsumptiontime" != h.lower(): timecols.append(i) if not errorcol and "error" in h.lower(): errorcol = True ## # add an artificial retryID ## header += ['retryID'] headindx = utils.name2Index(header) creationtDict = {} histograms = {} prodSourceLabel = headindx.get("prodSourceLabel") jobStatus = headindx.get("jobStatus") status4title = conditions.get("jobStatus") if not utils.isFilled(status4title): if len(info) == 1 and jobStatus != None: status4title = info[0][jobStatus] if utils.isFilled(status4title): title = status4title[0].upper() + status4title[1:] + " " + title prodtype = "production" in jobtype.lower() if jobtype != None else True taskidsum = prodtype if not utils.isValid(summary) and dump != "yes": jsets = setutils.factory(header, info, self._errorCodes, self._siteId, self._siteNicks) main["jobset"] = jsets.jobsets() for r in info: if not taskidsum and ((prodSourceLabel and r[prodSourceLabel] == "managed") and prodtype): taskidsum = True if errorcol and errorcolcheck: errdict = self.getErrors(r, headindx, errdict) for t in timecols: r[t] = utils.epochTime(r[t]) ## r += [None] # retryID # row = self.jobsetTime(utils.zip(header(info[r])),creationtDict,histograms) # info[r] = self.addFields(utils.zip(header(info[r])),creationtDict,histograms) # Action cleanUserID SSL_CLIENT_S_DN main["info"] = info if errorcolcheck: main["errorCodes"] = errdict main["colNames"] = sorted(header) main["jobsummary"] = copy.copy(self._jobSum) if taskidsum: main["jobsummary"] += [["Task ID", "taskID"]] # print utils.lineInfo(), "===============", self._description.get("JEDITASKID"), self._description.get("JediTaskID") # if self._description.get("JEDITASKID") or self._description.get("JediTaskID"): # main['jobsummary'] += [['Jedi ID','JediTaskID']] elif jobtype != "jedi": main["jobsummary"] += [["Jobsets", "jobsetID"]] # need constrain elif jobtype == "analysis": main["jobsummary"] += [["Analysis Tasks", "destinationDBlock"]] if showcpu == "yes": main["jobsummary"] += [["CPU Type", "cpuConsumptionUnit"]] self.publishTitle(title) return main
def doJson(self, table=None, column=None, db='pmt', doc=None): """ Describe the selected Panda table / column <br> <code>table = the Db table name regexp comma-separated patterns to check</code><br> <code>column = the Db table column name regexp comma-separated patterns to find</code><br> <code>doc = '*|all' - document all columns selected if available<br> = 'comma-separated list of the regexp' is to display the columns with the comments matched the regexp provided </code><br> """ title = 'Describe the Panda Db ' inp = False if table != None and table != 'None': title += "'%s*'" % table inp = True title += " table" else: title += " tables" if column != None and column != 'None': title += " with '%s*' column " % column inp = True main = {} if inp: self.publishTitle(title) timer = Stopwatch.Stopwatch() main["buffer"] = {} main["buffer"]["db"] = db dbAccess = self._db.get(db) if dbAccess != None: main["buffer"]["method"] = "describe" main["buffer"]["params"] = (table, column, doc) main["buffer"]["type"] = False if not utils.isFilled(doc): tdes = dbAccess.describe(table, column) iclmn = utils.name2Index(tdes['header'], 'COLUMN_NAME') rows = tdes['rows'] for h in rows: h[iclmn] = nmap.get(h[iclmn]) main["buffer"]["data"] = tdes else: docrows = [] if utils.isFilled(table): tdoc = dbAccess.pandatabs( table, comments=doc, select= "table_name,'Table Description' as COLUMN_NAME,comments" ) iclmn = utils.name2Index(tdoc['header'], 'COLUMN_NAME') docrows = tdoc['rows'] for h in docrows: h[iclmn] = nmap.get(h[iclmn]) tcol = dbAccess.pandacols( column, table, select='TABLE_NAME,COLUMN_NAME,COMMENTS', comments=doc) iclmn = utils.name2Index(tcol['header'], 'COLUMN_NAME') for h in tcol['rows']: h[iclmn] = nmap.get(h[iclmn]) tcol['rows'] += docrows main["buffer"]["data"] = tcol self.publish({ 's-maxage': 60000, 'max-age': 6000 }, role=pmRoles.cache()) else: self.publishTitle( "Error: 'describe' - unknown Db name '%s' has been provided" % db) self.publish({ 's-maxage': 0, 'max-age': 0 }, role=pmRoles.cache()) else: self.publishTitle( "Error: 'describe' - no parameter has been provided. Define one at least. It should be either 'column' or 'table'" ) self.publish({'s-maxage': 0, 'max-age': 0}, role=pmRoles.cache()) self.publish(main) self.publish( "%s/%s" % (self.server().fileScriptURL(), "taskBuffer/%s.js" % "describe"), role=pmRoles.script())
def doJobs(self, pandaid=None, params=None, jobparam=None, table=None, hours=None, tstart=None, tend=None, field='modificationTime', days=None, format=None, jobtype=None, testsite=None, limit=1500, summary=None, minjob=1, dump=None, region=None, lfn=None, showcpu=None): ## Perform the query main = {} errorcolcheck = False selection = '*' vals = self.extraValuesFilled(self._alias) title = 'PanDA Job(s)' if utils.isFilled(pandaid): hours = None days = None title += " %s " % pandaid if days != None: title += ' for %.2f Day' % float(days) if days > 1: title += "s" if hours != None: if days == None: title += " for " title += ' %d Hour' % int(hours) if hours > 1: title += "s" if vals != None: for k in vals.keys(): if vals[k] == 'undefined': vals[k] = None elif vals[k] != None and k.lower() == 'jobstatus' and ( vals[k].lower().find('prerun') >= 0 or vals[k].lower().find('finishing') >= 0 or vals[k].lower().find('unassigned') >= 0 or vals[k].lower().find('retried') >= 0): stvals = utils.parseArray(vals[k]) statvals = [] for v in stvals: if v.lower() == 'prerun': statvals += [ 'defined', 'assigned', 'waiting', 'activated', 'pending', 'sent', 'starting' ] elif v.lower() == 'finishing': statvals += ['holding', 'transferring'] elif v.lower() == 'unassigned': vals['computingSite'] = 'NULL' statvals += ['cancelled'] elif v.lower() == 'retried': # See the slide 3 from Tadashi: https://indico.cern.ch/getFile.py/access?contribId=13&sessionId=11&resId=0&materialId=slides&confId=119171 vals['taskBufferErrorCode'] = [106, 107] statvals += ['failed'] else: statvals += [v] vals[k] = ','.join(statvals) cleanVal = dict( (k, v) for k, v in vals.iteritems() if v is not None) vals = cleanVal if vals == None: vals = {} nav = '' certificate = None certusr = None try: cert = self.server().certificate() if cert != None: certusr = pmt.cleanUserID(cert) main['ceruser'] = certusr except: pass if vals.get('prodUserName') == 'auto' and certusr != None: if cert != None: vals['prodUserName'] = certusr for v in vals: nav += "<b>%s</b>=%s " % (v, vals[v]) # nav += "<span id=linkjobid stule='display:inline'></span>" # nav +="<script>$(document).ready(function(){$('#linkjobid').html(utils().linkJobs(s.cloud,jobtype" # nav += jobtype if jobtype!=None else "undefined" # nav += ", s.jobStatus, " # nav += "%d" % hours if hours!=None else "undefined" # nav += ", s.computingSite,'Classic Page' ))});</script>"; if not utils.isFilled(summary): nav += "<b>%s</b>=%s " % ('limit', limit) self.publishNav(nav) if len(vals) == 0: vals = None if vals == None and jobparam == None: main['Description'] = self.description() else: if lfn != None and lfn != "undefined": lfnselect = 'pandaid,lfn' lfnjobs = pmt.getJobLFN(select=lfnselect, table='new', lfn=lfn, pandaid=pandaid, type='*', limit=limit) if lfnjobs['rows'] > 0: pandaid = [ljobs[0] for ljobs in lfnjobs['rows']] selection = {} if vals != None: selection.update(vals) if pandaid != None: selection.update({nmap.get('pandaid'): pandaid}) if region != None: selection.update({'region': region}) main['Selection'] = selection conditions = vals if conditions == None: conditions = {} cs = conditions.get('computingSite') if cs == None: site4region = self.getSite4Region(region) if site4region != None and len(site4region) > 0: conditions['computingSite'] = site4region else: pass # ignore the region parameter if utils.isValid(summary): jobtype = 'production' if jobtype == None and jobparam.lower( ).find('taskid') >= 0 and jobparam.lower().find( 'jeditaskid') < 0 else jobtype jobs = pmt.getJobsAtt(pandaid, jobparam, conditions, None, hours, tstart, tend, field, days, format, jobtype, testsite, None, True) rows = jobs['rows'] indx = [] if len(rows) > 30: for i, r in enumerate(rows): try: if r[1] > minjob: indx.append(i) continue except: pass cleanrows = [rows[i] for i in indx] jobs['rows'] = cleanrows istatus = utils.name2Index(jobs['header'], 'jobstatus') if istatus != None: statOrder = utils.name2Index(utils.jobStates) def byStatus(a, b): return statOrder[a[istatus]] - statOrder[b[istatus]] jobs['rows'] = sorted(jobs['rows'], cmp=byStatus) else: if jobparam == self._allparams and conditions.get( self._jobParametersName) == None: conditions[self._jobParametersName] = 'any' jobs = pmt.getJobsAtt(pandaid, 'PandaID,' + jobparam, conditions, None, hours, tstart, tend, field, days, format, jobtype, testsite, limit) errorcolcheck = True if len(jobs) > 0: header = [nmap.get(h) for h in jobs['header']] info = jobs['rows'] main['header'] = header # + self._extraFields timecols = [] errdict = None errorcol = not errorcolcheck for i, h in enumerate(header): if 'time' in h.lower( ) and 'cpuconsumptiontime' != h.lower(): timecols.append(i) if not errorcol and 'error' in h.lower(): errorcol = True ## # add an artificial retryID ## header += ['retryID'] headindx = utils.name2Index(header) creationtDict = {} histograms = {} prodSourceLabel = headindx.get('prodSourceLabel') jobStatus = headindx.get('jobStatus') status4title = conditions.get('jobStatus') if not utils.isFilled(status4title): if len(info) == 1 and jobStatus != None: status4title = info[0][jobStatus] if utils.isFilled(status4title): title = status4title[0].upper( ) + status4title[1:] + " " + title prodtype = 'production' in jobtype.lower( ) if jobtype != None else True taskidsum = prodtype if not utils.isValid(summary) and dump != 'yes': jsets = setutils.factory(header, info, self._errorCodes, self._siteId, self._siteNicks) main['jobset'] = jsets.jobsets() for r in info: if not taskidsum and ( (prodSourceLabel and r[prodSourceLabel] == 'managed') and prodtype): taskidsum = True if errorcol and errorcolcheck: errdict = self.getErrors(r, headindx, errdict) for t in timecols: r[t] = utils.epochTime(r[t]) ## r += [None] # retryID # row = self.jobsetTime(utils.zip(header(info[r])),creationtDict,histograms) # info[r] = self.addFields(utils.zip(header(info[r])),creationtDict,histograms) # Action cleanUserID SSL_CLIENT_S_DN main['info'] = info if errorcolcheck: main['errorCodes'] = errdict main['colNames'] = sorted(header) main['jobsummary'] = copy.copy(self._jobSum) if taskidsum: main['jobsummary'] += [['Task ID', 'taskID']] # print utils.lineInfo(), "===============", self._description.get("JEDITASKID"), self._description.get("JediTaskID") # if self._description.get("JEDITASKID") or self._description.get("JediTaskID"): # main['jobsummary'] += [['Jedi ID','JediTaskID']] elif jobtype != 'jedi': main['jobsummary'] += [['Jobsets', 'jobsetID']] # need constrain elif jobtype == 'analysis': main['jobsummary'] += [[ 'Analysis Tasks', 'destinationDBlock' ]] if showcpu == 'yes': main['jobsummary'] += [['CPU Type', 'cpuConsumptionUnit']] self.publishTitle(title) return main