def runBackEnd(): # this will store the exit status for the json report retValues = 0, "OK" # flag in case we miss the info for some runs unknownRun = False unknownRunMsg = "" tier0Das = tier0DasInterface.Tier0DasInterface(tier0DasSrc) try: nextPromptRecoRun = tier0Das.firstConditionSafeRun() print "Tier0 DAS next run for prompt reco:",nextPromptRecoRun gtFromPrompt = tier0Das.promptGlobalTag(referenceDataset) print " GT for dataset: ", referenceDataset, "run:", str(nextPromptRecoRun), ":", gtFromPrompt except Exception as error: print '*** Error: Tier0-DAS query has failed' print error return 102, "Error: Tier0-DAS query has failed: " + str(error) print len(gtFromPrompt) if(len(gtFromPrompt) == 0): return 202, "No " + referenceDataset + " datset for run: " + str(nextPromptRecoRun) + " -> failed to get the GT name" gtName = gtFromPrompt.split('::')[0] gtConfFile = gtName + '.conf' if not gtTools.confFileFromDB(gtName, gtConfFile, gtconnstring, passwdfile): return 201, "GT: " + gtFromPrompt + " could not be found in ORACLE!" # create the collection of tags tagCollection = gtTools.GTEntryCollection() gtTools.fillGTCollection(gtName+'.conf', gtName, tagCollection) tagsTomonitor = [] print "Tags to be monitored: " for record in monitoredrecords.split(','): label = '' if ':' in record: label = record.split(':')[1] record = record.split(':')[0] rcdId = gtTools.RcdID([record, label]) if not tagCollection.hasRcdID(rcdId): print "Error: rcd: " + rcdIdn + " not found in GT: " + gtName else: print ' ', tagCollection.getByRcdID(rcdId) tagsTomonitor.append(tagCollection.getByRcdID(rcdId)) # -------------------------------------------------------------------------------- # --- read the cache allCachedRuns = o2oMonitoringTools.readCache(cacheFileName) cachedRuns = allCachedRuns[0] runReports = allCachedRuns[1] unknownRun = False # this is set to true only if one run can not be processed unknownRunMsg = '' # -------------------------------------------------------------------------------- # --- get the last cached run if len(cachedRuns) != 0: cachedRuns.sort() else: cachedRuns.append(1) lastCachedRun = cachedRuns[len(cachedRuns)-1] #lastCachedRun = 191419 print "last cached run #: " + str(lastCachedRun) # -------------------------------------------------------------------------------- # --- get the list of collision runs from RR (only for the runs not yet cached) runList = [] try: # FIXME: do we need to restrict to Collision12? #runList = RunRegistryTools.getRunListRR3(lastCachedRun+1,"Online", "Commissioning12") runList = RunRegistryTools.getRunListRR3(lastCachedRun+1, rrDatasetName, rrRunClassName) except Exception as error: print '*** Error 1: RR query has failed' print error return 101, "Error: failed to get collision runs from RunRegistry: " + str(error) print runList # -------------------------------------------------------------------------------- # --- check O2O and DB tag status for each record threshold = datetime.timedelta(hours=int(thresholdLastWrite)) thresholdSince = datetime.timedelta(hours=int(thresholdLastSince)) today = datetime.datetime.today() fromUTCToLocal = datetime.timedelta(hours=2) recordandlastsince = {} tableTitle = ["# run", "start-time", "end-time"] rcdJson = o2oMonitoringTools.O2ORecordJson("o2oMonitor") for entry in tagsTomonitor: print "- Tag:", entry tagName = entry.tagName() accountName = entry.account() recordName = entry.record() tableTitle.append(recordName) # create the report for this given record rcdRep = o2oMonitoringTools.RecordReport(recordName) rcdRep.setTagAndAccount(tagName, accountName) nDays = 1 nSec = nDays*24*60*60 popLog = popConLog.PopCon_Monitoring_last_updates(interval=nSec) # 0. get the last time the O2O run o2oLogfiles = {} for rcdEntry in o2oLogfileList.split(','): key = rcdEntry.split(':')[0] logFileForKey = rcdEntry.split(':')[1] o2oLogfiles[key] = logFileForKey jobData = popLog.PopConJobRunTime(authfile=passwdfile + "/authentication.xml", logFile=o2oLogfiles[recordName]) if len(jobData) != 0: lastO2ORun = jobData[0][0] + fromUTCToLocal previouO2ORun = jobData[0][1] + fromUTCToLocal runO2OAge = today - lastO2ORun statusForRpt = "OK" print " - Last O2O run on: " + str(lastO2ORun) + " (" + str(runO2OAge) + " ago)" if runO2OAge > 2*(lastO2ORun - previouO2ORun): print " " + colorPrintTools.error("Error") + ": the O2O for rcd " + recordName + " is not running since a while (" + str(runO2OAge) + ")" statusForRpt = "ERROR" if 2050 > retValues[0]: retValues = 2050, "Error: the O2O for rcd " + recordName + " is not running since a while (" + str(runO2OAge) + ")" rcdRep.setLastO2ORun(lastO2ORun, runO2OAge, statusForRpt) else: print "Error: No O2O job logs for tag: " + tagName + " in account: " + accountName + " could be found in the PopConLogs" if 2051 > retValues[0]: retValues = 2051, "Error: no O2O job logs found for rcd: " + recordName # 1. get the last updates from PopConLogger # FIXME: the auth.xml can it be read from a central place? logData = popLog.PopConRecentActivityRecorded(authfile=passwdfile + "/authentication.xml", account=accountName, iovtag=tagName) if len(logData['data']) != 0: datestring = logData['data'][0][1] status = logData['data'][0][6] token = logData['data'][0][8] #datelastupdate = datetime.datetime.strptime(datestring,"%B, %dnd %Y %H:%M:%S") + fromUTCToLocal datelastupdate = dateutil.parser.parse(datestring) + fromUTCToLocal updateage = today - datelastupdate statusForRpt = "OK" print " - Last O2O wrote on: " + str(datelastupdate) + " (" + str(updateage) + " ago)" print " status:",status, "payload token:", token.split("<br>")[4] if updateage > threshold: print " " + colorPrintTools.warning("Warning") + ": O2O is not writing since a while!" statusForRpt = "OLD" if 2001 > retValues[0]: retValues = 2001, "Warning: the O2O for rcd: " + recordName + " is not writing since a while!" if status != 'OK': print " Warning: O2O status is: " + status + "!" statusForRpt = "ERROR" if 2002 > retValues[0]: retValues = 2002, "Error: the O2O status for rcd: " + recordName + " is " + status + "!" rcdRep.setLastO2OWrite(datelastupdate, updateage, statusForRpt) else: print "Error: No O2O updates to tag: " + tagName + " in account: " + accountName + " could be found in the PopConLogs" if 2010 > retValues[0]: retValues = 2010, "Error: no O2O updates logged for rcd: " + recordName #rcdRep.setLastO2OWrite(datelastupdate, updateage, "ERROR") # 2. check the status of the tag outputAndStatus = gtTools.listIov(entry.getOraclePfn(False), tagName, passwdfile) iovtable = gtTools.IOVTable() iovtable.setFromListIOV(outputAndStatus[1]) datesince = iovtable.lastIOV().sinceDate() sinceage = today - datesince print " - Last IOV since:", datesince, "(" + str(sinceage),"ago)" print " with token: [" + iovtable.lastIOV().token() +"]"#.split("][")[4] recordandlastsince[recordName] = datesince #print iovtable.lastIOV() stat = "OK" if sinceage > thresholdSince: stat = "OLD" if 2101 > retValues[0]: retValues = 2101, "Error: the last IOV of rcd: " + recordName + " is OLD (since: " + str(datesince) + ")!" rcdRep.setLastSince(datesince, sinceage, stat) rcdJson.addRcd(gtTools.RcdID([recordName,""]), rcdRep) # -------------------------------------------------------------------------------- # --- Write the Rcd status to cache rcdJson.writeJsonFile(webArea) # -------------------------------------------------------------------------------- # --- check the status for each of the Collision runs for run in runList: if run == 167551:continue print "-- run #: " + colorPrintTools.blue(str(run)) #print run # get the information from runInfo runInfo = None try: runInfo = RunInfo.getRunInfoStartAndStopTime(runinfoTag, "", run) except Exception as error: print '*** Error XXX: RunInfo query failed!' print error unknownRun = True unknownRunMsg = "Error: can not get report for run: " + str(run) + ", since run-info query failed: " + str(error) print unknownRunMsg continue rRep = o2oMonitoringTools.RunReportTagCheck() rRep.setRunNumber(runInfo.run()) #rRep.setStartTime(runInfo.startTime()) rRep.setRunInfoContent(runInfo) deltaTRun = runInfo.stopTime() - runInfo.startTime() deltaTRunH = deltaTRun.seconds/(60.*60.) print " start: " + str(runInfo.startTime()) + " stop: " + str(runInfo.stopTime()) + " lenght (h): " + str(deltaTRunH) #pageWriter.setNextPromptReco(runInfo.run(), runInfo.startTime(), runInfo.stopTime(), deltaTRunH) for entry in tagsTomonitor: recordName = entry.record() datesince = recordandlastsince[recordName] if runInfo.stopTime() <= datesince: print " rcd: ", recordName,":",colorPrintTools.ok("OK") rRep.addRecordAndStatus(recordName, 0) elif runInfo.startTime() < datesince and runInfo.stopTime() > datesince: print " rcd: ", recordName,":",colorPrintTools.warning("partially covered!") rRep.addRecordAndStatus(recordName, 0.5) else: print " rcd: ", recordName,":",colorPrintTools.error("not covered!") rRep.addRecordAndStatus(recordName, 1) runReports.append(rRep) # print "---------------------------------------------------------" #print gtEntry, #print " # of updates:", len(logData['data']) #if gtEntry.updateType() != 1: # listofchanges.append(str(gtEntry) + " # of updates: " + str(len(logData['data']))) #else: # listofchangesO2O.append(str(gtEntry) + " # of updates: " + str(len(logData['data']))) # -------------------------------------------------------------------------------- # --- write to cache and to log runReports.sort(key=lambda rr: rr._runnumber) tableForCache =[] tableForCache.append(tableTitle) tableForLog =[] tableForLog.append(tableTitle) for rep in runReports: if int(rep.runNumber()) < int(nextPromptRecoRun): tableForCache.append(rep.getList()) tableForLog.append(rep.getList()) #out = sys.stdout print "writing cache file: " + cacheFileName cacheFile = file(cacheFileName,"w") tableWriter.pprint_table(cacheFile, tableForCache) cacheFile.close() out = sys.stdout logFile = file(webArea + "log.txt","w") tableWriter.pprint_table(logFile, tableForLog) logFile.close() status = retValues[0] message = retValues[1] if status == 0 and unknownRun: status = 10 message = unknownRunMsg return status, message
def getRunReport(runinfoTag, run, promptCalibDir, fileList, iovtableByRun_oracle, iovtableByLumi_oracle): #print run # input: runInfoTag, run, fileList, iovlist runInfo = None try: runInfo = RunInfo.getRunInfoStartAndStopTime(runinfoTag, '', run) except Exception as error: print "*** Error can not find run: " + str(run) + " in RunInfo: " + str(error) raise Exception("Error can not find run: " + str(run) + " in RunInfo: " + str(error)) rRep = RunReport() rRep.setRunNumber(runInfo.run()) rRep.setRunInfoContent(runInfo) deltaTRun = runInfo.stopTime() - runInfo.startTime() deltaTRunH = deltaTRun.days*24. + deltaTRun.seconds/(60.*60.) print "-- run #: " + colorTools.blue(runInfo.run()) print " start: " + str(runInfo.startTime()) + " stop: " + str(runInfo.stopTime()) + " lenght (h): " + str(deltaTRunH) # --- status flags for this run isFileFound = False emptyPayload = True isOutOfOrder = False allLumiIOVFound = False # --- look for the file on AFS fileName = "" fileForRun = [] # find the files associated to this run: for dbFile in fileList: if str(run) in dbFile: fileForRun.append(dbFile) if len(fileForRun) == 0: print " " + colorTools.warning("***Warning") + ": no sqlite file found!" isFileFound = False elif len(fileForRun) > 1: print " " + colorTools.warning("***Warning") + ": more than one file for this run!" for dbFile in fileForRun: modifDate = datetime.datetime.fromtimestamp(os.path.getmtime(promptCalibDir + dbFile)) print ' ',dbFile,'time-stamp:',modifDate for dbFile in fileForRun: isFileFound = True if isFileFound and not emptyPayload and isFileFound: # in this case the file was already identified continue print " file: " + dbFile modifDate = datetime.datetime.fromtimestamp(os.path.getmtime(promptCalibDir + dbFile)) rRep.setJobTime(modifDate) # # check this is not older than the one for the following run # if isFirst or modifDate < lastDate: # lastDate = modifDate # isFirst = False # isOutOfOrder = False # else: # print " " + warning("Warning: ") + " this comes after the following run!!!" # isOutOfOrder = True # delta-time from begin of run deltaTFromBegin = modifDate - runInfo.startTime() deltaTFromBeginH = deltaTFromBegin.days*24. + deltaTFromBegin.seconds/(60.*60.) # delta-time from end of run deltaTFromEndH = 0.01 if(modifDate > runInfo.stopTime()): deltaTFromEnd = modifDate - runInfo.stopTime() deltaTFromEndH = deltaTFromEnd.days*24. + deltaTFromEnd.seconds/(60.*60.) print " file time: " + str(modifDate) + " Delta_T begin (h): " + str(deltaTFromBeginH) + " Delta_T end (h): " + str(deltaTFromEndH) rRep.setLatencyFromBeginning(deltaTFromBeginH) rRep.setLatencyFromEnd(deltaTFromEndH) # check the file size fileSize = os.path.getsize(promptCalibDir + dbFile) if fileSize == 1 or fileSize == 32768: emptyPayload = True print " " + colorTools.warning("***Warning") + ": no payload in sqlite file!" else: emptyPayload = False # list the iov in the tag connect = "sqlite_file:" + promptCalibDir + dbFile listiov_run_sqlite = gtTools.listIov(connect, tagRun, '') if listiov_run_sqlite[0] == 0: iovtableByRun_sqlite = gtTools.IOVTable() iovtableByRun_sqlite.setFromListIOV(listiov_run_sqlite[1]) #iovtableByRun_sqlite.printList() for iov in iovtableByRun_sqlite._iovList: iovOracle = gtTools.IOVEntry() if iovtableByRun_oracle.search(iov.since(), iovOracle): print " runbased IOV found in Oracle!" #print iovOracle else: print " " + colorTools.warning("Warning:") + " runbased IOV not found in Oracle" missingIOV = False listiov_lumi_sqlite = gtTools.listIov(connect, tagLumi, '') if listiov_lumi_sqlite[0] == 0: iovtableByLumi_sqlite = gtTools.IOVTable() iovtableByLumi_sqlite.setFromListIOV(listiov_lumi_sqlite[1]) #iovtableByLumi_sqlite.printList() counterbla = 0 for iov in iovtableByLumi_sqlite._iovList: iovOracle = gtTools.IOVEntry() if not iovtableByLumi_oracle.search(iov.since(), iovOracle): #print " Lumi based IOV found in Oracle:" #print iovOracle counterbla += 1 print " " + colorTools.warning("Warning:") + " lumibased IOV not found in Oracle for since: " + str(iov.since()) missingIOV = True else: raise Exception("Error can not list IOV for file",connect) if not missingIOV: allLumiIOVFound = True print " All lumibased IOVs found in oracle!" else: allLumiIOVFound = False print " " + colorTools.warning("Warning:") + " not all lumibased IOVs found in Oracle!!!" # fill the run-report for this run if not isFileFound: rRep.sqliteFound(False) else: rRep.sqliteFound(True) if isOutOfOrder: rRep.isOutoforder(True) else: rRep.isOutoforder(False) if emptyPayload: rRep.payloadFound(False) else: rRep.payloadFound(True) if not allLumiIOVFound: rRep.isUploaded(False) else: rRep.isUploaded(True) return rRep
def getRunReport(pclTag, run, runInfo, fileList, oracleTables, lastUploadDate): if int(run) != int(runInfo.run()): print "*** Error can not find run: " + str(run) + " in RunInfo!" raise Exception("Error can not find run: " + str(run) + " in RunInfo!") # ------------------------------------------------------------------- # 0 --- print the timing information about this run if(type(runInfo.stopTime()) == datetime.datetime): deltaTRun = runInfo.stopTime() - runInfo.startTime() deltaTRunH = deltaTRun.days*24. + deltaTRun.seconds/(60.*60.) print "-- run #: " + colorTools.blue(str(runInfo.run())) print " start: " + str(runInfo.startTime()) + " stop: " + str(runInfo.stopTime()) + " lenght (h): " + str(deltaTRunH) else: print "-- run #: " + colorTools.blue(str(runInfo.run())) print " start: " + str(runInfo.startTime()) + " is ongoing according to RunInfo" # print ' -- Creating runReport for tag: ' + colorTools.blue(pclTag) rRep = RunReport(runInfo.run()) # set the info about the run from RunInfo rRep.setRunInfoContent(runInfo) # ------------------------------------------------------------------- # 1 --- look for the file on AFS for dbFile in fileList: if '.db' in dbFile: if str(run) in dbFile: if pclTag in dbFile: rRep.addFile(PCLOutputFiles(config.promptCalibDir + dbFile, False)) if not rRep.pclRun: print " " + colorTools.warning("***Warning") + ": no sqlite file found!" elif rRep.multipleFiles: print " " + colorTools.warning("***Warning") + ": more than one file for this run!" # print timestamps of all files for dbFile in rRep.fileList: print ' ',dbFile.fileName ,'time-stamp (creation):',dbFile.creationTime,' (upload)',dbFile.uploadTime # pick the most meaningful file (the first one? and run checks for dbFile in rRep.fileList: print " file: " + dbFile.fileName # ------------------------------------------------------------------- # 2 --- Check for empty payloads (if the case exit) # list IOV # - if empty -> setEmpty Payload and continue # - if full quit check for uplaod in oracle and quit the loop connect = "sqlite_file:" + dbFile.fileName listiov_sqlite = gtTools.listIov(connect, pclTag, '') rRep.setJobTime(dbFile.creationTime) if listiov_sqlite[0] != 0: # the payload was not found in the sqlite -> look for the next file print colorTools.warning("Warning") + " can not list IOV for this file -> no payload" continue # FIXME: use pywrappers iovtable_sqlite = gtTools.IOVTable() iovtable_sqlite.setFromListIOV(listiov_sqlite[1]) rRep.hasPayload = True print " sqlite contains a payload!" # ------------------------------------------------------------------- # 3 --- check if upload was tried (if not exit) if not dbFile.isUploaded(): print colorTools.warning(" Warning") + " upload not yet tried!",connect continue rRep.hasUpload = True rRep.setUploadTime(dbFile.uploadTime) # ------------------------------------------------------------------- # 4 --- check for out-of-order upload if not lastUploadDate == None: # make sure this is not the first run to be checked... # FIXME: check if lastUploadDate < dbFile.uploadTime: print " " + warning("Warning: ") + " this comes after the following run!!!" rRep.isOutofOrder = True lastUploadDate = dbFile.uploadTime # ------------------------------------------------------------------- # 5 --- check for the IOVs in ORACLE # get the target tag in oracle from the Drop-Box metadata metadatamap = getDropBoxMetadata(dbFile) #print metadatamap targetOracleTag = metadatamap['destinationTags'].keys()[0] targetOracleConnect = metadatamap['destinationDatabase'] # check for online connection strings if 'oracle://cms_orcon_prod' in targetOracleConnect: targetOracleConnect = 'oracle://cms_orcon_adg/CMS_COND_'+metadatamap['destinationDatabase'].split('CMS_COND_')[1] print " Target tag in Oracle:",targetOracleTag,'in account',targetOracleConnect # list IOV for the target tag (cache the list of IOV by pclTag: in case has not changed there is no need to re-run listIOv) iovtable_oracle = gtTools.IOVTable() if not targetOracleTag in oracleTables: # IOV for this tag in ORACLE has not yet been cached -> will now list IOV listiov_oracle = gtTools.listIov(targetOracleConnect, targetOracleTag, config.passwdfile) print " listing IOV..." if listiov_oracle[0] == 0: iovtable_oracle.setFromListIOV(listiov_oracle[1]) oracleTables[targetOracleTag] = iovtable_oracle else: # get the IOV from the cache dictionary print " getting IOV list from cache..." iovtable_oracle = oracleTables[targetOracleTag] # check if the IOVs of the sqlite are found in oracle and flip the corresponding flag in rRep missingIOV = False for iov in iovtable_sqlite._iovList: iovOracle = gtTools.IOVEntry() if not iovtable_oracle.search(iov.since(), iovOracle): print " " + colorTools.warning("Warning:") + " lumibased IOV not found in Oracle for since: " + str(iov.since()) missingIOV = True if missingIOV: print " " + colorTools.warning("Warning:") + " not all IOVs found in Oracle!!!" else: print " All IOVs found in oracle: upload OK!" rRep.uploadSucceeded = not missingIOV # no need to check the following files break return rRep