def get_tags(global_tag, records):
"""Get tags for `records` contained in `global_tag`.
Arguments:
- `global_tag`: global tag of interest
- `records`: database records of interest
"""
if len(records) == 0: return {} # avoid useless DB query
# check for auto GT
if global_tag.startswith("auto:"):
import Configuration.AlCa.autoCond as AC
try:
global_tag = AC.autoCond[global_tag.split("auto:")[-1]]
except KeyError:
print "Unsupported auto GT:", global_tag
sys.exit(1)
# setting up the DB session
con = conddb.connect(url = conddb.make_url())
session = con.session()
GlobalTagMap = session.get_dbtype(conddb.GlobalTagMap)
# query tag names for records of interest contained in `global_tag`
tags = session.query(GlobalTagMap.record, GlobalTagMap.tag_name).\
filter(GlobalTagMap.global_tag_name == global_tag,
GlobalTagMap.record.in_(records)).all()
# closing the DB session
session.close()
return {item[0]: {"tag": item[1], "connect": "pro"} for item in tags}
def get_iovs(db, tag):
##############################################
"""Retrieve the list of IOVs from `db` for `tag`.
Arguments:
- `db`: database connection string
- `tag`: tag of database record
"""
### unfortunately some gymnastics is needed here
### to make use of the conddb library
if db in officialdbs.keys():
db = officialdbs[db]
## allow to use input sqlite files as well
db = db.replace("sqlite_file:", "").replace("sqlite:", "")
con = conddb.connect(url=conddb.make_url(db))
session = con.session()
IOV = session.get_dbtype(conddb.IOV)
iovs = set(session.query(IOV.since).filter(IOV.tag_name == tag).all())
if len(iovs) == 0:
print("No IOVs found for tag '" + tag + "' in database '" + db + "'.")
sys.exit(1)
session.close()
return sorted([int(item[0]) for item in iovs])
def get_tags(global_tag, records):
"""Get tags for `records` contained in `global_tag`.
Arguments:
- `global_tag`: global tag of interest
- `records`: database records of interest
"""
if len(records) == 0: return {} # avoid useless DB query
# check for auto GT
if global_tag.startswith("auto:"):
import Configuration.AlCa.autoCond as AC
try:
global_tag = AC.autoCond[global_tag.split("auto:")[-1]]
except KeyError:
print "Unsupported auto GT:", global_tag
sys.exit(1)
# setting up the DB session
con = conddb.connect(url=conddb.make_url())
session = con.session()
GlobalTagMap = session.get_dbtype(conddb.GlobalTagMap)
# query tag names for records of interest contained in `global_tag`
tags = session.query(GlobalTagMap.record, GlobalTagMap.tag_name).\
filter(GlobalTagMap.global_tag_name == global_tag,
GlobalTagMap.record.in_(records)).all()
# closing the DB session
session.close()
return {item[0]: {"tag": item[1], "connect": "pro"} for item in tags}
def getAllRuns():
##############################################
con = conddb.connect(url=conddb.make_url("pro"))
session = con.session()
RUNINFO = session.get_dbtype(conddb.RunInfo)
allRuns = session.query(RUNINFO.run_number, RUNINFO.start_time,
RUNINFO.end_time).all()
return allRuns
def getTagsMap(db):
con = conddblib.connect(url = conddblib.make_url(db))
session = con.session()
TAG = session.get_dbtype(conddblib.Tag)
q1 = session.query(TAG.object_type).order_by(TAG.name).all()[0]
q2 = session.query(TAG.name).order_by(TAG.name).all()[0]
dictionary = dict(zip(q1, q2))
return dictionary
def getTagsMap(db):
con = conddblib.connect(url=conddblib.make_url(db))
session = con.session()
TAG = session.get_dbtype(conddblib.Tag)
dictionary = {}
for i in range(
0, len(session.query(TAG.object_type).order_by(TAG.name).all())):
q1 = session.query(TAG.object_type).order_by(TAG.name).all()[i][0]
q2 = session.query(TAG.name).order_by(TAG.name).all()[i][0]
dictionary[q1] = q2
return dictionary
def get_parent_tags(db, theHash):
#####################################################################
db = db.replace("sqlite_file:", "").replace("sqlite:", "")
db = db.replace("frontier://FrontierProd/CMS_CONDITIONS", "pro")
db = db.replace("frontier://FrontierPrep/CMS_CONDITIONS", "dev")
con = conddb.connect(url=conddb.make_url(db))
session = con.session()
IOV = session.get_dbtype(conddb.IOV)
Tag = session.get_dbtype(conddb.Tag)
query_result = session.query(
IOV.tag_name).filter(IOV.payload_hash == theHash).all()
tag_names = map(lambda entry: entry[0], query_result)
listOfOccur = []
for tag in tag_names:
synchro = session.query(
Tag.synchronization).filter(Tag.name == tag).all()
iovs = session.query(IOV.since).filter(IOV.tag_name == tag).filter(
IOV.payload_hash == theHash).all()
times = session.query(IOV.insertion_time).filter(
IOV.tag_name == tag).filter(IOV.payload_hash == theHash).all()
synchronization = [item[0] for item in synchro]
listOfIOVs = [item[0] for item in iovs]
listOfTimes = [str(item[0]) for item in times]
for iEntry in range(0, len(listOfIOVs)):
listOfOccur.append({
"tag": tag,
"synchronization": synchronization[0],
"since": listOfIOVs[iEntry],
"insertion_time": listOfTimes[iEntry]
})
#mapOfOccur[tag] = (synchronization,listOfIOVs,listOfTimes)
#mapOfOccur[tag]['synchronization'] = synchronization
#mapOfOccur[tag]['sinces'] = listOfIOVs
#mapOfOccur[tag]['times'] = listOfTimes
#print tag,synchronization,listOfIOVs,listOfTimes
return listOfOccur
def findRunStopTime(run_number):
con = conddb.connect(url = conddb.make_url("pro"))
session = con.session()
RunInfo = session.get_dbtype(conddb.RunInfo)
bestRun = session.query(RunInfo.run_number,RunInfo.start_time, RunInfo.end_time).filter(RunInfo.run_number >= run_number).first()
if bestRun is None:
raise Exception("Run %s can't be matched with an existing run in the database." % run_number)
start= bestRun[1]
stop = bestRun[2]
bestRunStartTime = calendar.timegm( bestRun[1].utctimetuple() ) << 32
bestRunStopTime = calendar.timegm( bestRun[2].utctimetuple() ) << 32
print("run start time:",start,"(",bestRunStartTime,")")
print("run stop time: ",stop,"(",bestRunStopTime,")")
return bestRunStopTime
def getTime ():
for i, info in enumerate(Info):
runs = os.popen('dasgoclient -query=\'run dataset=%s\'' % info[0]).read().split()
con = conddb.connect(url = conddb.make_url())
session = con.session()
RunInfo = session.get_dbtype(conddb.RunInfo)
timeDiff = 0
for run in runs:
bestRun = session.query(RunInfo.run_number, RunInfo.start_time, RunInfo.end_time).filter(RunInfo.run_number == int(run)).first()
if bestRun is None:
raise Exception("Run %s can't be matched with an existing run in the database." % run)
bestRunStartTime = calendar.timegm(bestRun[1].utctimetuple())# << 32
bestRunStopTime = calendar.timegm(bestRun[2].utctimetuple())# << 32
timeDiff += (bestRunStopTime - bestRunStartTime)
Info[i][3] = timeDiff # convert seconds into hours to avoid overflow
def get_iovs(db, tag):
"""Retrieve the list of IOVs from `db` for `tag`.
Arguments:
- `db`: database connection string
- `tag`: tag of database record
"""
db = db.replace("sqlite_file:", "").replace("sqlite:", "")
con = conddb.connect(url = conddb.make_url(db))
session = con.session()
IOV = session.get_dbtype(conddb.IOV)
iovs = set(session.query(IOV.since).filter(IOV.tag_name == tag).all())
if len(iovs) == 0:
print "No IOVs found for tag '"+tag+"' in database '"+db+"'."
sys.exit(1)
session.close()
return sorted([int(item[0]) for item in iovs])
def get_iovs(db, tag):
"""Retrieve the list of IOVs from `db` for `tag`.
Arguments:
- `db`: database connection string
- `tag`: tag of database record
"""
db = db.replace("sqlite_file:", "").replace("sqlite:", "")
con = conddb.connect(url=conddb.make_url(db))
session = con.session()
IOV = session.get_dbtype(conddb.IOV)
iovs = set(session.query(IOV.since).filter(IOV.tag_name == tag).all())
if len(iovs) == 0:
pass
#print "No IOVs found for tag '"+tag+"' in database '"+db+"'."
#sys.exit(1)
session.close()
return sorted([int(item[0]) for item in iovs])
def get_iovs(db, tag):
##############################################
"""Retrieve the list of IOVs from `db` for `tag`.
Arguments:
- `db`: database connection string
- `tag`: tag of database record
"""
### unfortunately some gymnastics is needed here
### to make use of the conddb library
officialdbs = {
# frontier connections
'frontier://PromptProd/CMS_CONDITIONS' :'pro',
'frontier://FrontierProd/CMS_CONDITIONS' :'pro',
'frontier://FrontierArc/CMS_CONDITIONS' :'arc',
'frontier://FrontierInt/CMS_CONDITIONS' :'int',
'frontier://FrontierPrep/CMS_CONDITIONS' :'dev',
}
if db in officialdbs.keys():
db = officialdbs[db]
## allow to use input sqlite files as well
db = db.replace("sqlite_file:", "").replace("sqlite:", "")
con = conddb.connect(url = conddb.make_url(db))
session = con.session()
IOV = session.get_dbtype(conddb.IOV)
iovs = set(session.query(IOV.since).filter(IOV.tag_name == tag).all())
if len(iovs) == 0:
print "No IOVs found for tag '"+tag+"' in database '"+db+"'."
sys.exit(1)
session.close()
return sorted([int(item[0]) for item in iovs])
def get_tags(global_tag, records):
"""Get tags for `records` contained in `global_tag`.
Arguments:
- `global_tag`: global tag of interest
- `records`: database records of interest
"""
if len(records) == 0: return {} # avoid useless DB query
# setting up the DB session
con = conddb.connect(url = conddb.make_url())
session = con.session()
GlobalTagMap = session.get_dbtype(conddb.GlobalTagMap)
# query tag names for records of interest contained in `global_tag`
tags = session.query(GlobalTagMap.record, GlobalTagMap.tag_name).\
filter(GlobalTagMap.global_tag_name == global_tag,
GlobalTagMap.record.in_(records)).all()
# closing the DB session
session.close()
return {item[0]: {"tag": item[1], "connect": "pro"} for item in tags}
def get_tags(global_tag, records):
"""Get tags for `records` contained in `global_tag`.
Arguments:
- `global_tag`: global tag of interest
- `records`: database records of interest
"""
if len(records) == 0: return {} # avoid useless DB query
# setting up the DB session
con = conddb.connect(url=conddb.make_url())
session = con.session()
GlobalTagMap = session.get_dbtype(conddb.GlobalTagMap)
# query tag names for records of interest contained in `global_tag`
tags = session.query(GlobalTagMap.record, GlobalTagMap.tag_name).\
filter(GlobalTagMap.global_tag_name == global_tag,
GlobalTagMap.record.in_(records)).all()
# closing the DB session
session.close()
return {item[0]: {"tag": item[1], "connect": "pro"} for item in tags}
def main():
##############################################
defaultFile = 'mySqlite.db'
defaultTag = 'myTag'
parser = optparse.OptionParser(
usage='Usage: %prog [options] <file> [<file> ...]\n')
parser.add_option(
'-f',
'--file',
dest='file',
default=defaultFile,
help='file to inspect',
)
parser.add_option(
'-t',
'--tag',
dest='tag',
default=defaultTag,
help='tag to be inspected',
)
(options, arguments) = parser.parse_args()
sqlite_db_url = options.file
db = options.file
db = db.replace("sqlite_file:", "").replace("sqlite:", "")
db = db.replace("frontier://FrontierProd/CMS_CONDITIONS", "pro")
db = db.replace("frontier://FrontierPrep/CMS_CONDITIONS", "dev")
con = conddb.connect(url=conddb.make_url(db))
session = con.session()
IOV = session.get_dbtype(conddb.IOV)
iovs = set(
session.query(IOV.since).filter(IOV.tag_name == options.tag).all())
if len(iovs) == 0:
print("No IOVs found for tag '" + options.tag + "' in database '" +
db + "'.")
sys.exit(1)
session.close()
sinces = sorted([int(item[0]) for item in iovs])
print sinces
for i, since in enumerate(sinces):
print i, since
print "============================================================"
if (i < len(sinces) - 1):
command = 'conddb_import -c sqlite_file:' + sqlite_db_url.rstrip(
".db"
) + "_IOV_" + str(
sinces[i]
) + '.db -f sqlite_file:' + sqlite_db_url + " -i " + options.tag + " -t " + options.tag + " -b " + str(
sinces[i]) + " -e " + str(sinces[i + 1] - 1)
print command
getCommandOutput(command)
else:
command = 'conddb_import -c sqlite_file:' + sqlite_db_url.rstrip(
".db"
) + "_IOV_" + str(
sinces[i]
) + '.db -f sqlite_file:' + sqlite_db_url + " -i " + options.tag + " -t " + options.tag + " -b " + str(
sinces[i])
print command
getCommandOutput(command)
# update the trigger bits
cmsRunCommand = "cmsRun AlCaRecoTriggerBitsRcdUpdate_TEMPL_cfg.py inputDB=sqlite_file:" + sqlite_db_url.rstrip(
".db"
) + "_IOV_" + str(
sinces[i]
) + ".db inputTag=" + options.tag + " outputDB=sqlite_file:" + sqlite_db_url.rstrip(
".db") + "_IOV_" + str(
sinces[i]
) + "_updated.db outputTag=" + options.tag + " firstRun=" + str(
sinces[i])
print cmsRunCommand
getCommandOutput(cmsRunCommand)
# merge the output
mergeCommand = 'conddb_import -f sqlite_file:' + sqlite_db_url.rstrip(
".db"
) + "_IOV_" + str(
sinces[i]
) + '_updated.db -c sqlite_file:' + options.tag + ".db -i " + options.tag + " -t " + options.tag + " -b " + str(
sinces[i])
print mergeCommand
getCommandOutput(mergeCommand)
# clean the house
cleanCommand = 'rm -fr *updated*.db *IOV_*.db'
getCommandOutput(cleanCommand)
parser.add_option('-s', '--since',
dest = 'since',
default = -1,
help = 'sinces to copy from validation tag',
)
(options, arguments) = parser.parse_args()
FSCR = getFSCR()
promptGT = getPromptGT()
expressGT = getExpressGT()
print "Current FSCR:",FSCR,"| Express Global Tag",expressGT,"| Prompt Global Tag",promptGT
con = conddb.connect(url = conddb.make_url("pro"))
session = con.session()
IOV = session.get_dbtype(conddb.IOV)
TAG = session.get_dbtype(conddb.Tag)
GT = session.get_dbtype(conddb.GlobalTag)
GTMAP = session.get_dbtype(conddb.GlobalTagMap)
RUNINFO = session.get_dbtype(conddb.RunInfo)
myGTMap = session.query(GTMAP.record, GTMAP.label, GTMAP.tag_name).\
filter(GTMAP.global_tag_name == str(expressGT)).\
order_by(GTMAP.record, GTMAP.label).\
all()
SiStripRecords = ["SiStripApvGain2Rcd", "SiStripApvGainRcd", "SiStripBadChannelRcd", "SiStripFedCablingRcd", "SiStripLatencyRcd", "SiStripNoisesRcd", "SiStripPedestalsRcd","SiStripThresholdRcd"]
for element in myGTMap:
def Run():
#ROOT.gSystem.Load("libFWCoreFWLite.so")
parser = parseOptions()
(options, args) = parser.parse_args()
sys.argv = grootargs
inputFileName = options.inputFileName
if os.path.isfile(inputFileName) == False:
print("File " + inputFileName + " not exist!")
return -1
plotConfigFile = open(options.plotConfigFile)
plotConfigJson = json.load(plotConfigFile)
plotConfigFile.close()
usePixelQuality = options.usePixelQuality
withPixelQuality = ""
if (usePixelQuality):
withPixelQuality = "WithPixelQuality"
showLumi = options.showLumi
# order years from old to new
years = options.years
years.sort()
# runs per year
runsPerYear = {}
for year in years:
runsPerYear[year] = []
# integrated lumi vs run
accumulatedLumiPerRun = {}
run_index = 0
lastRun = 1
# get lumi per IOV
CMSSW_Dir = os.getenv("CMSSW_BASE")
for year in years:
inputLumiFile = CMSSW_Dir + "/src/Alignment/OfflineValidation/data/lumiperrun" + str(
year) + ".txt"
if os.path.isfile(inputLumiFile) == False:
print("File " + inputLumiFile + " not exist!")
return -1
lumiFile = open(inputLumiFile, 'r')
lines = lumiFile.readlines()
for line in lines:
# line = "run inst_lumi"
run = int(line.split()[0])
integrated_lumi = float(
line.split()[1]) / lumiScaleFactor # 1/pb to 1/fb
# runs per year
runsPerYear[year].append(run)
# integrated luminosity per run
# run number must be ordered from small to large in the text file
if (run_index == 0):
accumulatedLumiPerRun[run] = integrated_lumi
else:
accumulatedLumiPerRun[
run] = accumulatedLumiPerRun[lastRun] + integrated_lumi
run_index += 1
lastRun = run
# close file
lumiFile.close()
# order by key (year)
runsPerYear = OrderedDict(sorted(runsPerYear.items(), key=lambda t: t[0]))
# order by key (run number)
accumulatedLumiPerRun = OrderedDict(
sorted(accumulatedLumiPerRun.items(), key=lambda t: t[0]))
#pixel local reco update (IOVs/sinces)
pixelLocalRecos = []
# connnect to ProdDB to access pixel local reco condition change
db = plotConfigJson["pixelDataBase"]
pixel_template = plotConfigJson["pixelLocalReco"]
db = db.replace("sqlite_file:", "").replace("sqlite:", "")
db = db.replace("frontier://FrontierProd/CMS_CONDITIONS", "pro")
db = db.replace("frontier://FrontierPrep/CMS_CONDITIONS", "dev")
con = conddb.connect(url=conddb.make_url(db))
session = con.session()
# get IOV table
IOV = session.get_dbtype(conddb.IOV)
iovs = set(
session.query(IOV.since).filter(IOV.tag_name == pixel_template).all())
session.close()
pixelLocalRecos = sorted([int(item[0]) for item in iovs])
#pixelLocalRecos = [1, 186500, 195360, 197749, 200961, 203368, 204601, 206446, 238341, 246866, 253914, 255655, 271866, 276315, 278271, 280928, 290543, 297281, 298653, 299443, 300389, 301046, 302131, 303790, 303998, 304911, 313041, 314881, 316758, 317475, 317485, 317527, 317661, 317664, 318227, 320377, 321831, 322510, 322603, 323232, 324245]
# substructures to plot
substructures = list(plotConfigJson["substructures"].keys())
# start barycentre plotter
bc = {}
try:
f = ROOT.TFile(inputFileName, "READ")
# read TTrees
for label in list(plotConfigJson["baryCentreLabels"].keys()):
isEOY = False
t = ROOT.TTree()
if label == "":
t = f.Get("PixelBaryCentreAnalyzer" + withPixelQuality +
"/PixelBarycentre")
else:
t = f.Get("PixelBaryCentreAnalyzer" + withPixelQuality +
"/PixelBarycentre_" + label)
if (label == "EOY"):
isEOY = True
bc[label] = readBaryCentreAnalyzerTree(t, substructures,
accumulatedLumiPerRun,
showLumi, isEOY)
except IOError:
print("File " + inputFileName + " not accessible")
# plot
for substructure in substructures:
for coord in ['x', 'y', 'z']:
plotbarycenter(bc, coord, plotConfigJson, substructure,
runsPerYear, pixelLocalRecos, accumulatedLumiPerRun,
withPixelQuality, showLumi)
def main():
### MAIN LOOP ###
if "CMSSW_RELEASE_BASE" in os.environ:
print "\n"
print "=================================================="
print "This script is powered by conddblib"
print "served to you by",os.getenv('CMSSW_RELEASE_BASE')
print "==================================================\n"
else:
print "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
print "+ This tool needs CMSSW libraries"
print "+ Easiest way to get it is via CMSSW is"
print "+ cmsrel CMSSW_X_Y_Z #get your favorite"
print "+ cd CMSSW_X_Y_Z/src"
print "+ cmsenv"
print "+ cd -"
print "and then you can proceed"
print "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
sys.exit(1)
desc="""This is a description of %prog."""
parser = OptionParser(description=desc,version='%prog version 0.1')
parser.add_option('-r','--run' ,help='test run number', dest='testRunNumber', action='store', default='251883')
parser.add_option('-R','--ReferenceGT',help='Reference Global Tag', dest='refGT', action='store', default='GR_H_V58C')
parser.add_option('-T','--TargetGT' ,help='Target Global Tag' , dest='tarGT', action='store', default='74X_dataRun2_HLTValidation_Queue')
parser.add_option('-L','--last' ,help='compares the very last IOV' , dest='lastIOV',action='store_true', default=False)
parser.add_option('-v','--verbose' ,help='returns more info', dest='isVerbose',action='store_true',default=False)
parser.add_option('-m','--match' ,help='print only matching',dest='stringToMatch',action='store',default='')
(opts, args) = parser.parse_args()
####################################
# Set up connections with the DB
####################################
con = conddb.connect(url = conddb.make_url("pro"))
session = con.session()
IOV = session.get_dbtype(conddb.IOV)
TAG = session.get_dbtype(conddb.Tag)
GT = session.get_dbtype(conddb.GlobalTag)
GTMAP = session.get_dbtype(conddb.GlobalTagMap)
RUNINFO = session.get_dbtype(conddb.RunInfo)
####################################
# Get the time info for the test run
####################################
bestRun = session.query(RUNINFO.run_number, RUNINFO.start_time, RUNINFO.end_time).filter(RUNINFO.run_number == int(opts.testRunNumber)).first()
if bestRun is None:
raise Exception("Run %s can't be matched with an existing run in the database." % opts.testRunNumber)
print "Run",opts.testRunNumber," |Start time",bestRun[1]," |End time",bestRun[2],"."
####################################
# Get the Global Tag snapshots
####################################
refSnap = session.query(GT.snapshot_time).\
filter(GT.name == opts.refGT).all()[0][0]
tarSnap = session.query(GT.snapshot_time).\
filter(GT.name == opts.tarGT).all()[0][0]
#print refSnap,tarSnap
####################################
# Get the Global Tag maps
####################################
GTMap_ref = session.query(GTMAP.record, GTMAP.label, GTMAP.tag_name).\
filter(GTMAP.global_tag_name == opts.refGT).\
order_by(GTMAP.record, GTMAP.label).\
all()
GTMap_tar = session.query(GTMAP.record, GTMAP.label, GTMAP.tag_name).\
filter(GTMAP.global_tag_name == opts.tarGT).\
order_by(GTMAP.record, GTMAP.label).\
all()
text_file = open(("diff_%s_vs_%s.twiki") % (opts.refGT,opts.tarGT), "w")
differentTags = {}
for element in GTMap_ref:
RefRecord = element[0]
RefLabel = element[1]
RefTag = element[2]
for element2 in GTMap_tar:
if (RefRecord == element2[0] and RefLabel==element2[1]):
if RefTag != element2[2]:
differentTags[(RefRecord,RefLabel)]=(RefTag,element2[2])
####################################
## Search for Records,Label not-found in the other list
####################################
temp1 = [item for item in GTMap_ref if (item[0],item[1]) not in zip(zip(*GTMap_tar)[0],zip(*GTMap_tar)[1])]
for elem in temp1:
differentTags[(elem[0],elem[1])]=(elem[2],"")
temp2 = [item for item in GTMap_tar if (item[0],item[1]) not in zip(zip(*GTMap_ref)[0],zip(*GTMap_ref)[1])]
for elem in temp2:
differentTags[(elem[0],elem[1])]=("",elem[2])
text_file.write("| *Record* | *"+opts.refGT+"* | *"+opts.tarGT+"* | Remarks | \n")
t = PrettyTable()
if(opts.isVerbose):
t.field_names = ['/','',opts.refGT,opts.tarGT,refSnap,tarSnap]
else:
t.field_names = ['/','',opts.refGT,opts.tarGT]
t.hrules=1
if(opts.isVerbose):
t.add_row(['Record','label','Reference Tag','Target Tag','hash1:time1:since1','hash2:time2:since2'])
else:
t.add_row(['Record','label','Reference Tag','Target Tag'])
isDifferent=False
####################################
# Loop on the difference
####################################
for Rcd in sorted(differentTags):
# empty lists at the beginning
refTagIOVs=[]
tarTagIOVs=[]
if( differentTags[Rcd][0]!=""):
refTagIOVs = session.query(IOV.since,IOV.payload_hash,IOV.insertion_time).filter(IOV.tag_name == differentTags[Rcd][0]).all()
refTagInfo = session.query(TAG.synchronization,TAG.time_type).filter(TAG.name == differentTags[Rcd][0]).all()[0]
if( differentTags[Rcd][1]!=""):
tarTagIOVs = session.query(IOV.since,IOV.payload_hash,IOV.insertion_time).filter(IOV.tag_name == differentTags[Rcd][1]).all()
tarTagInfo = session.query(TAG.synchronization,TAG.time_type).filter(TAG.name == differentTags[Rcd][1]).all()[0]
if(differentTags[Rcd][0]!="" and differentTags[Rcd][1]!=""):
if(tarTagInfo[1] != refTagInfo[1]):
print bcolors.WARNING+" *** Warning *** found mismatched time type for",Rcd,"entry. \n"+differentTags[Rcd][0],"has time type",refTagInfo[1],"while",differentTags[Rcd][1],"has time type",tarTagInfo[1]+". These need to be checked by hand. \n\n"+ bcolors.ENDC
if(opts.lastIOV):
if(sorted(differentTags).index(Rcd)==0):
print "=== COMPARING ONLY THE LAST IOV ==="
lastSinceRef=-1
lastSinceTar=-1
for i in refTagIOVs:
if (i[0]>lastSinceRef):
lastSinceRef = i[0]
hash_lastRefTagIOV = i[1]
time_lastRefTagIOV = str(i[2])
for j in tarTagIOVs:
if (j[0]>lastSinceTar):
lastSinceTar = j[0]
hash_lastTarTagIOV = j[1]
time_lastTarTagIOV = str(j[2])
if(hash_lastRefTagIOV!=hash_lastTarTagIOV):
isDifferent=True
text_file.write("| ="+Rcd[0]+"= ("+Rcd[1]+") | =="+differentTags[Rcd][0]+"== | =="+differentTags[Rcd][1]+"== | | \n")
text_file.write("|^|"+hash_lastRefTagIOV+" <br> ("+time_lastRefTagIOV+") "+ str(lastSinceRef) +" | "+hash_lastTarTagIOV+" <br> ("+time_lastTarTagIOV+") " + str(lastSinceTar)+" | ^| \n")
if(opts.isVerbose):
t.add_row([Rcd[0],Rcd[1],differentTags[Rcd][0],differentTags[Rcd][1],str(hash_lastRefTagIOV)+"\n"+str(time_lastRefTagIOV)+"\n"+str(lastSinceRef),str(hash_lastTarTagIOV)+"\n"+str(time_lastTarTagIOV)+"\n"+str(lastSinceTar)])
else:
t.add_row([Rcd[0],Rcd[1],differentTags[Rcd][0]+"\n"+str(hash_lastRefTagIOV),differentTags[Rcd][1]+"\n"+str(hash_lastTarTagIOV)])
else:
### reset all defaults
theGoodRefIOV=-1
theGoodTarIOV=-1
sinceRefTagIOV=0
sinceTarTagIOV=0
RefIOVtime = datetime.datetime(1970, 1, 1, 0, 0, 0)
TarIOVtime = datetime.datetime(1970, 1, 1, 0, 0, 0)
theRefPayload=""
theTarPayload=""
theRefTime=""
theTarTime=""
### loop on the reference IOV list
for refIOV in refTagIOVs:
if ( (refIOV[0] <= int(opts.testRunNumber)) and (refIOV[0]>=sinceRefTagIOV) and (refIOV[2]>RefIOVtime) and (refIOV[2]<refSnap) ):
sinceRefTagIOV = refIOV[0]
RefIOVtime = refIOV[2]
theGoodRefIOV=sinceRefTagIOV
theRefPayload=refIOV[1]
theRefTime=str(refIOV[2])
### loop on the target IOV list
for tarIOV in tarTagIOVs:
if ( (tarIOV[0] <= int(opts.testRunNumber)) and (tarIOV[0]>=sinceTarTagIOV) and (tarIOV[2]>TarIOVtime) and (tarIOV[2]<tarSnap)):
sinceTarTagIOV = tarIOV[0]
tarIOVtime = tarIOV[2]
theGoodTarIOV=sinceTarTagIOV
theTarPayload=tarIOV[1]
theTarTime=str(tarIOV[2])
if(theRefPayload!=theTarPayload):
isDifferent=True
text_file.write("| ="+Rcd[0]+"= ("+Rcd[1]+") | =="+differentTags[Rcd][0]+"== | =="+differentTags[Rcd][1]+"== |\n")
text_file.write("|^|"+theRefPayload+" ("+theRefTime+") | "+theTarPayload+" ("+theTarTime+") |\n")
### determinte if it is to be shown
isMatched=False
tokens=opts.stringToMatch.split(",")
decisions = [bool(Rcd[0].find(x)!=-1) for x in tokens]
for decision in decisions:
isMatched = (isMatched or decision)
if(opts.isVerbose):
if (opts.stringToMatch=="" or isMatched):
t.add_row([Rcd[0],Rcd[1],differentTags[Rcd][0],differentTags[Rcd][1],str(theRefPayload)+"\n"+str(theRefTime)+"\n"+str(theGoodRefIOV),str(theTarPayload)+"\n"+str(theTarTime)+"\n"+str(theGoodTarIOV)])
else:
if (opts.stringToMatch=="" or isMatched):
t.add_row([Rcd[0],Rcd[1],differentTags[Rcd][0]+"\n"+str(theRefPayload),differentTags[Rcd][1]+"\n"+str(theTarPayload)])
if(not isDifferent):
if(opts.isVerbose):
t.add_row(["None","None","None","None","None","None"])
else:
t.add_row(["None","None","None"])
print t
'-s',
'--since',
dest='since',
default=-1,
help='sinces to copy from validation tag',
)
(options, arguments) = parser.parse_args()
FCSR = getFCSR()
promptGT = getPromptGT()
expressGT = getExpressGT()
print("Current FCSR:", FCSR, "| Express Global Tag", expressGT,
"| Prompt Global Tag", promptGT)
con = conddb.connect(url=conddb.make_url("pro"))
session = con.session()
IOV = session.get_dbtype(conddb.IOV)
TAG = session.get_dbtype(conddb.Tag)
GT = session.get_dbtype(conddb.GlobalTag)
GTMAP = session.get_dbtype(conddb.GlobalTagMap)
RUNINFO = session.get_dbtype(conddb.RunInfo)
myGTMap = session.query(GTMAP.record, GTMAP.label, GTMAP.tag_name).\
filter(GTMAP.global_tag_name == str(expressGT)).\
order_by(GTMAP.record, GTMAP.label).\
all()
## connect to prep DB and get the list of IOVs to look at
con2 = conddb.connect(url=conddb.make_url("dev"))
session2 = con2.session()
#!/usr/bin/env python
import argparse
parser = argparse.ArgumentParser(description="Get the start time of the run")
parser.add_argument("runNumber", type=int)
args = parser.parse_args()
import sqlalchemy
import CondCore.Utilities.conddblib as conddb
session = conddb.connect(url=conddb.make_url()).session()
RunInfo = session.get_dbtype(conddb.RunInfo)
bestRun = session.query(
RunInfo.run_number, RunInfo.start_time,
RunInfo.end_time).filter(RunInfo.run_number >= args.runNumber).first()
if bestRun is None:
raise Exception(
"Run %s can't be matched with an existing run in the database." %
options.runNumber)
bestRun, runStart, runStop = bestRun
from calendar import timegm
bestRunStartTime = timegm(runStart.utctimetuple()) << 32
bestRunStopTime = timegm(runStop.utctimetuple()) << 32
print("{0} -> best run: {1}, start time {2}, stop time {3}".format(
args.runNumber, bestRun, runStart, runStop))
print(bestRunStartTime)
def main():
##############################################
defaultGT = 'auto:run3_data_prompt'
#defaultGT='123X_dataRun3_Prompt_v5'
defaultRun = 346512
parser = optparse.OptionParser(
usage='Usage: %prog [options] <file> [<file> ...]\n')
parser.add_option(
'-G',
'--inputGT',
dest='inputGT',
default=defaultGT,
help='Global Tag to get conditions',
)
parser.add_option(
'-r',
'--inputRun',
dest='inputRun',
default=defaultRun,
help='run to be used',
)
(options, arguments) = parser.parse_args()
print("Input configuration")
print("globalTag: ", options.inputGT)
print("runNumber: ", options.inputRun)
con = conddb.connect(url=conddb.make_url())
session = con.session()
RunInfo = session.get_dbtype(conddb.RunInfo)
bestRun = session.query(
RunInfo.run_number, RunInfo.start_time, RunInfo.end_time).filter(
RunInfo.run_number >= options.inputRun).first()
if bestRun is None:
raise Exception(
"Run %s can't be matched with an existing run in the database." %
options.runNumber)
start = bestRun[1]
stop = bestRun[2]
bestRunStartTime = calendar.timegm(bestRun[1].utctimetuple()) << 32
bestRunStopTime = calendar.timegm(bestRun[2].utctimetuple()) << 32
print("run start time:", start, "(", bestRunStartTime, ")")
print("run stop time: ", stop, "(", bestRunStopTime, ")")
gtstring = str(options.inputGT)
if ('auto' in gtstring):
from Configuration.AlCa import autoCond
key = gtstring.replace('auto:', '')
print("An autoCond Global Tag was used, will use key %s" % key)
gtstring = autoCond.autoCond[key]
print("Will use the resolved key %s" % gtstring)
command = 'cmsRun $CMSSW_BASE/src/CondTools/SiStrip/test/db_tree_dump.py outputRootFile=sistrip_db_tree_' + gtstring + '_' + str(
options.inputRun
) + '.root GlobalTag=' + options.inputGT + ' runNumber=' + str(
options.inputRun) + ' runStartTime=' + str(bestRunStartTime)
data = execme(command)
print("\n output of execution: \n\n", data)
def main():
##############################################
defaultFile = 'DQM_V0001_R000291659__Cosmics__Commissioning2017-PromptReco-v1__DQMIO.root'
defaultGT = '90X_dataRun2_Prompt_v3'
defaultRun = 291659
parser = optparse.OptionParser(
usage='Usage: %prog [options] <file> [<file> ...]\n')
parser.add_option(
'-f',
'--inputDQMFile',
dest='inputDQMFile',
default=defaultFile,
help='DQM File for FED Errors',
)
parser.add_option(
'-G',
'--inputGT',
dest='inputGT',
default=defaultGT,
help='Global Tag to get conditions',
)
parser.add_option(
'-r',
'--inputRun',
dest='inputRun',
default=defaultRun,
help='run to be used',
)
(options, arguments) = parser.parse_args()
print "Input configuration"
print "globalTag: ", options.inputGT
print "runNumber: ", options.inputRun
print "dqmFile: ", options.inputDQMFile
con = conddb.connect(url=conddb.make_url())
session = con.session()
RunInfo = session.get_dbtype(conddb.RunInfo)
bestRun = session.query(
RunInfo.run_number, RunInfo.start_time, RunInfo.end_time).filter(
RunInfo.run_number >= options.inputRun).first()
if bestRun is None:
raise Exception(
"Run %s can't be matched with an existing run in the database." %
options.runNumber)
start = bestRun[1]
stop = bestRun[2]
bestRunStartTime = calendar.timegm(bestRun[1].utctimetuple()) << 32
bestRunStopTime = calendar.timegm(bestRun[2].utctimetuple()) << 32
print "run start time:", start, "(", bestRunStartTime, ")"
print "run stop time: ", stop, "(", bestRunStopTime, ")"
command = 'cmsRun mergeBadChannel_Template_cfg.py globalTag=' + options.inputGT + ' runNumber=' + str(
options.inputRun
) + ' dqmFile=' + options.inputDQMFile + ' runStartTime=' + str(
bestRunStopTime)
execme(command)
