def read( serviceName ):
try:
svc = coral.ConnectionService()
connectionName = '/dbdev/service_'+serviceName
print(connectionName)
session = svc.connect( connectionName = connectionName, accessMode = coral.access_ReadOnly )
transaction = session.transaction()
transaction.start(True)
schema = session.nominalSchema()
table = schema.tableHandle( tableName )
query = table.newQuery()
query.addToOrderList( "ID" )
global m_rowBuffer
query.defineOutput( m_rowBuffer )
query.setRowCacheSize( rowsWithSimpleInsert )
cursor = query.execute()
for i in range( 0,rowsWithSimpleInsert + rowsWithBulkInsert ):
if ( not cursor.next() ):
raise Exception( "Unexpected number of returned rows")
checkData( i )
if ( cursor.next() ):
raise Exception( "Unexpected number of returned rows" )
del query
del session
print("Read SUCCESS for /dbdev/service_" , serviceName)
except Exception as e:
raise Exception( "Read FAILURE for /dbdev/service_" + " " + serviceName + " " + str(e) )
def writeBulk(serviceName):
try:
svc = coral.ConnectionService()
connectionName = '/dbdev/service_' + serviceName
print connectionName
session = svc.connect(connectionName=connectionName,
accessMode=coral.access_Update)
transaction = session.transaction()
transaction.start()
schema = session.nominalSchema()
table = schema.tableHandle(tableName)
dataEditor = table.dataEditor()
global m_rowBuffer
rowBuffer = m_rowBuffer
bulkOperation = dataEditor.bulkInsert(rowBuffer, rowsWithSimpleInsert)
for i in range(rowsWithSimpleInsert,
rowsWithSimpleInsert + rowsWithBulkInsert):
fillData(i)
bulkOperation.processNextIteration()
bulkOperation.flush()
del bulkOperation
transaction.commit()
del session
print "WriteBulk SUCCESS for /dbdev/service_", serviceName
except Exception, e:
raise Exception("WriteBulk FAILURE for /dbdev/service_" + " " +
serviceName + " " + str(e))
def writeSimple(serviceName):
try:
svc = coral.ConnectionService()
connectionName = '/dbdev/service_' + serviceName
print connectionName
session = svc.connect(connectionName=connectionName,
accessMode=coral.access_Update)
transaction = session.transaction()
transaction.start()
schema = session.nominalSchema()
table = schema.tableHandle(tableName)
dataEditor = table.dataEditor()
global m_rowBuffer
rowBuffer = m_rowBuffer
for i in range(0, rowsWithSimpleInsert):
fillData(i)
dataEditor.insertRow(rowBuffer)
transaction.commit()
del session
print "WriteSimple SUCCESS for /dbdev/service_", serviceName
except Exception, e:
raise Exception("WriteSimple FAILURE for /dbdev/service_" + " " +
serviceName + " " + str(e))
def run(self):
"Run method of MultipleSessionsApp"
try:
print "In MultipleSessionsApp: run"
print "MultipleSessions 1: Creating ISessionProxy and Schema objects"
svc = coral.ConnectionService()
session1 = svc.connect(self.m_connectionString,
accessMode=coral.access_Update)
schema = session1.nominalSchema()
print time.ctime()
threadBodies = []
for i in range(0, 1):
current = ThreadBody.ThreadBody(session1, i, schema,
self.m_userName,
self.m_password)
threadBodies.append(current)
current.start()
print "Waiting for the Threads....."
for threadBody in threadBodies:
threadBody.join()
del threadBody
print time.ctime()
print time.ctime()
except Exception, e:
raise Exception("Error in Run method: " + str(e))
def main():
parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]),description="Lumi DB schema operations.")
# add the arguments
parser.add_argument('-c',dest='connect',action='store',required=True,help='connect string to lumiDB')
parser.add_argument('-P',dest='authpath',action='store',help='path to authentication file')
parser.add_argument('action',choices=['create','createbranch','drop','describe','addindex','dropindex'],help='action on the schema')
parser.add_argument('--verbose',dest='verbose',action='store_true',help='verbose')
parser.add_argument('--debug',dest='debug',action='store_true',help='debug mode')
# parse arguments
args=parser.parse_args()
connectstring=args.connect
if args.debug:
msg=coral.MessageStream('')
msg.setMsgVerbosity(coral.message_Level_Debug)
svc = coral.ConnectionService()
if args.authpath and len(args.authpath)!=0:
os.environ['CORAL_AUTH_PATH']=args.authpath
session=svc.connect(connectstring,accessMode=coral.access_Update)
if args.action == 'create':
createLumi(session)
if args.action == 'createbranch':
createBranch(session,'TRUNK',None,'root')
createBranch(session,'NORM','TRUNK','hold normalization factor')
createBranch(session,'DATA','TRUNK','hold data')
if args.action == 'drop':
dropLumi(session)
if args.action == 'describe':
describeLumi(session)
if args.action == 'addindex':
createIndex(session)
if args.action == 'dropindex':
dropIndex(session)
if args.verbose :
print 'verbose mode'
def __init__(self, connectionString):
print("InsertSelect Constructor")
try:
print("In InsertSelect", " ConnectionString = ", connectionString)
self.m_svc = coral.ConnectionService()
self.m_connectionString = connectionString
except Exception as e:
raise Exception("Error in Initializer: " + str(e))
def __init__(self, connectionString):
print("DmlOperations Constructor")
try:
print("In DmlOperations", " connectionString = ", connectionString)
self.m_svc = coral.ConnectionService()
#self.loadServices()
self.m_connectionString = connectionString
except Exception as e:
raise Exception("Error in Initializer: " + str(e))
def main():
parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]),description="apply normalization factor to inst lumi")
parser.add_argument('-c',dest='connectstr',action='store',required=True,help='connectstr')
parser.add_argument('-norm',dest='normfactor',action='store',required=True,help='normalization factor to apply')
parser.add_argument('-P',dest='authpath',action='store',required=True,help='path to authentication file')
parser.add_argument('-r',dest='runnumber',action='store',required=False,help='run number')
parser.add_argument('-startrun',dest='startrun',action='store',required=False,help='start run for range action')
parser.add_argument('-endrun',dest='endrun',action='store',required=False,help='end run for range action')
parser.add_argument('action',choices=['run','range'],help='calibrate run')
parser.add_argument('--debug',dest='debug',action='store_true',help='debug')
args=parser.parse_args()
connectstr=args.connectstr
normfactor=args.normfactor
if len(args.authpath)!=0:
os.environ['CORAL_AUTH_PATH']=args.authpath
else:
raise '-P authpath argument is required'
svc=coral.ConnectionService()
session=svc.connect(connectstr,accessMode=coral.access_Update)
if args.debug:
msg=coral.MessageStream('')
msg.setMsgVerbosity(coral.message_Level_Debug)
n=0
if args.action == 'run':
runnumber=0
if args.runnumber:
runnumber=args.runnumber
else:
raise 'argument -r is required for action run'
if args.debug:
print('connectstr : ',connectstr)
print('normfactor : ',normfactor)
print('authpath : ',os.environ['CORAL_AUTH_PATH'])
print('runnumber : ',runnumber)
n=calibrateRun(session,normfactor,runnumber)
if args.action == 'range':
startrun=0
endrun=0
if args.startrun:
startrun=args.startrun
else:
raise 'argument -startrun is required for action range'
if args.endrun:
endrun=args.endrun
else:
raise 'argument -endrun is required for action range'
if args.debug:
print('connectstr : ',connectstr)
print('normfactor : ',normfactor)
print('authpath : ',os.environ['CORAL_AUTH_PATH'])
print('startrun : ',startrun)
print('endrun : ',endrun)
n=calibrateRange(session,normfactor,startrun,endrun)
print('number of rows changed: ',n)
del session
del svc
def __init__(self, connectionString):
print "SchemaDefinition Constructor"
try:
print "In SchemaDefinition", " connectionString = ", connectionString
self.m_connectionString = connectionString
self.m_svc = coral.ConnectionService()
print "In SchemaTest", " m_connectionString = ", self.m_connectionString
except Exception, e:
raise Exception("Error in Initializer: " + str(e))
def __init__(self, connectionString, workingSchema):
print "SetOperations Constructor"
try:
print "In SetOperations", " ConnectionString = ", connectionString
print "In SetOperations", " WorkingSchema = ", workingSchema
self.m_connectionString = connectionString
self.m_workingSchema = workingSchema
self.m_svc = coral.ConnectionService()
except Exception, e:
raise Exception("Error in SetOperations Initializer: " + str(e))
def main():
fillinfo = parseFillFile(filename)
#print fillinfo
msg = coral.MessageStream('')
msg.setMsgVerbosity(coral.message_Level_Debug)
os.environ['CORAL_AUTH_PATH'] = '/afs/cern.ch/user/x/xiezhen'
svc = coral.ConnectionService()
dbsession = svc.connect(conn, accessMode=coral.access_Update)
updateLumiSummarydata(dbsession, fillinfo)
del dbsession
del svc
def main():
c=constants()
parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]),description="Dump trigger info in wbm")
parser.add_argument('-c',dest='connect',action='store',required=True,help='connect string to trigger DB(required)')
parser.add_argument('-P',dest='authpath',action='store',required=True,help='path to authentication file')
parser.add_argument('-r',dest='runnumber',action='store',required=True,help='run number')
parser.add_argument('action',choices=['deadtime','deadfraction'],help='dump deadfraction')
parser.add_argument('--debug',dest='debug',action='store_true',help='debug')
args=parser.parse_args()
runnumber=args.runnumber
c.wbmdb=args.connect
if args.authpath and len(args.authpath)!=0:
os.environ['CORAL_AUTH_PATH']=args.authpath
#print 'authpath ',args.authpath
svc=coral.ConnectionService()
#print 'about to access ',c.wbmdb
session=svc.connect(c.wbmdb,accessMode=coral.access_ReadOnly)
session.typeConverter().setCppTypeForSqlType("unsigned int","NUMBER(10)")
session.typeConverter().setCppTypeForSqlType("unsigned long long","NUMBER(20)")
if args.debug:
msg=coral.MessageStream('')
msg.setMsgVerbosity(coral.message_Level_Debug)
if args.action == 'deadtime':
deadresult=deadcountForRun(session,c,runnumber)
if deadresult and len(deadresult)!=0:
print 'run',runnumber
print 'ls deadcount'
for cmsls,deadcount in deadresult.items():
print cmsls,deadcount
else:
print 'no deadtime found for run ',runnumber
if args.action == 'deadfraction':
deadresult=deadcountForRun(session,c,runnumber)
bitzeroresult=bitzeroForRun(session,c,runnumber)
print 'run',runnumber
print 'ls deadfraction'
if deadresult and len(deadresult)!=0:
#print 'run',runnumber
#print 'ls deadfraction'
for cmsls,deadcount in deadresult.items():
bitzero=bitzeroresult[cmsls]
bitzero_prescale=1.0
if int(runnumber)>=146315:
bitzero_prescale=17.0
if bitzero==0:
print cmsls,'no beam'
else:
print cmsls,'%.5f'%float(float(deadcount)/(float(bitzero)*bitzero_prescale))
else:
print 'no deadtime found for run ',runnumber
del session
del svc
def main():
c=constants()
parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]),description="Dump Run info")
parser.add_argument('-c',dest='connect',action='store',required=True,help='connect string to trigger DB(required)')
parser.add_argument('-P',dest='authpath',action='store',required=True,help='path to authentication file')
parser.add_argument('-r',dest='runnumber',action='store',required=True,help='run number')
parser.add_argument('action',choices=['hltkey','l1key','fill'],help='information to show')
parser.add_argument('--debug',dest='debug',action='store_true',help='debug')
parser.add_argument('--collision-only',dest='collisiononly',action='store_true',help='return only collision runs')
args=parser.parse_args()
runnumber=args.runnumber
c.runinfodb=args.connect
if args.authpath and len(args.authpath)!=0:
os.environ['CORAL_AUTH_PATH']=args.authpath
svc=coral.ConnectionService()
session=svc.connect(c.runinfodb,accessMode=coral.access_ReadOnly)
session.typeConverter().setCppTypeForSqlType("unsigned int","NUMBER(10)")
session.typeConverter().setCppTypeForSqlType("unsigned long long","NUMBER(20)")
if args.debug:
msg=coral.MessageStream('')
msg.setMsgVerbosity(coral.message_Level_Debug)
if args.action == 'hltkey':
p=re.compile(r'^/cdaq/physics/.+')
result=hltkeyForRun(session,c,runnumber)
print 'runnumber hltkey'
for runnum,hltkey in result.items():
if not args.collisiononly:
print runnum,hltkey
if args.collisiononly and re.match(p,hltkey):
fillnum=fillnumForRun(session,c,runnumber)
if len(fillnum)!=0:
print runnum,hltkey
if args.action == 'l1key':
p=re.compile(r'^TSC_.+_collisions_.+')
result=l1keyForRun(session,c,runnumber)
print 'runnumber tsc_key'
for runnum,l1key in result.items():
if not args.collisiononly:
print runnum,l1key
if args.collisiononly and re.match(p,l1key):
fillnum=fillnumForRun(session,c,runnumber)
if len(fillnum)!=0:
print runnum,l1key
if args.action == 'fill':
result=fillnumForRun(session,c,runnumber)
print 'runnumber fill'
if not args.collisiononly:
print runnumber,result
else:
if len(result)!=0:
print runnumber,result
del session
del svc
def __init__( self, connectionString, workingSchema ):
print("Queries Constructor")
try:
print("In Queries" , " ConnectionString = " , connectionString)
print("In Queries" , " WorkingSchema = " , workingSchema)
self.m_connectionString = connectionString
self.m_workingSchema = workingSchema
self.m_svc = coral.ConnectionService()
except Exception as e:
raise Exception("Error in Initializer: " + str(e))
def __init__(self,**kwargs):
self._c = constants()
connectstring='frontier://LumiProd/CMS_LUMI_PROD' #default value
siteconfpath = None
for arg in kwargs:
if arg == 'connectstring':
connectstring = kwargs[arg]
if arg == 'siteconfpath':
siteconfpath = kwargs[arg]
if arg == 'verbose':
self._c.VERBOSE = True
if arg == 'debug':
msg=coral.MessageStream('')
msg.setMsgVerbosity(coral.message_Level_Debug)
self._c.VERBOSE = True
if arg == 'nowarning':
self._c.NOWARNING = True
if arg == 'normfactor':
self._c.NORM = kwargs[arg]
if arg == 'lumiversion':
self._c.LUMIVERSION = kwargs[arg]
if arg == "beammode":
self._c.BEAMMODE = kwargs[arg]
connectparser = connectstrParser.connectstrParser(connectstring)
connectparser.parse()
usedefaultfrontierconfig=False
cacheconfigpath=''
if connectparser.needsitelocalinfo():
if siteconfpath is None:
cacheconfigpath=os.environ['CMS_PATH']
if cacheconfigpath:
cacheconfigpath=os.path.join(cacheconfigpath,'SITECONF','local','JobConfig','site-local-config.xml')
else:
usedefaultfrontierconfig = True
else:
cacheconfigpath = siteconfpath
cacheconfigpath = os.path.join(cacheconfigpath,'site-local-config.xml')
p=cacheconfigParser.cacheconfigParser()
if usedefaultfrontierconfig:
p.parseString(c.defaultfrontierConfigString)
else:
p.parse(cacheconfigpath)
connectstring=connectparser.fullfrontierStr(connectparser.schemaname(),p.parameterdict())
self._dbsvc = coral.ConnectionService()
self._session = self._dbsvc.connect(connectstring,
accessMode=coral.access_Update)
self._session.typeConverter().setCppTypeForSqlType("unsigned int","NUMBER(10)")
self._session.typeConverter().setCppTypeForSqlType("unsigned long long","Number(20)")
def connect(self):
try:
self.svc = coral.ConnectionService()
self.session1 = self.svc.connect(
'oracle://lcg_coral_nightly_proxy/lcg_coral_nightly',
accessMode=coral.access_Update)
self.transaction = self.session1.transaction()
self.transaction.start()
self.schema = self.session1.nominalSchema()
self.rowBuffer = coral.AttributeList()
except Exception, e:
print str(e)
def __init__( self, connectionString, userName, password ):
print("SchemaDefinition Constructor")
try:
print("In GroupApp" , " connectionString = " , connectionString)
self.m_connectionString = connectionString
self.m_userName = userName
self.m_password = password
self.m_svc = coral.ConnectionService()
print("In GroupApp" , " m_connectionString = " , self.m_connectionString)
#print "In GroupApp" , " m_userName = "******"In GroupApp" , " m_password = "******"Error in Initializer: " + str(e))
def main(*args):
perlsrawdata=parseLSFile(ilsfilename)
#print perlsrawdata
perbunchrawdata=parsebunchFile(ibunchfilename)
#print perbunchrawdata
msg=coral.MessageStream('')
msg.setMsgVerbosity(coral.message_Level_Error)
os.environ['CORAL_AUTH_PATH']='/afs/cern.ch/user/x/xiezhen'
svc = coral.ConnectionService()
dbsession=svc.connect(conn,accessMode=coral.access_Update)
summaryidlsmap=insertLumiSummarydata(dbsession,perlsrawdata)
insertLumiDetaildata(dbsession,perlsrawdata,perbunchrawdata,summaryidlsmap)
del dbsession
del svc
def main():
c=constants()
os.environ['CORAL_AUTH_PATH']='/afs/cern.ch/user/x/xiezhen'
svc=coral.ConnectionService()
session=svc.connect(c.lumidb,accessMode=coral.access_ReadOnly)
session.typeConverter().setCppTypeForSqlType("unsigned int","NUMBER(10)")
session.typeConverter().setCppTypeForSqlType("unsigned long long","NUMBER(20)")
msg=coral.MessageStream('')
msg.setMsgVerbosity(coral.message_Level_Error)
runnum=161119
##here arg 4 is default to ['OCC1'], if you want to see all the algorithms do
## detailForRun(session,c,runnum,['OCC1','OCC2','ET']) then modify detailForRun adding an outer loop on algos argument. I'm lazy
#detailForRun(session,c,runnum)
detailForRunLumisummaryV2(session,c,runnum)
def main(*args):
runnum = 0
try:
runnum = args[1]
report = csvReporter.csvReporter('instlumibytime-' + str(runnum) +
'.csv')
msg = coral.MessageStream('')
msg.setMsgVerbosity(coral.message_Level_Error)
os.environ['CORAL_AUTH_PATH'] = '/afs/cern.ch/cms/lumi'
svc = coral.ConnectionService()
connectstr = 'oracle://cms_orcoff_prod/cms_lumi_prod'
session = svc.connect(connectstr, accessMode=coral.access_ReadOnly)
session.typeConverter().setCppTypeForSqlType("unsigned int",
"NUMBER(10)")
session.typeConverter().setCppTypeForSqlType("unsigned long long",
"NUMBER(20)")
session.transaction().start(True)
schema = session.nominalSchema()
q = schema.newQuery()
runsummaryOut = lumiQueryAPI.runsummaryByrun(q, runnum)
del q
q = schema.newQuery()
lumisummaryOut = lumiQueryAPI.lumisummaryByrun(q, runnum, '0001')
del q
session.transaction().commit()
del session
del svc
#print runsummaryOut
starttimestr = runsummaryOut[3]
t = lumiTime.lumiTime()
report.writeRow(['cmslsnum', 'utctime', 'unixtimestamp', 'instlumi'])
for dataperls in lumisummaryOut:
cmslsnum = dataperls[0]
instlumi = dataperls[1]
startorbit = dataperls[3]
orbittime = t.OrbitToTime(starttimestr, startorbit)
orbittimestamp = time.mktime(
orbittime.timetuple()) + orbittime.microsecond / 1e6
report.writeRow([cmslsnum, orbittime, orbittimestamp, instlumi])
except IndexError:
print 'runnumber should be provided'
return 1
except Exception as er:
print str(er)
return 2
else:
return 0
def __init__(self,
connectString,
authpath=None,
siteconfpath=None,
debugON=False):
self.__connectString = connectString
self.__svc = None
self.__connectparser = connectstrParser.connectstrParser(
self.__connectString)
usedefaultfrontierconfig = False
cacheconfigpath = ''
try:
self.__connectparser.parse()
if self.__connectparser.needsitelocalinfo():
if not siteconfpath:
cacheconfigpath = os.environ['CMS_PATH']
if cacheconfigpath:
cacheconfigpath = os.path.join(
cacheconfigpath, 'SITECONF', 'local', 'JobConfig',
'site-local-config.xml')
else:
usedefaultfrontierconfig = True
else:
cacheconfigpath = siteconfpath
cacheconfigpath = os.path.join(cacheconfigpath,
'site-local-config.xml')
ccp = cacheconfigParser.cacheconfigParser()
if usedefaultfrontierconfig:
ccp.parseString(self.defaultfrontierConfigString())
else:
ccp.parse(cacheconfigpath)
self.__connectString = self.__connectparser.fullfrontierStr(
self.__connectparser.schemaname(), ccp.parameterdict())
if self.__connectparser.protocol() == 'oracle':
if authpath:
os.environ['CORAL_AUTH_PATH'] = authpath
else:
os.environ['CORAL_AUTH_PATH'] = '.'
if debugON:
msg = coral.MessageStream('')
msg.setMsgVerbosity(coral.message_Level_Debug)
self.__svc = coral.ConnectionService()
except:
if self.__svc: del self.__svc
raise
def main(*args):
perlsrawdata=parseLSFile(ilsfilename)
print(perlsrawdata)
perbunchrawdata=parsebunchFile(ibunchfilename)
print(sum(perbunchrawdata.values()))
msg=coral.MessageStream('')
msg.setMsgVerbosity(coral.message_Level_Error)
os.environ['CORAL_AUTH_PATH']='/afs/cern.ch/user/x/xiezhen'
svc = coral.ConnectionService()
dbsession=svc.connect(conn,accessMode=coral.access_Update)
print(len(args))
if len(args)>1 and args[1]=='--v2':
insertLumischemaV2(dbsession,runnum,ilsfilename,perlsrawdata,perbunchrawdata)
else:
summaryidlsmap=insertLumiSummarydata(dbsession,perlsrawdata)
insertLumiDetaildata(dbsession,perlsrawdata,perbunchrawdata,summaryidlsmap)
del dbsession
del svc
def ConnectToDB(self):
svc = coral.ConnectionService()
svcconfig = svc.configuration()
svcconfig.disablePoolAutomaticCleanUp()
svcconfig.setConnectionTimeOut(0)
# Check that the FRONTIER_SERVER is set properly, if not reduce the retrial period and time out values
if not ('FRONTIER_SERVER' in os.environ and os.environ['FRONTIER_SERVER']):
svcconfig.setConnectionRetrialPeriod(1)
svcconfig.setConnectionRetrialTimeOut(1)
else:
svcconfig.setConnectionRetrialPeriod(300)
svcconfig.setConnectionRetrialTimeOut(3600)
self.dbSession = svc.connect( 'ATLASDD', accessMode = coral.access_ReadOnly )
transaction = self.dbSession.transaction()
self.dbSchema = self.dbSession.nominalSchema()
transaction.start(True)
def getGTList(self):
'''returns list of GT'''
svc = coral.ConnectionService()
config = svc.configuration()
os.environ['CORAL_AUTH_PATH'] = self.authpath
config.setDefaultAuthenticationService(
'CORAL/Services/XMLAuthenticationService')
session = svc.connect(self.global_tag_schema,
accessMode=coral.access_ReadOnly)
transaction = session.transaction()
os.unsetenv('CORAL_AUTH_PATH')
transaction.start(True)
schema = session.nominalSchema()
tablelist = schema.listTables()
gt_names = []
for tablename in tablelist:
if tablename.startswith("TAGTREE_TABLE_"):
gt_name = tablename[len("TAGTREE_TABLE_"):]
gt_names.append(gt_name)
transaction.commit()
del session
return gt_names
def main():
c=constants()
parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]),description="Patch LumiData")
parser.add_argument('-c',dest='destination',action='store',required=True,help='destination lumi db (required)')
parser.add_argument('-s',dest='source',action='store',required=False,help='source db (required except for lumicalib)')
parser.add_argument('-P',dest='authpath',action='store',required=True,help='path to authentication file (required)')
parser.add_argument('-r',dest='runnumber',action='store',required=False,help='run number (optional)')
parser.add_argument('-i',dest='inputfile',action='store',required=False,help='run selection file(optional)')
parser.add_argument('-delta',dest='delta',action='store',required=False,help='calibration factor wrt old data in lumiDB (required for lumicalib)')
parser.add_argument('action',choices=['deadtimeGT','deadtimeWBM','lumicalib','runtimestamp'],help='deadtimeGT: patch deadtime to deadtimebeamactive,\ndeadtimeWBM: patch deadtimeWBM to deadtimebeamactive,\nlumicalib: recalibrate inst lumi by delta where delta>1\n runtimestamp: add start,stop run timestamp where empty')
parser.add_argument('--dryrun',dest='dryrun',action='store_true',help='only print datasource query result, do not update destination')
parser.add_argument('--debug',dest='debug',action='store_true',help='debug')
args=parser.parse_args()
runnumber=args.runnumber
destConnect=args.destination
sourceConnect=args.source
if args.authpath and len(args.authpath)!=0:
os.environ['CORAL_AUTH_PATH']=args.authpath
svc=coral.ConnectionService()
sourcesession=None
if sourceConnect:
sourcesession=svc.connect(sourceConnect,accessMode=coral.access_ReadOnly)
sourcesession.typeConverter().setCppTypeForSqlType("unsigned int","NUMBER(10)")
sourcesession.typeConverter().setCppTypeForSqlType("unsigned long long","NUMBER(20)")
destsession=svc.connect(destConnect,accessMode=coral.access_Update)
destsession.typeConverter().setCppTypeForSqlType("unsigned int","NUMBER(10)")
destsession.typeConverter().setCppTypeForSqlType("unsigned long long","NUMBER(20)")
if args.debug:
msg=coral.MessageStream('')
msg.setMsgVerbosity(coral.message_Level_Debug)
if args.dryrun:
c.isdryrun=True
else:
c.isdryrun=False
deadresult={}
if args.action == 'deadtimeGT':
if not sourceConnect:
raise Exception('deadtimeGT action requies -s option for source connection string')
deadresult=GTdeadtimeBeamActiveForRun(sourcesession,c,runnumber)
print 'reading from ',sourceConnect
print 'run : ',runnumber
print 'LS:deadtimebeamactive'
#print deadresult
if deadresult and len(deadresult)!=0:
for cmsls,deadtimebeamactive in deadresult.items():
print cmsls,deadtimebeamactive
else:
print 'no deadtime found for run ',runnumber
print 'exit'
return
print 'total LS: ',len(deadresult)
# if len(deadresult)!=max( [ (deadresult[x],x) for x in deadresult] )[1]:
if len(deadresult)!=max( [ x for x in deadresult.keys() ] ):
print 'total ls: ',len(deadresult)
#print 'max key: ',max( [ x for x in deadresult.keys()])
print 'alert: missing Lumi Sections in the middle'
for x in range(1,max( [ x for x in deadresult.keys()] ) ):
if x not in deadresult:
print 'filling up LS deadtime with 0: LS : ',x
deadresult[x]=0
#print deadresult
if not args.dryrun:
print 'updating ',destConnect
nupdated=patchDeadtimeForRun(destsession,c,int(runnumber),deadresult)
print 'number of updated rows ',nupdated
elif args.action == 'deadtimeWBM':
if not sourceConnect:
raise Exception('deadtimeWBM action requies -s option for source connection string')
deadresult=WBMdeadtimeBeamActiveForRun(sourcesession,c,runnumber)
print 'reading from ',sourceConnect
print 'run : ',runnumber
print 'LS:deadtimebeamactive'
#print deadresult
if deadresult and len(deadresult)!=0:
for cmsls,deadtimebeamactive in deadresult.items():
print cmsls,deadtimebeamactive
else:
print 'no deadtime found for run ',runnumber
print 'exit'
return
print 'total LS: ',len(deadresult)
if len(deadresult)!=max( [ (deadresult[x],x) for x in deadresult])[1]:
print 'alert: missing Lumi Sections in the middle'
for x in range(1,max( [ (deadresult[x],x) for x in deadresult])[1]):
if x not in deadresult:
print 'filling up LS deadtime with 0: LS : ',x
deadresult[x]=0
print deadresult
if not args.dryrun:
print 'updating ',destConnect
nupdated=patchDeadtimeForRun(destsession,c,int(runnumber),deadresult)
print 'number of updated rows ',nupdated
elif args.action == 'lumicalib':
if not args.delta or args.delta==0:
raise Exception('Must provide non-zero -delta argument')
runnums=[]
if args.runnumber:
runnums.append(args.runnumber)
elif args.inputfile:
basename,extension=os.path.splitext(args.inputfile)
if extension=='.csv':#if file ends with .csv,use csv parser,else parse as json file
fileparsingResult=csvSelectionParser.csvSelectionParser(args.inputfile)
else:
f=open(args.inputfile,'r')
inputfilecontent=f.read()
fileparsingResult=selectionParser.selectionParser(inputfilecontent)
if not fileparsingResult:
raise Exception('failed to parse the input file '+ifilename)
#print fileparsingResult.runsandls()
runnums=fileparsingResult.runs()
#print runnums
else:
raise Exception('Must provide -r or -i argument as input')
nupdated=recalibrateLumiForRun(destsession,c,args.delta,runnums)
elif args.action == 'runtimestamp':
if not sourceConnect:
raise Exception('runtimestamp action requies -s option for source connection string')
if not args.runnumber and not args.inputfile: #if no runnumber nor input file specified, check all
runnums=missingTimeRuns(destsession,c)
print 'these runs miss start/stop time: ',runnums
print 'total : ',len(runnums)
elif args.runnumber:
runnums=[int(args.runnumber)]
elif args.inputfile:
basename,extension=os.path.splitext(args.inputfile)
if extension=='.csv':#if file ends with .csv,use csv parser,else parse as json file
fileparsingResult=csvSelectionParser.csvSelectionParser(args.inputfile)
else:
f=open(args.inputfile,'r')
inputfilecontent=f.read()
fileparsingResult=selectionParser.selectionParser(inputfilecontent)
if not fileparsingResult:
raise Exception('failed to parse the input file '+ifilename)
runnums=fileparsingResult.runs()
result=getTimeForRun(sourcesession,c,runnums)
#for run,(startTimeT,stopTimeT) in result.items():
#print 'run: ',run
#if not startTimeT or not stopTimeT:
#print 'None'
#else:
#print 'start: ',startTimeT
#print 'stop: ',stopTimeT
addTimeForRun(destsession,c,result)
if sourcesession:
del sourcesession
del destsession
del svc
#Coral package:CoralTools
import os
import coral
from CondCore.TagCollection.exporter import exporter
#os.environ["CORAL_AUTH_PATH"]="/afs/cern.ch/sw/lcg/app/pool/db/python"
#os.environ["CORAL_DBLOOKUP_PATH"]="/afs/cern.ch/sw/lcg/app/pool/db/python"
try:
#Initialize Connection Service
svc = coral.ConnectionService()
#open session proxy for source schema providing logical service name & access mode
sourceSession = svc.connect( 'sqlite_file:source.db', accessMode = coral.access_Update )
#open session proxy for destination schema providing logical service name & access mode
destSession = svc.connect( 'sqlite_file:dest.db', accessMode = coral.access_Update )
rowCachesize=1000
exp=exporter( sourceSession,destSession,rowCachesize )
try:
print "copyschema() : Copies the schema objects from source to destination, without copying data."
exp.copyschema()
print "Tables created"
except Exception, e:
print "Test Failed"
print str(e)
try:
print "copydata(rowCount) : Copies the schema objects from source to destination, including data copy."
def prepareSchema(serviceName):
try:
svc = coral.ConnectionService()
connectionName = '/dbdev/service_' + serviceName
print connectionName
session = svc.connect(connectionName=connectionName,
accessMode=coral.access_Update)
transaction = session.transaction()
transaction.start()
schema = session.nominalSchema()
# Clear the old table
schema.dropIfExistsTable(tableName)
global m_rowBuffer
print type(m_rowBuffer)
if (m_rowBuffer):
del m_rowBuffer
m_rowBuffer = coral.AttributeList()
rowBuffer = m_rowBuffer
# Create the new table
description = coral.TableDescription()
description.setName(tableName)
description.insertColumn("ID", "int")
description.setNotNullConstraint("ID")
description.setPrimaryKey("ID")
rowBuffer.extend("ID", "int")
description.insertColumn("V_B", "bool")
rowBuffer.extend("V_B", "bool")
description.insertColumn("V_UC", "unsigned char")
rowBuffer.extend("V_UC", "unsigned char")
description.insertColumn("V_SC", "char")
rowBuffer.extend("V_SC", "char")
description.insertColumn("V_US", "unsigned short")
rowBuffer.extend("V_US", "unsigned short")
description.insertColumn("V_SS", "short")
rowBuffer.extend("V_SS", "short")
description.insertColumn("V_UI", "unsigned int")
rowBuffer.extend("V_UI", "unsigned int")
description.insertColumn("V_SI", "int")
rowBuffer.extend("V_SI", "int")
description.insertColumn("V_UL", "unsigned long")
rowBuffer.extend("V_UL", "unsigned long")
description.insertColumn("V_SL", "long")
rowBuffer.extend("V_SL", "long")
description.insertColumn("V_ULL", "unsigned long long")
rowBuffer.extend("V_ULL", "unsigned long long")
description.insertColumn("V_SLL", "long long")
rowBuffer.extend("V_SLL", "long long")
description.insertColumn("V_F", "float")
rowBuffer.extend("V_F", "float")
description.insertColumn("V_D", "double")
rowBuffer.extend("V_D", "double")
description.insertColumn("V_LD", "double")
rowBuffer.extend("V_LD", "double")
description.insertColumn("V_BLOB", "blob")
rowBuffer.extend("V_BLOB", "blob")
description.insertColumn("V_DATE", "date")
rowBuffer.extend("V_DATE", "date")
description.insertColumn("V_TIMESTAMP", "time stamp")
rowBuffer.extend("V_TIMESTAMP", "time stamp")
description.insertColumn("V_SG", "string")
rowBuffer.extend("V_SG", "string")
description.insertColumn("V_SF", "string", 5)
rowBuffer.extend("V_SF", "string")
description.insertColumn("V_SV", "string", 50, False)
rowBuffer.extend("V_SV", "string")
schema.createTable(description)
transaction.commit()
del session
print "PrepareSchema SUCCESS for /dbdev/service_", serviceName
except Exception, e:
raise Exception("PrepareSchema FAILURE for /dbdev/service_" + " " +
serviceName + " " + str(e))
#!/usr/bin/env python
import os, sys, fcntl
import coral
print("Message #1")
# See http://www.parallelpython.com/component/option,com_smf/Itemid,29/topic,414.msg1225#msg1225
stdout = sys.__stdout__.fileno() # This fd is normally 1
stderr = sys.__stderr__.fileno() # This fd is normally 2
savestdout = os.dup(stdout)
savestderr = os.dup(stderr)
rpipe, wpipe = os.pipe()
os.dup2(wpipe, stdout)
os.dup2(wpipe, stderr)
coral.MessageStream("").setMsgVerbosity(3)
coral.ConnectionService().configuration()
print("Message #2")
os.dup2(savestdout, stdout)
os.dup2(savestderr, stderr)
os.close(wpipe)
print("Message #3")
fcntl.fcntl(rpipe, fcntl.F_SETFL, os.O_NONBLOCK)
rpipe = os.fdopen(rpipe,'r',0)
while True:
try: cline = rpipe.readline()
except: break # Stop reading if the pipe has been closed
if cline == "" : break # Stop reading if EOF was read
sys.stdout.write(cline)
rpipe.close()