return (self.error, query, cQuery) # error
#
# main
#
if __name__ == "__main__":
localOpt = ["[ -move <fileIn> <fileOutDir> ]"]
usage = es_init.helpMsg("ESMove", localOpt)
usageDescription = """
Option description:
"""
examples = es_init.ESExamples()
userCommand = "ESMove.py"
optList, dictOpt = es_init.ESOptions(userCommand, sys.argv, usage,
usageDescription)
dbName, dbHost, userName, userPass, dbPort, dbSocket = optList[0]
historyFile, logFile, verbose, profile = optList[1]
userCommand = optList[2]
# default values
fileInList = []
fileIn = ""
fileOut = ""
# parse the rest of the options and form user's command
x = 1
doNotRead = 0
counter = 0
while x < len(sys.argv):
try:
if sys.argv[x] == "-move":
def ESBuilder(args):
"""ESBuilder is a main injection tool. It supports two types of DBs:
MySQL and SQLite. The injection can be done for variety of file formats:
evio, hddm, idxa. For option information and usage please use '-help' option.
For option description '--help'.
For specific injection types please use '-examples' option.
Please note, ESBuilder is a wrapper shell script around ESBuilder.py
module which does the work.
"""
localOpt = ["[ -add <dir or file or pattern of files> ]"]
localOpt.append("[ -grade <grade> ] [ -time <timeStamp> ]")
localOpt.append("[ -dataVersionName <name> ] [ -view <skim> ]")
localOpt.append("[ -listOfParents <dataVersionName's> ]")
localOpt.append("[ -output <dir> ] [ -HSMDir <HSM directory> ]")
localOpt.append("[ -dupRead <fileName> ] [ -skim ] [ -no-skim ]")
localOpt.append("[ -masterDB <name@host:port:socket or fileName> ]")
usage = es_init.helpMsg("ESBuilder", localOpt)
usageDescription = """
Option description:
* -grade: specifies the grade, e.g. "physics", "p2-unchecked"
* -add: adds data file(s) to the EventStore
You may specify: directory, file name or a list of files
For patterns use '*', e.g MC*tau*.pds
* -output: output location for storing key/location files
* -dataVersionName: specifies the data version name (aka svName)
-time: specifies the timeStamp, e.g. 20090227. If time is not provided
will try to append to existing grade/dataVersionName or use a
one day in a future as new timeStamp if no grade/dataVersionName
combination is found.
-view: specifies the view, e.g. "tau"
-listOfParents specifies list of parents for given injection,
e.g. while injecting p2-unchecked grade its parent is 'daq'.
-newDB: force the creation of a new EventStore
-sqlite use the SQLite version of EventStore
default sqlite.db, otherwise a fileName needs to be provided
-mysql use the MySQL version of EventStore. In order to access MySQL
you need either provide login/password through the -user/-password
options or create $HOME/.esdb.conf with user:password entry
-masterDB specifies host and db name of the master you want to use
-verbose: verbose mode, a lot of useful printout
-idleMode when this flag is specified, no key/location file will be
generated (useful once you have them and want reproduce DB
content). But content of DB will be updated. USE WITH CAUTION.
-delete delete a grade from EventStore. USE WITH CAUTION.
You need to provide the grade and the timeStamp.
-HSMDir specifies output HSM directory.
-logFile specifies the log file name. You may either provide a full file name
(including path) or 'stdout' or 'stderr' to redirect your log to
appropriate I/O stream. During injection an intermidiate files
esdb.log.YYYYMMDD_HHMMSS_PID will be created.
Once a job successfully finishes, the esdb.log.YYYYMMDD_HHMMSS_PID
is moved to your logFile, otherwise esdb.log.YYYYMMDD_HHMMSS_PID remains.
-profile perform internal profiling.
-dupRead in the case of duplicated records force to use this source
-skim force ESBuilder to use input files as a skim, i.e. find their parents
and build combined location file for all of them
-no-skim force ESBuilder to use input files as is
Please note: required parameters are marked with (*). All options can be
specified in any order. By default: view='all', EventStoreTMP DB is used and key/location
files are generated.
"""
examples = es_init.ESExamples()
userCommand = "ESBuilder.py"
optList, dictOpt = es_init.ESOptions(userCommand, args, usage,
usageDescription)
dbName, dbHost, userName, userPass, dbPort, dbSocket = optList[0]
historyFile, logFile, verbose, profile = optList[1]
userCommand = optList[2]
# default values
grade = ""
timeS = gen_util.dayAhead()
oDir = ""
view = "all"
run = 0
file = ""
newDB = 0
delete = 0
genMode = 1
minRun = 0
maxRun = 1000000
localtime = time.strftime("%Y%m%d_%H%M%S", time.localtime())
uname = os.uname()
svName = ""
tempLogFile = "esdb.log.%s_%s" % (localtime, os.getpid())
fileList = []
listOfParents = []
oHSMDir = ""
dupRead = ""
skim = 0
noskim = 0
masterDBName = dbName
masterDBHost = dbHost
master = ""
masterDB = ""
masterDBPort = dbPort
masterDBSocket = dbSocket
# parse the rest of the options and form user's command
x = 1
doNotRead = 0
while x < len(args):
try:
if args[x] == "-newDB":
newDB = 1
x += 1
continue
if args[x] == "-HSMDir":
oHSMDir = args[x + 1]
checkArg([oHSMDir])
x += 2
continue
if args[x] == "-dupRead":
dupRead = args[x + 1]
checkArg([dupRead])
x += 2
continue
if args[x] == "-dataVersionName":
svName = args[x + 1]
checkArg([svName])
x += 2
continue
if args[x] == "-grade":
grade = args[x + 1]
checkArg([grade])
x += 2
continue
if args[x] == "-time":
timeS = args[x + 1]
checkArg([timeS])
x += 2
continue
if args[x] == "-output":
oDir = args[x + 1] + "/"
checkArg([oDir])
x += 2
continue
if args[x] == "-runRange":
minRun = int(args[x + 1])
maxRun = int(args[x + 2])
checkArg([minRun, maxRun])
x += 3
continue
if args[x] == "-listOfParents":
x += 1
while (args[x][0] != "-"):
newArg = args[x]
listOfParents.append(args[x])
x += 1
if len(args) == x:
break
checkArg(listOfParents)
continue
if args[x] == "-add":
file = os_path_util.formAbsolutePath(args[x + 1])
# first check if pattern is present
if len(args) > x + 2 and args[x + 2][0] != "-":
counter = 0
for idx in xrange(x + 1, len(args)):
newArg = args[idx]
if newArg[0] == "-":
break
counter += 1
if os.path.isfile(newArg):
fileList.append(
os_path_util.formAbsolutePath(newArg))
x += counter + 1
continue
elif os.path.isdir(file):
dir = file + "/"
for f in os.listdir(dir):
if string.split(f, ".")[-1] != "pds": continue
fileName = dir + f
fileList.append(
os_path_util.formAbsolutePath(fileName))
x += 2
continue
elif os_path_util.isFile(file):
if file[-5:] == ".list":
tmpList = open(file).readlines()
for item in tmpList:
fileList.append(string.split(item)[0])
else:
fileList = [file]
x += 2
continue
# check if this file exists
else:
print "ESBuilder: no such file", file
raise
checkArg(fileList)
if args[x] == "-view":
view = args[x + 1]
checkArg([view])
x += 2
continue
if args[x] == "-idleMode":
genMode = 0
x += 1
continue
if args[x] == "-skim":
skim = 1
x += 1
continue
if args[x] == "-no-skim":
noskim = 1
x += 1
continue
if args[x] == "-masterDB":
masterDB = args[x + 1]
master = 1
checkArg([masterDB])
x += 2
continue
# if we reach here, that means we found unkown option
if dictOpt.has_key(args[x]):
x += dictOpt[args[x]]
else:
print "Option '%s' is not allowed" % args[x]
raise
except:
sys.exit(1)
### AUTHENTICATION???
# check that USER=pass2, otherwise exit
authUsers = ['gluex', 'sdobbs'] ### CHECK
# check if USER environment is set up, otherwise use LOGNAME
env = os.environ
if not env.has_key('USER'):
os.environ['USER'] = env['LOGNAME']
if not authUsers.count(
os.environ["USER"]) and dbName == "EventStore" and string.find(
dbHost, 'hallddb') != -1:
print "ERROR: Injection to db='EventStore' should be done from official (gluex) account for %s DB\n" % dbName
print "For your own injection please use another db name"
sys.exit(1)
# check underlying OS, so far we only allow to inject from SunOS
#if os.environ["USER"]=="pass2" and uname[0]!="SunOS":
# print "ERROR: for pass2 account the EventStore injection should be done from SunOS\n"
# sys.exit(1)
#######################################
# form normalized abosulte paths
oDir = os_path_util.formAbsolutePath(oDir)
# check required parameters
if not len(grade):
print "ESBuilder requires to specify a grade, see -grade option"
sys.exit(1)
if string.find(grade, "unchecked") == -1 and view == "all": ### CHECK
print "ESBuilder only allow to inject 'unchecked' grades"
print " daq-unechecked, p2-unchecked, physics-unchecked"
print "Either specify different view or inject as unchecked grade"
print "Given grade='%s' view='%s'" % (grade, view)
sys.exit(1)
if not len(fileList):
print "ESBuilder requires to specify input file(s) with -add option"
sys.exit(1)
# check permissions and access to output dir
if not os.path.isdir(oDir):
print "Output directory '%s' doesn't exists" % oDir
print "ESBuilder requires to specify output dir to store key/location files, see -output option"
sys.exit(1)
if oDir and not os_path_util.checkPermission(oDir):
print "You don't have permission to write to output area '%s'" % oDir
sys.exit(1)
# check permission to write to HSM
if oHSMDir and not os.path.isdir(oHSMDir):
print "HSM directory '%s' does not exists" % oHSMDir
sys.exit(1)
if not os_path_util.checkPermission(oHSMDir):
print "You don't have permission to write to HSM location '%s'" % oHSMDir
sys.exit(1)
# check that all inputs are in place
for file in fileList:
if not os.path.isfile(file):
print "File '%s' does not exists" % file
sys.exit(1)
if dupRead and not os.path.isfile(dupRead):
print "File '%s' does not exists" % dupRead
sys.exit(1)
# connect to MySQL EventStoreDB
outputLog, globalLog = es_init.ESOutputLog(logFile)
db, dbType = es_init.ESDBConnector(dbHost, dbName, userName, userPass, '',
dbPort, dbSocket)
es_init.ESInput(userCommand, outputLog, dbType)
# Be verbose
dbinfo = "\t grade\t'%s'\n\t timeStamp\t'%s'\n\t view\t\t'%s'\n" % (
grade, timeS, view)
if newDB:
if verbose:
print "Creating new tables DB:"
print dbinfo
else:
if verbose:
print "Updating existing tables in DB:"
print dbinfo
if genMode == 0 and verbose:
print "\n\t ===> Process running in Idle mode"
# create instance of ESManager class
mydb = ESManager.ESManager(db, dbType, outputLog)
# set-up all parameters
mydb.setOutputDir(oDir)
mydb.setGenerateDB(newDB)
mydb.setSVName(svName)
mydb.setParents(listOfParents)
mydb.setGrade(grade)
mydb.setTimeStamp(timeS)
mydb.setView(view)
mydb.setMinRun(minRun)
mydb.setMaxRun(maxRun)
mydb.setVerboseLevel(verbose)
mydb.setReadDuplicatesSource(dupRead)
mydb.setSkimFlag(skim)
mydb.setNoSkimFlag(noskim)
mydb.setDBHost(dbHost)
mydb.setDBName(dbName)
mydb.setDBPort(dbPort)
mydb.setDBSocket(dbSocket)
# interpret the master option
if masterDB:
dbComponents = string.split(masterDB, "@")
if len(dbComponents) == 2:
masterDBName = dbComponents[0]
newComponents = string.split(dbComponents[1], ":")
masterDBHost = newComponents[0]
port = socket = ""
if len(newComponents) == 2:
port = newComponents[1]
elif len(newComponents) == 3:
socket = newComponents[2]
# masterDBHost,port,socket=string.split(dbComponents[1],":")
if port:
masterDBPort = port
if socket:
masterDBSocket = socket
else:
masterDBHost = dbComponents[0]
else:
login, adminInfo, cMasterName, cMasterHost, cMasterPort, cMasterSocket = esdb_auth.readConfigFile(
)
if cMasterHost:
masterDBHost = cMasterHost
masterDBName = cMasterName
masterDBPort = cMasterPort
masterDBSocket = cMasterSocket
mydb.setMasterDB(masterDBName, masterDBHost, masterDBPort, masterDBSocket)
# update DB using transaction
if delete:
status = mydb.deleteGrade(delGrade, delTime)
else: # for anything else
try:
status = mydb.updateDB(genMode, fileList, oHSMDir)
except:
print "ERROR: fail to process:"
for item in fileList:
print item
print "--------------- See traceback ----------------"
raise
# close connection to db
mydb.commit()
mydb.close()
returnStatus = es_init.ESOutput(status, userCommand, historyFile,
outputLog, globalLog)
return returnStatus
def db2db(args):
"""A tool to merge one EventStore DB into another. We use the following algorithm:
- lock tables of output DB for writing
- loop over available grade,timeStamps in input DB => get graphid
- update Version table in output DB
- loop over files in FileID
- using new graphid and new (fileName,fileType) update KeyFile
and Location tables. In the case of location file, change fileIds in its
header to new values. In both cases change fileName of index/location files.
- update FileID with new fileName (if any, in the case of data file we don't
change its name).
- rename the file.
Please note, we only intend to change a prefix of files and rely on the fact
that all files in input DB were properly configured (according with EventStore
specs. This routine accepts a list of command line arguments to do the merging.
Here an example:
ESDB2DB -dbin EventStore1@/home/vk/sqlite.db -dbout EventStore2@lnx151 -changeprefix /nfs /cdat
"""
msg="""
Usage: ESDB2DB [ -help ] [ --help ] [ -examples ] [ -profile ]
[ -verbose ] [ -historyfile <filename> ]
[ -logFile </path/filename or 'stdout' or 'stderr'> ]
[ -dbin [<dbName>@<dbHost> or <fileName>] ]
[ -dbout [<dbName>@<dbHost> or <fileName>] ]
[ -changeprefix <fromDir> <toDir> ]
"""
msg+= "\n"
msg+= "\nOptions can be specified in any order."
msg+= "\nFor option description please run 'ESDB2DB --help'"
msg+= "\nFor use cases please run 'ESDB2DB -examples'"
msg+= "\nContact: Sean Dobbs, [email protected]\n"
usage = msg
usageDescription=""
examples="""
Examples:
to merge content of SQLite EventStore1 DB into MySQL EventStore2 DB
ESDB2DB -dbin EventStore1@/home/vk/sqlite.db -dbout EventStore2@lnx151
to merge content of SQLite EventStore1 DB into MySQL EventStore2 DB, plus
change prefix of all files in SQLite from /nfs to /cdat
ESDB2DB -dbin EventStore1@/home/vk/sqlite.db -dbout EventStore2@lnx151
-changeprefix /nfs /cdat
"""
userCommand="ESDB2DB.py"
optList, dictOpt = es_init.ESOptions(userCommand,args,usage,usageDescription,examples)
dbName,dbHost,userName,userPass,dbPort,dbSocket = optList[0]
historyFile,logFile,verbose,profile = optList[1]
userCommand = optList[2]
dbInHost=bInName=dbOutHost=dbOutHost=oldDir=newDir=""
x = 1
inOut = 0
while x < len(args):
try:
if string.lower(args[x]) == "-dbin" :
dbInName,dbInHost,dbInPort,dbInSocket=es_init.decodeHostNameString(args[x+1])
inOut+=1
x+=2
continue
if string.lower(args[x]) == "-dbout" :
dbOutName,dbOutHost,dbOutPort,dbOutSocket=es_init.decodeHostNameString(args[x+1])
inOut+=1
x+=2
continue
if string.lower(args[x]) == "-changeprefix":
oldDir = args[x+1]
newDir = args[x+2]
x+=3
continue
if dictOpt.has_key(args[x]):
x+=dictOpt[args[x]]
else:
print "Option '%s' is not allowed"%args[x]
raise
except:
gen_util.printExcept()
sys.exit(1)
if inOut!=2:
print usage
sys.exit(1)
if newDir:
if not os_path_util.checkPermission(newDir):
print "You don't have permission to write to",newDir
sys.exit(1)
# initialize log
outputLog, globalLog = es_init.ESOutputLog(logFile)
# connect to dbIn EventStore
dbIn, dbInType = es_init.ESDBConnector(dbInHost,dbInName,userName,userPass,'EXCLUSIVE',dbInPort,dbInSocket)
sqlIn = sql_util.SQLUtil(dbIn,dbInType,outputLog)
inTableList = sqlIn.getTables()
# in the case of SQLite we need to open DB with isolation level EXCLUSIVE
# in order to lock all tables.
# dbOut,dbOutType= es_init.ESDBConnector(dbOutHost,dbOutName,userName,userPass,'EXCLUSIVE',dbPort,dbSocket)
dbOut,dbOutType= es_init.ESDBConnector(dbOutHost,dbOutName,userName,userPass,'',dbOutPort,dbOutSocket)
sqlOut = sql_util.SQLUtil(dbOut,dbOutType,outputLog)
outTableList = sqlOut.getTables()
if verbose:
print "input table list:",inTableList
print "output table list:",outTableList
if inTableList!=outTableList:
for table in inTableList:
if not outTableList.count(table) and table!='MaxMasterID':
if verbose:
print "No '%s' EventStore table found, we will create it"%table
sqlOut.createTables(table)
# we're going to use a dictionary of old/new ids for data files
dict = {}
# Initialize ESManager for dbOut
es_init.ESInput(userCommand,outputLog,dbOutType)
esManager = ESManager.ESManager(dbOut,dbOutType,outputLog)
esManager.setVerboseLevel(verbose)
# lock all tables for write operation in dbOut
esManager.lockTables()
# postpone all commits to DB, once job will finish we'll invoke commit.
esManager.setCommitFlag(0)
# get list of parents from dbIn
parentList=[]
query="SELECT parentId FROM PathDepend"
tup = sqlIn.fetchAll(query)
for item in tup:
parentId = item[0]
query="""SELECT svName FROM GraphPath,SpecificVersion WHERE
GraphPath.svid=SpecificVersion.svid AND
graphid='%s'"""%parentId
data = sqlIn.fetchAll(query)
for name in data:
if not parentList.count(name[0]):
parentList.append(name[0])
# Clean-up dbIn and remove empty fileNames (they may be created in the case of dskim when
# parent files where not store into db)
query = "DELETE FROM FileID where fileName IS NULL"
tup = sqlIn.updateDBAndLog(query)
# First step is to modify path prefix in dbIn to new one
if newDir:
print "Modifying original DB to use new path prefix"
try:
query = "SELECT fileId,fileName FROM FileID"
tup = sqlIn.fetchAll(query)
for item in tup:
fileId = item[0]
fileName= item[1]
# replace fileName with new fileId
newFileName = os.path.normpath(gen_util.changeFileName(fileName,oldDir,newDir))
query = "UPDATE FileID SET fileName='%s' WHERE fileId='%s'"%(newFileName,fileId)
sqlIn.updateDBAndLog(query)
if verbose: print fileName,"=>",newFileName
except:
dbIn.rollback()
print "Caught an error during merging step."
gen_util.printExcept()
status = es_init.ESOutput(0,userCommand,historyFile,outputLog,globalLog)
return status
# commit changes
dbIn.commit()
# put everything in try/except block, if something bad happens we'll rollback
maxFileId = 0
if dbOutType=="sqlite":
query="BEGIN IMMEDIATE"
sqlOut.updateDBAndLog(query)
try:
# get max fileId from dbOut, in the case if we'll need to rollback
query = "SELECT MAX(fileId) FROM FileID"
tup = sqlOut.fetchOne(query)
maxFileId = tup[0]
# get info from Version dbIn.
query = """SELECT grade,timeStamp,minRunNumber,maxRunNumber,
Version.graphid,svName
FROM Version,GraphPath,SpecificVersion
WHERE Version.graphid=GraphPath.graphid
AND GraphPath.svid=SpecificVersion.svid"""
tup = sqlIn.fetchAll(query)
dictGraphId= {}
lastGraphId= 0
locList = []
keyList = []
# loop over entries in Version dbIn and fill out Version dbOut.
for item in tup:
_grade = item[0]
_timeS = item[1]
_minR = item[2]
_maxR = item[3]
_gid = item[4]
_svName= item[5]
# set-up all parameters
esManager.setSVName(_svName)
esManager.setGrade(_grade)
esManager.setTimeStamp(_timeS)
esManager.setMinRun(_minR)
esManager.setMaxRun(_maxR)
esManager.setParents(parentList)
# update GraphPath, SpecificVersion tables
newGraphId=esManager.updateVersion()
if not newGraphId:
print "Fail to update Version table"
sys.exit(1)
# fill out dictionary of old->new graphid's
if not dictGraphId.has_key(_gid):
dictGraphId[_gid]=newGraphId
parentgidList=""
for parent in parentList:
query="""SELECT graphid FROM GraphPath,SpecificVersion WHERE
GraphPath.svid=SpecificVersion.svid AND
SpecificVersion.svName='%s'"""%parent
parentgidList= sqlOut.fetchAll(query)
# Loop over FileID table in dbIn and update FileID, KeyFile, Location, FileType,
# RunUID tables in dbOut
#
# while inserting into dbOut we need to change file names
query="SELECT fileId FROM FileID"
tup = sqlIn.fetchAll(query)
print "Processing:"
for item in tup:
fileId = item[0]
query = """SELECT fileName,type FROM FileID,FileType
WHERE FileID.typeId=FileType.id AND fileId=%s"""%fileId
tup = sqlIn.fetchOne(query)
# print query
# print tup
if not tup:
continue
fileName= tup[0]
fileType= tup[1]
print fileName
# check dbOut if file already there (e.g. we used parents)
query = "SELECT fileId FROM FileID where fileName='%s'"%fileName
tup = sqlOut.fetchOne(query)
if tup and tup[0]:
if verbose:
print "Found '%s' in dbOut with fileId=%s"%(fileName,tup[0])
dict[fileId]=tup[0]
# look-up Location table and save it to the list to preserve order of loc.files
query="SELECT id,run,uid,graphid FROM Location WHERE locationFileId=%s"%fileId
tup = sqlIn.fetchAll(query)
for item in tup:
id = item[0]
run = item[1]
uid = item[2]
gid = item[3]
try:
newGraphId = dictGraphId[gid]
except:
continue
locList.append((id,newGraphId,run,uid,fileId))
# look-up KeyFile table and save it
query="SELECT graphid,view,run,uid FROM KeyFile WHERE keyFileId=%s"%fileId
tup = sqlIn.fetchAll(query)
for item in tup:
gid = item[0]
view= item[1]
run = item[2]
uid = item[3]
try:
newGraphId = dictGraphId[gid]
except:
continue
keyList.append((newGraphId,view,run,uid,fileId))
continue
# change file permission to be user-writable
if fileType=='levio':
os.chmod(fileName,0644)
query = "SELECT id FROM FileType WHERE type='%s'"%fileType
try:
tup = sqlOut.fetchOne(query)
except:
print "No typeId found for %s"%fileType
raise
if not tup:
typeId = esManager.updateFileType(fileName)
else:
typeId = tup[0]
# get new file id
newFileId = esManager.getIds(1)
dict[fileId]=newFileId
# make new fileName and copy loc. files and make hard link for key files
#if fileType=='ikey' or fileType=='levio' or fileType=='lhddm':
# fromField = 'esdb-%s'%fileId
# toField = 'esdb-%s'%newFileId
# newFileName=gen_util.changeFileName(fileName,fromField,toField)
# dir,file = os.path.split(newFileName)
# if not os.path.isdir(dir):
# if verbose:
# print "Create",dir
# os.makedirs(dir)
# if fileType=='ikey':
# if not os.path.isfile(newFileName):
# os.link(fileName,newFileName)
# else:
# if not os.path.islink(newFileName):
# print "File '%s' not found"%newFileName
# raise
# if verbose:
# print "Link",fileName,newFileName
# else:
# if fileName!=newFileName:
# shutil.copy(fileName,newFileName)
# if verbose:
# print "Copy",fileName,newFileName
#else:
newFileName=fileName
if fileType=='ikey':
# update KeyFile table
query="SELECT view,run,uid,graphid FROM KeyFile WHERE keyFileId=%s"%fileId
tup = sqlIn.fetchAll(query)
for item in tup:
view= item[0]
esManager.setView(view)
run = item[1]
uid = item[2]
gid = item[3]
newGraphId = dictGraphId[gid]
esManager.updateKeyFile(newGraphId,view,run,uid,newFileId)
# we also need to make a copy of key file ids for all parents
# for parentgid in parentgidList:
# query="""SELECT keyFileId,view FROM KeyFile WHERE
# graphid='%s' AND run='%s' AND uid='%s'"""%(parentgid[0],run,uid)
# data = sqlOut.fetchAll(query)
# for id in data:
# kid = id[0]
# view = id[1]
# check if we already injected key file into newGraphId
# query="""SELECT keyFileId FROM KeyFIle WHERE graphid='%s'
# AND view='%s' AND run='%s'
# AND uid='%s'"""%(newGraphId,view,run,uid)
# tup = sqlOut.fetchOne(query)
# if not tup:
# esManager.updateKeyFile(newGraphId,view,run,uid,kid)
# update RunUID table
esManager.updateRunUID(run,uid)
# update FileID table
if verbose:
print "Update:",newFileId,newFileName,typeId
esManager.updateFileID(newFileId,newFileName,typeId)
# update KeyFile table
for item in keyList:
gid = item[0]
view = item[1]
run = item[2]
uid = item[3]
keyFileId = item[4]
# check if this entry exists in DB
query = """SELECT graphid FROM KeyFile WHERE graphid='%s' AND view='%s' AND run='%s'
AND uid='%s' AND keyFileId='%s'"""%(gid,view,run,uid,keyFileId)
tup = sqlOut.fetchOne(query)
if tup:
if verbose:
msg = "Found (%s,%s,%s,%s,%s)"%(gid,view,run,uid,keyFileId)
msg+= "in KeyFile table, no update necessary"
else:
esManager.updateKeyFile(gid,view,run,uid,keyFileId)
except:
print "Caught an error during merging step."
gen_util.printExcept()
# something wrong, loop over new entries in dbOut and remove newly created files
query = """SELECT fileName FROM FileID WHERE fileId='%s'"""%maxFileId
tup = sqlIn.fetchAll(query)
origList = []
for item in tup:
origList.append(item[0])
query = """SELECT fileName,type FROM FileID,FileType WHERE
FileID.typeId=FileType.id AND fileId>'%s'"""%maxFileId
tup = sqlOut.fetchAll(query)
for item in tup:
fileName = item[0]
fileType = item[1]
if fileType=='ikey' or fileType=='lpds' or fileType=='lbin' or fileType=='lhddm':
os.chmod(fileName,0644)
if verbose: print "Remove",fileName
try:
if not origList.count(fileName):
os.remove(fileName)
except:
pass
# remove as well last file we process
# if oldDir!=newDir:
# if verbose: print "Remove",newFileName
# try:
# os.chmod(newFileName,0644)
# os.remove(newFileName)
# except:
# pass
# let's rollback
if verbose: print "Invoke rollback"
dbOut.rollback()
esManager.rollback()
status = es_init.ESOutput(0,userCommand,historyFile,outputLog,globalLog)
return status
#if dbOutType=="sqlite":
# query="COMMIT"
# sqlOut.updateDBAndLog(query)
query = """SELECT fileName,type FROM FileID,FileType WHERE
FileID.typeId=FileType.id"""
tupOut= sqlOut.fetchAll(query)
dbOut.commit()
esManager.unlockTables()
esManager.close()
# everything is fine and we may clean-up original files
query = """SELECT fileName,type FROM FileID,FileType WHERE
FileID.typeId=FileType.id"""
tupIn = sqlIn.fetchAll(query)
listOut = list(tupOut)
for item in tupIn:
if listOut.count(item): continue
fileName = item[0]
fileType = item[1]
if fileType=='ikey' or fileType=='levio' or fileType=='lhddm':
if verbose: print "Remove",fileName
try:
os.remove(fileName)
except:
pass
returnStatus = es_init.ESOutput(1,userCommand,historyFile,outputLog,globalLog)
return returnStatus
def ESFixPath(args):
"""Fix paths in EventStoreDB. The CLEOc data path specifications:
/cleo/{detector,simulated}/{event,calibration}/{daq,pass2_version}/123400/123456/{specific_version_path}
"""
localOpt = ["[ -prefix <Site CLEOc data prefix, e.g. /cdat> ]"]
usage = es_init.helpMsg("ESFixPath", localOpt)
usageDescription = ""
examples = """
"""
userCommand = "ESFixPath.py"
optList, dictOpt = es_init.ESOptions(userCommand, args, usage,
usageDescription, examples)
dbName, dbHost, userName, userPass, dbPort, dbSocket = optList[0]
historyFile, logFile, verbose, profile = optList[1]
userCommand = optList[2]
prefix = "/cdat/" ### CHECK
x = 1
while x < len(args):
try:
if string.lower(args[x]) == "-prefix":
prefix = args[x + 1]
x += 2
continue
if dictOpt.has_key(args[x]):
x += dictOpt[args[x]]
else:
print "Option '%s' is not allowed" % args[x]
raise
except:
sys.exit(1)
if prefix[0] != "/":
print "Prefix should start with /"
print usage
sys.exit(1)
# initialize log
outputLog, globalLog = es_init.ESOutputLog(logFile)
# connect to dbIn EventStore
db, dbType = es_init.ESDBConnector(dbHost, dbName, userName, userPass)
sql = sql_util.SQLUtil(db, dbType, outputLog)
tableList = sql.getTables()
# Initialize ESManager for dbOut
es_init.ESInput(userCommand, outputLog, dbType)
# lock all tables for write operation in dbOut
sql.lockTables()
# postpone all commits to DB, once job will finish we'll invoke commit.
sql.setCommitFlag(0)
sql.startTxn()
try:
# read all releases (from Solaris) which would be used to find out pass2_version
relList = os.listdir(RELPATH)
# start processing
print "Processing:"
sys.__stdout__.flush()
# we can replace retrieval of all files from DB, by using min/max fileId and then
# loop over file Ids.
# query="SELECT fileId,fileName FROM FileID"
# tup = sql.fetchAll(query)
# for item in tup:
# fileId = item[0]
# file = item[1]
query = "SELECT MIN(fileId),MAX(fileId) FROM FileID"
tup = sql.fetchOne(query)
minId = long(tup[0])
maxId = long(tup[1]) + 1
for fileId in xrange(minId, maxId):
query = "SELECT fileName FROM FileID WHERE fileId='%s'" % fileId
tup = sql.fetchOne(query)
if not tup: continue
file = tup[0]
if not os_path_util.isFile(file):
print "Found non existing file", file
continue
# Open KeyFile table, locate fileId, graphid=>svName (with all parents)
# create a new link
keyFile = file
query = "SELECT run,graphid FROM KeyFile WHERE keyFileId='%s'" % fileId
tup = sql.fetchAll(query)
for item in tup:
run = item[0]
gid = item[1]
query = """SELECT svName FROM SpecificVersion,GraphPath
WHERE GraphPath.svid=SpecificVersion.svid
AND GraphPath.graphid='%s'""" % gid
tup = sql.fetchOne(
query) # FIXME, there're many svid's assigned to gid
svName = tup[0]
dir, fileName = os.path.split(keyFile)
dList, idList, dict, dictId, graph = sql.getAllParents(svName)
parentList = newParentList(sql, run, dList)
if string.lower(svName) == 'daq':
release = 'daq'
else:
release = getRelease(relList, svName)
newPath = formNewPath(prefix, run, release, svName, parentList)
newDir = os.path.join(newPath, 'index')
if not os.path.isdir(newDir):
os.makedirs(newDir)
newFile = os.path.join(newDir, fileName)
print "Link (key)", newFile, "=>", keyFile
sys.__stdout__.flush()
if not os_path_util.isFile(newFile):
os.symlink(file, newFile)
# change db entry
query = "UPDATE FileID SET fileName='%s' WHERE fileId='%s'" % (
newFile, fileId)
sql.updateDBAndLog(query)
# Open Location table, locate fileId, graphid=>svName and open
# loc. file header to locate data files
locFile = file
query = "SELECT run,graphid FROM Location WHERE locationFileId='%s'" % fileId
tup = sql.fetchAll(query)
for item in tup:
run = item[0]
gid = item[1]
query = """SELECT svName FROM SpecificVersion,GraphPath
WHERE GraphPath.svid=SpecificVersion.svid
AND GraphPath.graphid='%s'""" % gid
tup = sql.fetchOne(
query) # FIXME, there're many svid's assigned to gid
svName = tup[0]
dir, fileName = os.path.split(locFile)
dList, idList, dict, dictId, graph = sql.getAllParents(svName)
parentList = newParentList(sql, run, dList)
if string.lower(svName) == 'daq':
release = 'daq'
else:
release = getRelease(relList, svName)
newPath = formNewPath(prefix, run, release, svName, parentList)
newDir = os.path.join(newPath, 'index')
if not os.path.isdir(newDir):
os.makedirs(newDir)
newFile = os.path.join(newDir, fileName)
print "Link (loc)", newFile, "=>", locFile
sys.__stdout__.flush()
if not os_path_util.isFile(newFile):
os.symlink(locFile, newFile)
# change db entry
query = "UPDATE FileID SET fileName='%s' WHERE fileId='%s'" % (
newFile, fileId)
sql.updateDBAndLog(query)
# open loc. file header and get data file id's
query = "SELECT fileName,fileId FROM FileID WHERE"
count = 0
for id in file_util.getFileIds(locFile):
if not count:
query += " fileId='%s'" % id
count = 1
else:
query += " OR fileId='%s'" % id
tup = sql.fetchAll(query)
for item in tup:
datFile = '%s' % item[0]
if not os_path_util.isFile(datFile):
print "Found non existing file", datFile
continue
fileId = item[1]
dir, fileName = os.path.split(datFile)
newDir = os.path.join(newPath, 'data')
if not os.path.isdir(newDir):
os.makedirs(newDir)
newFile = os.path.join(newDir, fileName)
print "Link (dat)", newFile, "=>", datFile
sys.__stdout__.flush()
if not os_path_util.isFile(newFile):
os.symlink(datFile, newFile)
# change db entry
query = "UPDATE FileID SET fileName='%s' WHERE fileId='%s'" % (
newFile, fileId)
sql.updateDBAndLog(query)
except:
print "Caught an error during merging step."
gen_util.printExcept()
db.rollback()
return
# everything is ready for commit
sql.setCommitFlag(1)
sql.endTxn()
sql.commit()
sql.unlockTables()
sql.close()
returnStatus = es_init.ESOutput(1, userCommand, historyFile, outputLog,
globalLog)
return returnStatus
def ESAddComment(args):
"""ESAddComment was designed to add comments into EventStore DB.
To add your comment you need to provide old/new grade or data
version name and old/new timeStamp. The comment can be added
either from command line or can be read from ASCII file.
Please note, ESAddComment is a wrapper shell script around addComment.py
module which does the work.
"""
localOpt = [
"[ -grade <grade> -timeStamp <time> | -dataVersionName <name> ]"
]
localOpt.append("[ -date <date> ] [ -comment <someText> ]")
usage = es_init.helpMsg("ESAddComment", localOpt)
usageDescription = """
Option description:
To add your comment to EventStore, either use grade/time or dataVersionName.
"""
examples = """
Examples:
ESAddComment -grade physics -timeStamp 20090101 -comment Add new physics grade
ESAddComment -dataVersionName PP2-20090101 -comment myComment.txt
in last example we add comment from a myComment.txt file.
"""
userCommand = "addComment.py"
optList, dictOpt = es_init.ESOptions(userCommand, args, usage,
usageDescription, examples)
dbName, dbHost, userName, userPass, dbPort, dbSocket = optList[0]
historyFile, logFile, verbose, profile = optList[1]
userCommand = optList[2]
grade = ""
cTime = time.strftime("%Y%m%d", time.localtime())
timeStamp = ""
x = 1
svName = ""
host = ""
verbose = 0
comment = "%s -- " % time.strftime("%H:%M:%S", time.localtime())
duplicateDBEntry = 0
while x < len(args):
try:
if args[x] == "-dataVersionName":
svName = args[x + 1]
checkArg([svName])
x += 2
continue
if args[x] == "-grade":
grade = args[x + 1]
checkArg([grade])
x += 2
continue
if args[x] == "-timeStamp":
timeStamp = args[x + 1]
checkArg([timeStamp])
x += 2
continue
if args[x] == "-date":
cTime = args[x + 1]
checkArg([cTime])
x += 2
continue
if args[x] == "-comment":
x += 1
while (args[x][0] != "-"):
if os.path.isfile(args[x]):
comment += open(args[x]).read()
break
comment += args[x]
x += 1
if len(args) == x:
break
continue
# if we reach here, that means we found unkown option
if dictOpt.has_key(args[x]):
x += dictOpt[args[x]]
else:
print "Option '%s' is not allowed" % args[x]
raise
except:
sys.exit(1)
if grade and svName:
print "Both grade=%s and dataVersionName=%s found" % (grade, svName)
print usage
sys.exit(1)
if len(cTime) != 8:
print "Incorrect date=%s format found, please use YYYYMMDD" % cTime
sys.exit(1)
# connect to MySQL EventStoreDB
outputLog, globalLog = es_init.ESOutputLog(logFile)
db, dbType = es_init.ESDBConnector(dbHost, dbName, userName, userPass,
dbPort, dbSocket)
pid = "%s" % os.getpid()
localtime = "%s" % time.strftime("%H:%M:%S", time.localtime())
outputLog.write("\n%s %s ###### %s initialization is completed" %
(pid, localtime, dbType))
if dbType == "sqlite":
addToQuery = ""
else:
addToQuery = " FOR UPDATE"
cu = db.cursor()
listOfSVIDs = []
if not svName:
query = """SELECT SpecificVersion.svid FROM Version,SpecificVersion,GraphPath WHERE
timeStamp='%s' AND grade='%s' AND Version.graphid=GraphPath.graphid
AND GraphPath.svid=SpecificVersion.svid
""" % (timeStamp, grade)
if verbose:
print string.join(string.split(query))
cu.execute(query)
tup = cu.fetchall()
if not len(tup) or not tup:
print "Upon your request, the following query return NULL results\n", query
print "Please check that provided grade/time or dataVersionName exists in ES"
sys.exit(1)
for item in tup:
if not listOfSVIDs.count(item[0]):
listOfSVIDs.append(item[0])
else:
query = "SELECT svid FROM SpecificVersion WHERE svName='%s'" % svName
if verbose:
print string.join(string.split(query))
cu.execute(query)
tup = cu.fetchone()
if not tup:
print "Upon your request, the following query return NULL results\n", query
print "Please check that provided grade/time or dataVersionName exists in ES"
sys.exit(1)
listOfSVIDs.append(tup[0])
if not len(listOfSVIDs):
print "No matches in ES DB found for your request"
sys.exit(1)
for svid in listOfSVIDs:
modComment = string.replace(comment, "'", "\\'")
query = """INSERT INTO SpecificVersionComment (svid,CommentDate,Comment)
VALUES('%s','%s','%s')""" % (svid, cTime, modComment)
if verbose:
print string.join(string.split(query))
if dbType == "mysql":
cu.execute("BEGIN")
cu.execute(query)
if dbType == "mysql":
cu.execute("COMMIT")
else:
db.commit()
def ESGetComment(args):
"""ESGetComment was designed to get comments from EventStore DB.
In order to use it youm need to provide grade/timeStamp
and a date (or date range) of desired comments.
Please note, ESGetComment is a wrapper shell script around getComment.py
module which does the work.
"""
localOpt = ["[ -grade <grade> -timeStamp <time> ]"]
localOpt.append("[ -date <t1 t2> ] [ -dataVersionName <name> ]")
usage = es_init.helpMsg("ESGetComment", localOpt)
usageDescription = """Option description:
Retrieve comments from EventStore for given grade/timeStamp or dataVersionName
and date.
"""
examples = """
Examples:
# get all comments happened on 20090101
ESGetComment -date 20090101
# get all comments between 20090101 20090202
ESGetComment -date 20090101 20090202
# get all comments for PP2-20090101 between 20090101 20090202
ESGetComment -dataVersionName PP2-20090101 -date 20090101 20090202
# get all comments for physics/20050323 between 20090101 20090202
ESGetComment -grade physics -timeStamp -date 20090101 20090202
"""
userCommand = "getComment.py"
optList, dictOpt = es_init.ESOptions(userCommand, args, usage,
usageDescription, examples)
dbName, dbHost, userName, userPass, dbPort, dbSocket = optList[0]
historyFile, logFile, verbose, profile = optList[1]
userCommand = optList[2]
grade = ""
timeS = ""
time1 = time.strftime("%Y%m%d", time.localtime())
time2 = time1
x = 1
svName = ""
comment = "%s -- " % time.strftime("%H:%M:%S", time.localtime())
duplicateDBEntry = 0
while x < len(args):
try:
if args[x] == "-dataVersionName":
svName = args[x + 1]
checkArg([svName])
x += 2
continue
if args[x] == "-grade":
grade = args[x + 1]
checkArgument([grade])
x += 2
continue
if args[x] == "-timeStamp":
timeS = args[x + 1]
checkArg([timeS])
x += 2
continue
if args[x] == "-date":
time1 = args[x + 1]
if x + 2 < len(args):
time2 = args[x + 2]
if time2[0] == "-":
time2 = time1
x += 2
continue
x += 1
x += 2
continue
# if we reach here, that means we found unkown option
if dictOpt.has_key(args[x]):
x += dictOpt[args[x]]
else:
print "Option '%s' is not allowed" % args[x]
raise
except:
sys.exit(1)
if grade and svName:
print "Both grade=%s and dataVersionName=%s found" % (grade, svName)
print usage
sys.exit(1)
if (grade and not timeS) or (not grade and timeS):
print "You need to specify both grade and timeStamp"
print usage
sys.exit(1)
if len(time1) != 8 or len(time2) != 8:
print "Incorrect time format found, please use YYYYMMDD format"
sys.exit(1)
# connect to MySQL EventStoreDB
outputLog, globalLog = es_init.ESOutputLog(logFile)
db, dbType = es_init.ESDBConnector(dbHost, dbName, userName, userPass, '',
dbPort, dbSocket)
pid = "%s" % os.getpid()
localtime = "%s" % time.strftime("%H:%M:%S", time.localtime())
outputLog.write("\n%s %s ###### %s initialization is completed" %
(pid, localtime, dbType))
if dbType == "sqlite":
addToQuery = ""
else:
addToQuery = " FOR UPDATE"
cu = db.cursor()
if not svName and not grade:
query = """SELECT CommentDate,Comment FROM SpecificVersionComment WHERE
CommentDate>='%s' AND CommentDate<='%s'""" % (time1, time2)
elif len(svName):
query = """SELECT CommentDate,Comment
FROM SpecificVersion,SpecificVersionComment WHERE
CommentDate>='%s' AND CommentDate<='%s'
AND SpecificVersion.svid=SpecificVersionComment.svid
AND SpecificVersion.svName='%s'""" % (time1, time2, svName)
elif len(grade):
query = """SELECT CommentDate,Comment
FROM GraphPath,Version,SpecificVersion,SpecificVersionComment WHERE
CommentDate>='%s' AND CommentDate<='%s'
AND Version.grade='%s' AND Version.timeStamp='%s'
AND Version.graphid=GraphPath.graphid
AND GraphPath.svid=SpecificVersion.svid""" % (time1, time2, grade, timeS)
if verbose:
print string.join(string.split(query))
cu.execute(query)
print "### Between %s-%s the following comments found:" % (time1, time2)
tup = cu.fetchall()
finalList = []
for item in tup:
cTime = item[0]
msg = item[1]
if not finalList.count((cTime, msg)):
print cTime, msg
finalList.append((cTime, msg))
