graphID, view, run)
if uid > 0:
query += " AND uid='%d'" % uid
self.cu.execute(query)
#
print " Key filename = %s" % self.cu.fetchone()
return self.ok
#
# main
#
if __name__ == "__main__":
# localOpt=["[ -noHeader ][ -runList <minRun> <maxRun> ] [ -time <YYYYDDMM> ]"]
localOpt = ["[ -noHeader ]"]
usage = es_init.helpMsg("ESQuery", localOpt)
usageDescription = """
ESQuery provides a user-friendly interface to get information from EventStore databases.
Command list with brief description:
grades - list of grades in DB
timestamps <grade> - list of timestamps for <grade>
skims <grade> [-time <YYYYDDMM>] - list of skims for <grade>
runs <grades> - list of runs for <grade>
actualDate <grade> [-time <YYYYDDMM>] - more recent date in DB to given time
verions <grade> [-time <YYYYDDMM>] - data version info for <grade>
graphVerions <grade> [-time <YYYYDDMM>] - graph version info for <grade>
Development commands:
def ESAddComment(args):
"""ESAddComment was designed to add comments into EventStore DB.
To add your comment you need to provide old/new grade or data
version name and old/new timeStamp. The comment can be added
either from command line or can be read from ASCII file.
Please note, ESAddComment is a wrapper shell script around addComment.py
module which does the work.
"""
localOpt = ["[ -grade <grade> -timeStamp <time> | -dataVersionName <name> ]"]
localOpt.append("[ -date <date> ] [ -comment <someText> ]")
usage = es_init.helpMsg("ESAddComment", localOpt)
usageDescription = """
Option description:
To add your comment to EventStore, either use grade/time or dataVersionName.
"""
examples = """
Examples:
ESAddComment -grade physics -timeStamp 20090101 -comment Add new physics grade
ESAddComment -dataVersionName PP2-20090101 -comment myComment.txt
in last example we add comment from a myComment.txt file.
"""
userCommand = "addComment.py"
optList, dictOpt = es_init.ESOptions(userCommand, args, usage, usageDescription, examples)
dbName, dbHost, userName, userPass, dbPort, dbSocket = optList[0]
historyFile, logFile, verbose, profile = optList[1]
userCommand = optList[2]
grade = ""
cTime = time.strftime("%Y%m%d", time.localtime())
timeStamp = ""
x = 1
svName = ""
host = ""
verbose = 0
comment = "%s -- " % time.strftime("%H:%M:%S", time.localtime())
duplicateDBEntry = 0
while x < len(args):
try:
if args[x] == "-dataVersionName":
svName = args[x + 1]
checkArg([svName])
x += 2
continue
if args[x] == "-grade":
grade = args[x + 1]
checkArg([grade])
x += 2
continue
if args[x] == "-timeStamp":
timeStamp = args[x + 1]
checkArg([timeStamp])
x += 2
continue
if args[x] == "-date":
cTime = args[x + 1]
checkArg([cTime])
x += 2
continue
if args[x] == "-comment":
x += 1
while args[x][0] != "-":
if os.path.isfile(args[x]):
comment += open(args[x]).read()
break
comment += args[x]
x += 1
if len(args) == x:
break
continue
# if we reach here, that means we found unkown option
if dictOpt.has_key(args[x]):
x += dictOpt[args[x]]
else:
print "Option '%s' is not allowed" % args[x]
raise
except:
sys.exit(1)
if grade and svName:
print "Both grade=%s and dataVersionName=%s found" % (grade, svName)
print usage
sys.exit(1)
if len(cTime) != 8:
print "Incorrect date=%s format found, please use YYYYMMDD" % cTime
sys.exit(1)
# connect to MySQL EventStoreDB
outputLog, globalLog = es_init.ESOutputLog(logFile)
db, dbType = es_init.ESDBConnector(dbHost, dbName, userName, userPass, dbPort, dbSocket)
pid = "%s" % os.getpid()
localtime = "%s" % time.strftime("%H:%M:%S", time.localtime())
outputLog.write("\n%s %s ###### %s initialization is completed" % (pid, localtime, dbType))
if dbType == "sqlite":
addToQuery = ""
else:
addToQuery = " FOR UPDATE"
cu = db.cursor()
listOfSVIDs = []
if not svName:
query = """SELECT SpecificVersion.svid FROM Version,SpecificVersion,GraphPath WHERE
timeStamp='%s' AND grade='%s' AND Version.graphid=GraphPath.graphid
AND GraphPath.svid=SpecificVersion.svid
""" % (
timeStamp,
grade,
)
if verbose:
print string.join(string.split(query))
cu.execute(query)
tup = cu.fetchall()
if not len(tup) or not tup:
print "Upon your request, the following query return NULL results\n", query
print "Please check that provided grade/time or dataVersionName exists in ES"
sys.exit(1)
for item in tup:
if not listOfSVIDs.count(item[0]):
listOfSVIDs.append(item[0])
else:
query = "SELECT svid FROM SpecificVersion WHERE svName='%s'" % svName
if verbose:
print string.join(string.split(query))
cu.execute(query)
tup = cu.fetchone()
if not tup:
print "Upon your request, the following query return NULL results\n", query
print "Please check that provided grade/time or dataVersionName exists in ES"
sys.exit(1)
listOfSVIDs.append(tup[0])
if not len(listOfSVIDs):
print "No matches in ES DB found for your request"
sys.exit(1)
for svid in listOfSVIDs:
modComment = string.replace(comment, "'", "\\'")
query = """INSERT INTO SpecificVersionComment (svid,CommentDate,Comment)
VALUES('%s','%s','%s')""" % (
svid,
cTime,
modComment,
)
if verbose:
print string.join(string.split(query))
if dbType == "mysql":
cu.execute("BEGIN")
cu.execute(query)
if dbType == "mysql":
cu.execute("COMMIT")
else:
db.commit()
def ESGetComment(args):
"""ESGetComment was designed to get comments from EventStore DB.
In order to use it youm need to provide grade/timeStamp
and a date (or date range) of desired comments.
Please note, ESGetComment is a wrapper shell script around getComment.py
module which does the work.
"""
localOpt = ["[ -grade <grade> -timeStamp <time> ]"]
localOpt.append("[ -date <t1 t2> ] [ -dataVersionName <name> ]")
usage = es_init.helpMsg("ESGetComment", localOpt)
usageDescription = """Option description:
Retrieve comments from EventStore for given grade/timeStamp or dataVersionName
and date.
"""
examples = """
Examples:
# get all comments happened on 20090101
ESGetComment -date 20090101
# get all comments between 20090101 20090202
ESGetComment -date 20090101 20090202
# get all comments for PP2-20090101 between 20090101 20090202
ESGetComment -dataVersionName PP2-20090101 -date 20090101 20090202
# get all comments for physics/20050323 between 20090101 20090202
ESGetComment -grade physics -timeStamp -date 20090101 20090202
"""
userCommand = "getComment.py"
optList, dictOpt = es_init.ESOptions(userCommand, args, usage,
usageDescription, examples)
dbName, dbHost, userName, userPass, dbPort, dbSocket = optList[0]
historyFile, logFile, verbose, profile = optList[1]
userCommand = optList[2]
grade = ""
timeS = ""
time1 = time.strftime("%Y%m%d", time.localtime())
time2 = time1
x = 1
svName = ""
comment = "%s -- " % time.strftime("%H:%M:%S", time.localtime())
duplicateDBEntry = 0
while x < len(args):
try:
if args[x] == "-dataVersionName":
svName = args[x + 1]
checkArg([svName])
x += 2
continue
if args[x] == "-grade":
grade = args[x + 1]
checkArgument([grade])
x += 2
continue
if args[x] == "-timeStamp":
timeS = args[x + 1]
checkArg([timeS])
x += 2
continue
if args[x] == "-date":
time1 = args[x + 1]
if x + 2 < len(args):
time2 = args[x + 2]
if time2[0] == "-":
time2 = time1
x += 2
continue
x += 1
x += 2
continue
# if we reach here, that means we found unkown option
if dictOpt.has_key(args[x]):
x += dictOpt[args[x]]
else:
print "Option '%s' is not allowed" % args[x]
raise
except:
sys.exit(1)
if grade and svName:
print "Both grade=%s and dataVersionName=%s found" % (grade, svName)
print usage
sys.exit(1)
if (grade and not timeS) or (not grade and timeS):
print "You need to specify both grade and timeStamp"
print usage
sys.exit(1)
if len(time1) != 8 or len(time2) != 8:
print "Incorrect time format found, please use YYYYMMDD format"
sys.exit(1)
# connect to MySQL EventStoreDB
outputLog, globalLog = es_init.ESOutputLog(logFile)
db, dbType = es_init.ESDBConnector(dbHost, dbName, userName, userPass, '',
dbPort, dbSocket)
pid = "%s" % os.getpid()
localtime = "%s" % time.strftime("%H:%M:%S", time.localtime())
outputLog.write("\n%s %s ###### %s initialization is completed" %
(pid, localtime, dbType))
if dbType == "sqlite":
addToQuery = ""
else:
addToQuery = " FOR UPDATE"
cu = db.cursor()
if not svName and not grade:
query = """SELECT CommentDate,Comment FROM SpecificVersionComment WHERE
CommentDate>='%s' AND CommentDate<='%s'""" % (time1, time2)
elif len(svName):
query = """SELECT CommentDate,Comment
FROM SpecificVersion,SpecificVersionComment WHERE
CommentDate>='%s' AND CommentDate<='%s'
AND SpecificVersion.svid=SpecificVersionComment.svid
AND SpecificVersion.svName='%s'""" % (time1, time2, svName)
elif len(grade):
query = """SELECT CommentDate,Comment
FROM GraphPath,Version,SpecificVersion,SpecificVersionComment WHERE
CommentDate>='%s' AND CommentDate<='%s'
AND Version.grade='%s' AND Version.timeStamp='%s'
AND Version.graphid=GraphPath.graphid
AND GraphPath.svid=SpecificVersion.svid""" % (time1, time2, grade, timeS)
if verbose:
print string.join(string.split(query))
cu.execute(query)
print "### Between %s-%s the following comments found:" % (time1, time2)
tup = cu.fetchall()
finalList = []
for item in tup:
cTime = item[0]
msg = item[1]
if not finalList.count((cTime, msg)):
print cTime, msg
finalList.append((cTime, msg))
query = """INSERT INTO Location (graphid,run,uid,locationFileId)
VALUES ('%s','%s','%s','%s')"""%(newgid,run,uid,lid)
self.updateDBAndLog(query)
self.endTxn("updateLocation")
#
# main
#
if __name__ == "__main__":
# initialize user options
localOpt=["[ -grade <gradeName> ] [ -time <timeStamp> ]"]
localOpt.append("[ -listOfNames <name1,name2> ]")
localOpt.append("")
usage=es_init.helpMsg("ESMergeSVManager",localOpt)
usageDescription="""
Option description (required options marked with '*'):
* -grade: specifies the grade, e.g. "physics", "mc-unchecked"
you need to provide two of them, 'old' from which you're reading
and 'new' to which you're moving
* -time: specifies the timeStamp, e.g. 20090227
you may provide one or two of them, 'old' from which you're reading
and 'new' to which you're moving
* -listOfNames specify a list of data version names to be merged together under new graphid.
"""
examples="""
ESMergeSVManager -grade physics-unchecked -time 20090909 -listOfNames svName1 svName2
"""
userCommand="ESMergeSVManager.py"
def ESBuilder(args):
"""ESBuilder is a main injection tool. It supports two types of DBs:
MySQL and SQLite. The injection can be done for variety of file formats:
evio, hddm, idxa. For option information and usage please use '-help' option.
For option description '--help'.
For specific injection types please use '-examples' option.
Please note, ESBuilder is a wrapper shell script around ESBuilder.py
module which does the work.
"""
localOpt = ["[ -add <dir or file or pattern of files> ]"]
localOpt.append("[ -grade <grade> ] [ -time <timeStamp> ]")
localOpt.append("[ -dataVersionName <name> ] [ -view <skim> ]")
localOpt.append("[ -listOfParents <dataVersionName's> ]")
localOpt.append("[ -output <dir> ] [ -HSMDir <HSM directory> ]")
localOpt.append("[ -dupRead <fileName> ] [ -skim ] [ -no-skim ]")
localOpt.append("[ -masterDB <name@host:port:socket or fileName> ]")
usage = es_init.helpMsg("ESBuilder", localOpt)
usageDescription = """
Option description:
* -grade: specifies the grade, e.g. "physics", "p2-unchecked"
* -add: adds data file(s) to the EventStore
You may specify: directory, file name or a list of files
For patterns use '*', e.g MC*tau*.pds
* -output: output location for storing key/location files
* -dataVersionName: specifies the data version name (aka svName)
-time: specifies the timeStamp, e.g. 20090227. If time is not provided
will try to append to existing grade/dataVersionName or use a
one day in a future as new timeStamp if no grade/dataVersionName
combination is found.
-view: specifies the view, e.g. "tau"
-listOfParents specifies list of parents for given injection,
e.g. while injecting p2-unchecked grade its parent is 'daq'.
-newDB: force the creation of a new EventStore
-sqlite use the SQLite version of EventStore
default sqlite.db, otherwise a fileName needs to be provided
-mysql use the MySQL version of EventStore. In order to access MySQL
you need either provide login/password through the -user/-password
options or create $HOME/.esdb.conf with user:password entry
-masterDB specifies host and db name of the master you want to use
-verbose: verbose mode, a lot of useful printout
-idleMode when this flag is specified, no key/location file will be
generated (useful once you have them and want reproduce DB
content). But content of DB will be updated. USE WITH CAUTION.
-delete delete a grade from EventStore. USE WITH CAUTION.
You need to provide the grade and the timeStamp.
-HSMDir specifies output HSM directory.
-logFile specifies the log file name. You may either provide a full file name
(including path) or 'stdout' or 'stderr' to redirect your log to
appropriate I/O stream. During injection an intermidiate files
esdb.log.YYYYMMDD_HHMMSS_PID will be created.
Once a job successfully finishes, the esdb.log.YYYYMMDD_HHMMSS_PID
is moved to your logFile, otherwise esdb.log.YYYYMMDD_HHMMSS_PID remains.
-profile perform internal profiling.
-dupRead in the case of duplicated records force to use this source
-skim force ESBuilder to use input files as a skim, i.e. find their parents
and build combined location file for all of them
-no-skim force ESBuilder to use input files as is
Please note: required parameters are marked with (*). All options can be
specified in any order. By default: view='all', EventStoreTMP DB is used and key/location
files are generated.
"""
examples = es_init.ESExamples()
userCommand = "ESBuilder.py"
optList, dictOpt = es_init.ESOptions(userCommand, args, usage,
usageDescription)
dbName, dbHost, userName, userPass, dbPort, dbSocket = optList[0]
historyFile, logFile, verbose, profile = optList[1]
userCommand = optList[2]
# default values
grade = ""
timeS = gen_util.dayAhead()
oDir = ""
view = "all"
run = 0
file = ""
newDB = 0
delete = 0
genMode = 1
minRun = 0
maxRun = 1000000
localtime = time.strftime("%Y%m%d_%H%M%S", time.localtime())
uname = os.uname()
svName = ""
tempLogFile = "esdb.log.%s_%s" % (localtime, os.getpid())
fileList = []
listOfParents = []
oHSMDir = ""
dupRead = ""
skim = 0
noskim = 0
masterDBName = dbName
masterDBHost = dbHost
master = ""
masterDB = ""
masterDBPort = dbPort
masterDBSocket = dbSocket
# parse the rest of the options and form user's command
x = 1
doNotRead = 0
while x < len(args):
try:
if args[x] == "-newDB":
newDB = 1
x += 1
continue
if args[x] == "-HSMDir":
oHSMDir = args[x + 1]
checkArg([oHSMDir])
x += 2
continue
if args[x] == "-dupRead":
dupRead = args[x + 1]
checkArg([dupRead])
x += 2
continue
if args[x] == "-dataVersionName":
svName = args[x + 1]
checkArg([svName])
x += 2
continue
if args[x] == "-grade":
grade = args[x + 1]
checkArg([grade])
x += 2
continue
if args[x] == "-time":
timeS = args[x + 1]
checkArg([timeS])
x += 2
continue
if args[x] == "-output":
oDir = args[x + 1] + "/"
checkArg([oDir])
x += 2
continue
if args[x] == "-runRange":
minRun = int(args[x + 1])
maxRun = int(args[x + 2])
checkArg([minRun, maxRun])
x += 3
continue
if args[x] == "-listOfParents":
x += 1
while (args[x][0] != "-"):
newArg = args[x]
listOfParents.append(args[x])
x += 1
if len(args) == x:
break
checkArg(listOfParents)
continue
if args[x] == "-add":
file = os_path_util.formAbsolutePath(args[x + 1])
# first check if pattern is present
if len(args) > x + 2 and args[x + 2][0] != "-":
counter = 0
for idx in xrange(x + 1, len(args)):
newArg = args[idx]
if newArg[0] == "-":
break
counter += 1
if os.path.isfile(newArg):
fileList.append(
os_path_util.formAbsolutePath(newArg))
x += counter + 1
continue
elif os.path.isdir(file):
dir = file + "/"
for f in os.listdir(dir):
if string.split(f, ".")[-1] != "pds": continue
fileName = dir + f
fileList.append(
os_path_util.formAbsolutePath(fileName))
x += 2
continue
elif os_path_util.isFile(file):
if file[-5:] == ".list":
tmpList = open(file).readlines()
for item in tmpList:
fileList.append(string.split(item)[0])
else:
fileList = [file]
x += 2
continue
# check if this file exists
else:
print "ESBuilder: no such file", file
raise
checkArg(fileList)
if args[x] == "-view":
view = args[x + 1]
checkArg([view])
x += 2
continue
if args[x] == "-idleMode":
genMode = 0
x += 1
continue
if args[x] == "-skim":
skim = 1
x += 1
continue
if args[x] == "-no-skim":
noskim = 1
x += 1
continue
if args[x] == "-masterDB":
masterDB = args[x + 1]
master = 1
checkArg([masterDB])
x += 2
continue
# if we reach here, that means we found unkown option
if dictOpt.has_key(args[x]):
x += dictOpt[args[x]]
else:
print "Option '%s' is not allowed" % args[x]
raise
except:
sys.exit(1)
### AUTHENTICATION???
# check that USER=pass2, otherwise exit
authUsers = ['gluex', 'sdobbs'] ### CHECK
# check if USER environment is set up, otherwise use LOGNAME
env = os.environ
if not env.has_key('USER'):
os.environ['USER'] = env['LOGNAME']
if not authUsers.count(
os.environ["USER"]) and dbName == "EventStore" and string.find(
dbHost, 'hallddb') != -1:
print "ERROR: Injection to db='EventStore' should be done from official (gluex) account for %s DB\n" % dbName
print "For your own injection please use another db name"
sys.exit(1)
# check underlying OS, so far we only allow to inject from SunOS
#if os.environ["USER"]=="pass2" and uname[0]!="SunOS":
# print "ERROR: for pass2 account the EventStore injection should be done from SunOS\n"
# sys.exit(1)
#######################################
# form normalized abosulte paths
oDir = os_path_util.formAbsolutePath(oDir)
# check required parameters
if not len(grade):
print "ESBuilder requires to specify a grade, see -grade option"
sys.exit(1)
if string.find(grade, "unchecked") == -1 and view == "all": ### CHECK
print "ESBuilder only allow to inject 'unchecked' grades"
print " daq-unechecked, p2-unchecked, physics-unchecked"
print "Either specify different view or inject as unchecked grade"
print "Given grade='%s' view='%s'" % (grade, view)
sys.exit(1)
if not len(fileList):
print "ESBuilder requires to specify input file(s) with -add option"
sys.exit(1)
# check permissions and access to output dir
if not os.path.isdir(oDir):
print "Output directory '%s' doesn't exists" % oDir
print "ESBuilder requires to specify output dir to store key/location files, see -output option"
sys.exit(1)
if oDir and not os_path_util.checkPermission(oDir):
print "You don't have permission to write to output area '%s'" % oDir
sys.exit(1)
# check permission to write to HSM
if oHSMDir and not os.path.isdir(oHSMDir):
print "HSM directory '%s' does not exists" % oHSMDir
sys.exit(1)
if not os_path_util.checkPermission(oHSMDir):
print "You don't have permission to write to HSM location '%s'" % oHSMDir
sys.exit(1)
# check that all inputs are in place
for file in fileList:
if not os.path.isfile(file):
print "File '%s' does not exists" % file
sys.exit(1)
if dupRead and not os.path.isfile(dupRead):
print "File '%s' does not exists" % dupRead
sys.exit(1)
# connect to MySQL EventStoreDB
outputLog, globalLog = es_init.ESOutputLog(logFile)
db, dbType = es_init.ESDBConnector(dbHost, dbName, userName, userPass, '',
dbPort, dbSocket)
es_init.ESInput(userCommand, outputLog, dbType)
# Be verbose
dbinfo = "\t grade\t'%s'\n\t timeStamp\t'%s'\n\t view\t\t'%s'\n" % (
grade, timeS, view)
if newDB:
if verbose:
print "Creating new tables DB:"
print dbinfo
else:
if verbose:
print "Updating existing tables in DB:"
print dbinfo
if genMode == 0 and verbose:
print "\n\t ===> Process running in Idle mode"
# create instance of ESManager class
mydb = ESManager.ESManager(db, dbType, outputLog)
# set-up all parameters
mydb.setOutputDir(oDir)
mydb.setGenerateDB(newDB)
mydb.setSVName(svName)
mydb.setParents(listOfParents)
mydb.setGrade(grade)
mydb.setTimeStamp(timeS)
mydb.setView(view)
mydb.setMinRun(minRun)
mydb.setMaxRun(maxRun)
mydb.setVerboseLevel(verbose)
mydb.setReadDuplicatesSource(dupRead)
mydb.setSkimFlag(skim)
mydb.setNoSkimFlag(noskim)
mydb.setDBHost(dbHost)
mydb.setDBName(dbName)
mydb.setDBPort(dbPort)
mydb.setDBSocket(dbSocket)
# interpret the master option
if masterDB:
dbComponents = string.split(masterDB, "@")
if len(dbComponents) == 2:
masterDBName = dbComponents[0]
newComponents = string.split(dbComponents[1], ":")
masterDBHost = newComponents[0]
port = socket = ""
if len(newComponents) == 2:
port = newComponents[1]
elif len(newComponents) == 3:
socket = newComponents[2]
# masterDBHost,port,socket=string.split(dbComponents[1],":")
if port:
masterDBPort = port
if socket:
masterDBSocket = socket
else:
masterDBHost = dbComponents[0]
else:
login, adminInfo, cMasterName, cMasterHost, cMasterPort, cMasterSocket = esdb_auth.readConfigFile(
)
if cMasterHost:
masterDBHost = cMasterHost
masterDBName = cMasterName
masterDBPort = cMasterPort
masterDBSocket = cMasterSocket
mydb.setMasterDB(masterDBName, masterDBHost, masterDBPort, masterDBSocket)
# update DB using transaction
if delete:
status = mydb.deleteGrade(delGrade, delTime)
else: # for anything else
try:
status = mydb.updateDB(genMode, fileList, oHSMDir)
except:
print "ERROR: fail to process:"
for item in fileList:
print item
print "--------------- See traceback ----------------"
raise
# close connection to db
mydb.commit()
mydb.close()
returnStatus = es_init.ESOutput(status, userCommand, historyFile,
outputLog, globalLog)
return returnStatus
else:
keyFileDict[run]=[tup[0]]
return keyFileDict
#
# main
#
if __name__ == "__main__":
# initialize user options
localOpt=["[ -grade <grade> ] [ -time <timeStamp> ] [ -skim <skim> ]"]
localOpt+=["[ -runRange <minR> <maxR> -dataVersionName <svName> |"]
localOpt+=[" -dataset <dataset> -energy <energyName> ]"]
localOpt+=["[ -instream <streamName> ]"]
localOpt.append("")
usage=es_init.helpMsg("ESEventCounter",localOpt)
usageDescription="""
Option description (required options marked with '*'):
* -grade: specifies the grade, e.g. "physics", "p2-unchecked"
you need to provide two of them, 'old' from which you're reading
and 'new' to which you're moving
* -time: specifies the timeStamp, e.g. 20090227
you may provide one or two of them, 'old' from which you're reading
and 'new' to which you're moving
-dataVersionName
specifies a data version name associated with given grade.
It can be found by using ESDump command.
-runRange specifies a run range within ESVersionManager will operate
-dataset specifies a data set name, e.g. data32
-energy specifies an enenrgy name, e.g. psi(2S)
-skim specifies the view you want to use, e.g. "2photon"
self.endTxn()
return 1
#
# main
#
if __name__ == "__main__":
# initialize user options
localOpt = ["[ -grade <old> <new> ] [ -time <old> <new> ]"]
localOpt.append("[ -badRunList <fileName> | -goodRunList <fileName> ]")
localOpt.append("[ -dataVersionName <name> ] [ -runRange <minR> <maxR> ]")
localOpt.append("")
usage = es_init.helpMsg("ESVersionManager", localOpt)
usageDescription = """
Option description (required options marked with '*'):
* -grade: specifies the grade, e.g. "physics", "mc-unchecked"
you need to provide two of them, 'old' from which you're reading
and 'new' to which you're moving
-time: specifies the timeStamp, e.g. 20090227
you may provide one or two of them, 'old' from which you're reading
and 'new' to which you're moving
-badRunList or -goodRunList
specifies the file which contains a list of runs which need to be
excluded (badRunList) or out of which new run ranges
need to be generated (goodRunList)
* -dataVersionName
specifies a data version name associated with given grade.
It can be found by using ESDump command.
return keyFileDict
#
# main
#
if __name__ == "__main__":
# initialize user options
localOpt = ["[ -grade <grade> ] [ -time <timeStamp> ] [ -skim <skim> ]"]
localOpt += ["[ -runRange <minR> <maxR> -dataVersionName <svName> |"]
localOpt += [" -dataset <dataset> -energy <energyName> ]"]
localOpt += ["[ -instream <streamName> ]"]
localOpt.append("")
usage = es_init.helpMsg("ESEventCounter", localOpt)
usageDescription = """
Option description (required options marked with '*'):
* -grade: specifies the grade, e.g. "physics", "p2-unchecked"
you need to provide two of them, 'old' from which you're reading
and 'new' to which you're moving
* -time: specifies the timeStamp, e.g. 20090227
you may provide one or two of them, 'old' from which you're reading
and 'new' to which you're moving
-dataVersionName
specifies a data version name associated with given grade.
It can be found by using ESDump command.
-runRange specifies a run range within ESVersionManager will operate
-dataset specifies a data set name, e.g. data32
-energy specifies an enenrgy name, e.g. psi(2S)
-skim specifies the view you want to use, e.g. "2photon"
self.updateDBAndLog(query)
self.endTxn()
return 1
#
# main
#
if __name__ == "__main__":
# initialize user options
localOpt=["[ -grade <old> <new> ] [ -time <old> <new> ]"]
localOpt.append("[ -badRunList <fileName> | -goodRunList <fileName> ]")
localOpt.append("[ -dataVersionName <name> ] [ -runRange <minR> <maxR> ]")
localOpt.append("")
usage=es_init.helpMsg("ESVersionManager",localOpt)
usageDescription="""
Option description (required options marked with '*'):
* -grade: specifies the grade, e.g. "physics", "mc-unchecked"
you need to provide two of them, 'old' from which you're reading
and 'new' to which you're moving
-time: specifies the timeStamp, e.g. 20090227
you may provide one or two of them, 'old' from which you're reading
and 'new' to which you're moving
-badRunList or -goodRunList
specifies the file which contains a list of runs which need to be
excluded (badRunList) or out of which new run ranges
need to be generated (goodRunList)
* -dataVersionName
specifies a data version name associated with given grade.
It can be found by using ESDump command.
def ESGetComment(args):
"""ESGetComment was designed to get comments from EventStore DB.
In order to use it youm need to provide grade/timeStamp
and a date (or date range) of desired comments.
Please note, ESGetComment is a wrapper shell script around getComment.py
module which does the work.
"""
localOpt=["[ -grade <grade> -timeStamp <time> ]"]
localOpt.append("[ -date <t1 t2> ] [ -dataVersionName <name> ]")
usage=es_init.helpMsg("ESGetComment",localOpt)
usageDescription="""Option description:
Retrieve comments from EventStore for given grade/timeStamp or dataVersionName
and date.
"""
examples="""
Examples:
# get all comments happened on 20090101
ESGetComment -date 20090101
# get all comments between 20090101 20090202
ESGetComment -date 20090101 20090202
# get all comments for PP2-20090101 between 20090101 20090202
ESGetComment -dataVersionName PP2-20090101 -date 20090101 20090202
# get all comments for physics/20050323 between 20090101 20090202
ESGetComment -grade physics -timeStamp -date 20090101 20090202
"""
userCommand="getComment.py"
optList, dictOpt = es_init.ESOptions(userCommand,args,usage,usageDescription,examples)
dbName,dbHost,userName,userPass,dbPort,dbSocket = optList[0]
historyFile,logFile,verbose,profile = optList[1]
userCommand = optList[2]
grade =""
timeS =""
time1 = time.strftime("%Y%m%d",time.localtime())
time2 = time1
x = 1
svName = ""
comment = "%s -- "%time.strftime("%H:%M:%S",time.localtime())
duplicateDBEntry=0
while x < len(args):
try:
if args[x]=="-dataVersionName":
svName = args[x+1]
checkArg([svName])
x+=2
continue
if args[x]=="-grade":
grade = args[x+1]
checkArgument([grade])
x+=2
continue
if args[x]=="-timeStamp":
timeS = args[x+1]
checkArg([timeS])
x+=2
continue
if args[x]=="-date":
time1 = args[x+1]
if x+2<len(args):
time2 = args[x+2]
if time2[0]=="-":
time2=time1
x+=2
continue
x+=1
x+=2
continue
# if we reach here, that means we found unkown option
if dictOpt.has_key(args[x]):
x+=dictOpt[args[x]]
else:
print "Option '%s' is not allowed"%args[x]
raise
except:
sys.exit(1)
if grade and svName:
print "Both grade=%s and dataVersionName=%s found"%(grade,svName)
print usage
sys.exit(1)
if (grade and not timeS) or (not grade and timeS):
print "You need to specify both grade and timeStamp"
print usage
sys.exit(1)
if len(time1)!=8 or len(time2)!=8:
print "Incorrect time format found, please use YYYYMMDD format"
sys.exit(1)
# connect to MySQL EventStoreDB
outputLog, globalLog = es_init.ESOutputLog(logFile)
db, dbType = es_init.ESDBConnector(dbHost,dbName,userName,userPass,'',dbPort,dbSocket)
pid = "%s"%os.getpid()
localtime = "%s"%time.strftime("%H:%M:%S",time.localtime())
outputLog.write("\n%s %s ###### %s initialization is completed"%(pid,localtime,dbType))
if dbType=="sqlite":
addToQuery=""
else:
addToQuery=" FOR UPDATE"
cu = db.cursor()
if not svName and not grade:
query = """SELECT CommentDate,Comment FROM SpecificVersionComment WHERE
CommentDate>='%s' AND CommentDate<='%s'"""%(time1,time2)
elif len(svName):
query = """SELECT CommentDate,Comment
FROM SpecificVersion,SpecificVersionComment WHERE
CommentDate>='%s' AND CommentDate<='%s'
AND SpecificVersion.svid=SpecificVersionComment.svid
AND SpecificVersion.svName='%s'"""%(time1,time2,svName)
elif len(grade):
query = """SELECT CommentDate,Comment
FROM GraphPath,Version,SpecificVersion,SpecificVersionComment WHERE
CommentDate>='%s' AND CommentDate<='%s'
AND Version.grade='%s' AND Version.timeStamp='%s'
AND Version.graphid=GraphPath.graphid
AND GraphPath.svid=SpecificVersion.svid"""%(time1,time2,grade,timeS)
if verbose:
print string.join(string.split(query))
cu.execute(query)
print "### Between %s-%s the following comments found:"%(time1,time2)
tup = cu.fetchall()
finalList=[]
for item in tup:
cTime=item[0]
msg =item[1]
if not finalList.count((cTime,msg)):
print cTime,msg
finalList.append((cTime,msg))
def ESFixPath(args):
"""Fix paths in EventStoreDB. The CLEOc data path specifications:
/cleo/{detector,simulated}/{event,calibration}/{daq,pass2_version}/123400/123456/{specific_version_path}
"""
localOpt = ["[ -prefix <Site CLEOc data prefix, e.g. /cdat> ]"]
usage = es_init.helpMsg("ESFixPath", localOpt)
usageDescription = ""
examples = """
"""
userCommand = "ESFixPath.py"
optList, dictOpt = es_init.ESOptions(userCommand, args, usage,
usageDescription, examples)
dbName, dbHost, userName, userPass, dbPort, dbSocket = optList[0]
historyFile, logFile, verbose, profile = optList[1]
userCommand = optList[2]
prefix = "/cdat/" ### CHECK
x = 1
while x < len(args):
try:
if string.lower(args[x]) == "-prefix":
prefix = args[x + 1]
x += 2
continue
if dictOpt.has_key(args[x]):
x += dictOpt[args[x]]
else:
print "Option '%s' is not allowed" % args[x]
raise
except:
sys.exit(1)
if prefix[0] != "/":
print "Prefix should start with /"
print usage
sys.exit(1)
# initialize log
outputLog, globalLog = es_init.ESOutputLog(logFile)
# connect to dbIn EventStore
db, dbType = es_init.ESDBConnector(dbHost, dbName, userName, userPass)
sql = sql_util.SQLUtil(db, dbType, outputLog)
tableList = sql.getTables()
# Initialize ESManager for dbOut
es_init.ESInput(userCommand, outputLog, dbType)
# lock all tables for write operation in dbOut
sql.lockTables()
# postpone all commits to DB, once job will finish we'll invoke commit.
sql.setCommitFlag(0)
sql.startTxn()
try:
# read all releases (from Solaris) which would be used to find out pass2_version
relList = os.listdir(RELPATH)
# start processing
print "Processing:"
sys.__stdout__.flush()
# we can replace retrieval of all files from DB, by using min/max fileId and then
# loop over file Ids.
# query="SELECT fileId,fileName FROM FileID"
# tup = sql.fetchAll(query)
# for item in tup:
# fileId = item[0]
# file = item[1]
query = "SELECT MIN(fileId),MAX(fileId) FROM FileID"
tup = sql.fetchOne(query)
minId = long(tup[0])
maxId = long(tup[1]) + 1
for fileId in xrange(minId, maxId):
query = "SELECT fileName FROM FileID WHERE fileId='%s'" % fileId
tup = sql.fetchOne(query)
if not tup: continue
file = tup[0]
if not os_path_util.isFile(file):
print "Found non existing file", file
continue
# Open KeyFile table, locate fileId, graphid=>svName (with all parents)
# create a new link
keyFile = file
query = "SELECT run,graphid FROM KeyFile WHERE keyFileId='%s'" % fileId
tup = sql.fetchAll(query)
for item in tup:
run = item[0]
gid = item[1]
query = """SELECT svName FROM SpecificVersion,GraphPath
WHERE GraphPath.svid=SpecificVersion.svid
AND GraphPath.graphid='%s'""" % gid
tup = sql.fetchOne(
query) # FIXME, there're many svid's assigned to gid
svName = tup[0]
dir, fileName = os.path.split(keyFile)
dList, idList, dict, dictId, graph = sql.getAllParents(svName)
parentList = newParentList(sql, run, dList)
if string.lower(svName) == 'daq':
release = 'daq'
else:
release = getRelease(relList, svName)
newPath = formNewPath(prefix, run, release, svName, parentList)
newDir = os.path.join(newPath, 'index')
if not os.path.isdir(newDir):
os.makedirs(newDir)
newFile = os.path.join(newDir, fileName)
print "Link (key)", newFile, "=>", keyFile
sys.__stdout__.flush()
if not os_path_util.isFile(newFile):
os.symlink(file, newFile)
# change db entry
query = "UPDATE FileID SET fileName='%s' WHERE fileId='%s'" % (
newFile, fileId)
sql.updateDBAndLog(query)
# Open Location table, locate fileId, graphid=>svName and open
# loc. file header to locate data files
locFile = file
query = "SELECT run,graphid FROM Location WHERE locationFileId='%s'" % fileId
tup = sql.fetchAll(query)
for item in tup:
run = item[0]
gid = item[1]
query = """SELECT svName FROM SpecificVersion,GraphPath
WHERE GraphPath.svid=SpecificVersion.svid
AND GraphPath.graphid='%s'""" % gid
tup = sql.fetchOne(
query) # FIXME, there're many svid's assigned to gid
svName = tup[0]
dir, fileName = os.path.split(locFile)
dList, idList, dict, dictId, graph = sql.getAllParents(svName)
parentList = newParentList(sql, run, dList)
if string.lower(svName) == 'daq':
release = 'daq'
else:
release = getRelease(relList, svName)
newPath = formNewPath(prefix, run, release, svName, parentList)
newDir = os.path.join(newPath, 'index')
if not os.path.isdir(newDir):
os.makedirs(newDir)
newFile = os.path.join(newDir, fileName)
print "Link (loc)", newFile, "=>", locFile
sys.__stdout__.flush()
if not os_path_util.isFile(newFile):
os.symlink(locFile, newFile)
# change db entry
query = "UPDATE FileID SET fileName='%s' WHERE fileId='%s'" % (
newFile, fileId)
sql.updateDBAndLog(query)
# open loc. file header and get data file id's
query = "SELECT fileName,fileId FROM FileID WHERE"
count = 0
for id in file_util.getFileIds(locFile):
if not count:
query += " fileId='%s'" % id
count = 1
else:
query += " OR fileId='%s'" % id
tup = sql.fetchAll(query)
for item in tup:
datFile = '%s' % item[0]
if not os_path_util.isFile(datFile):
print "Found non existing file", datFile
continue
fileId = item[1]
dir, fileName = os.path.split(datFile)
newDir = os.path.join(newPath, 'data')
if not os.path.isdir(newDir):
os.makedirs(newDir)
newFile = os.path.join(newDir, fileName)
print "Link (dat)", newFile, "=>", datFile
sys.__stdout__.flush()
if not os_path_util.isFile(newFile):
os.symlink(datFile, newFile)
# change db entry
query = "UPDATE FileID SET fileName='%s' WHERE fileId='%s'" % (
newFile, fileId)
sql.updateDBAndLog(query)
except:
print "Caught an error during merging step."
gen_util.printExcept()
db.rollback()
return
# everything is ready for commit
sql.setCommitFlag(1)
sql.endTxn()
sql.commit()
sql.unlockTables()
sql.close()
returnStatus = es_init.ESOutput(1, userCommand, historyFile, outputLog,
globalLog)
return returnStatus
def ESAddComment(args):
"""ESAddComment was designed to add comments into EventStore DB.
To add your comment you need to provide old/new grade or data
version name and old/new timeStamp. The comment can be added
either from command line or can be read from ASCII file.
Please note, ESAddComment is a wrapper shell script around addComment.py
module which does the work.
"""
localOpt = [
"[ -grade <grade> -timeStamp <time> | -dataVersionName <name> ]"
]
localOpt.append("[ -date <date> ] [ -comment <someText> ]")
usage = es_init.helpMsg("ESAddComment", localOpt)
usageDescription = """
Option description:
To add your comment to EventStore, either use grade/time or dataVersionName.
"""
examples = """
Examples:
ESAddComment -grade physics -timeStamp 20090101 -comment Add new physics grade
ESAddComment -dataVersionName PP2-20090101 -comment myComment.txt
in last example we add comment from a myComment.txt file.
"""
userCommand = "addComment.py"
optList, dictOpt = es_init.ESOptions(userCommand, args, usage,
usageDescription, examples)
dbName, dbHost, userName, userPass, dbPort, dbSocket = optList[0]
historyFile, logFile, verbose, profile = optList[1]
userCommand = optList[2]
grade = ""
cTime = time.strftime("%Y%m%d", time.localtime())
timeStamp = ""
x = 1
svName = ""
host = ""
verbose = 0
comment = "%s -- " % time.strftime("%H:%M:%S", time.localtime())
duplicateDBEntry = 0
while x < len(args):
try:
if args[x] == "-dataVersionName":
svName = args[x + 1]
checkArg([svName])
x += 2
continue
if args[x] == "-grade":
grade = args[x + 1]
checkArg([grade])
x += 2
continue
if args[x] == "-timeStamp":
timeStamp = args[x + 1]
checkArg([timeStamp])
x += 2
continue
if args[x] == "-date":
cTime = args[x + 1]
checkArg([cTime])
x += 2
continue
if args[x] == "-comment":
x += 1
while (args[x][0] != "-"):
if os.path.isfile(args[x]):
comment += open(args[x]).read()
break
comment += args[x]
x += 1
if len(args) == x:
break
continue
# if we reach here, that means we found unkown option
if dictOpt.has_key(args[x]):
x += dictOpt[args[x]]
else:
print "Option '%s' is not allowed" % args[x]
raise
except:
sys.exit(1)
if grade and svName:
print "Both grade=%s and dataVersionName=%s found" % (grade, svName)
print usage
sys.exit(1)
if len(cTime) != 8:
print "Incorrect date=%s format found, please use YYYYMMDD" % cTime
sys.exit(1)
# connect to MySQL EventStoreDB
outputLog, globalLog = es_init.ESOutputLog(logFile)
db, dbType = es_init.ESDBConnector(dbHost, dbName, userName, userPass,
dbPort, dbSocket)
pid = "%s" % os.getpid()
localtime = "%s" % time.strftime("%H:%M:%S", time.localtime())
outputLog.write("\n%s %s ###### %s initialization is completed" %
(pid, localtime, dbType))
if dbType == "sqlite":
addToQuery = ""
else:
addToQuery = " FOR UPDATE"
cu = db.cursor()
listOfSVIDs = []
if not svName:
query = """SELECT SpecificVersion.svid FROM Version,SpecificVersion,GraphPath WHERE
timeStamp='%s' AND grade='%s' AND Version.graphid=GraphPath.graphid
AND GraphPath.svid=SpecificVersion.svid
""" % (timeStamp, grade)
if verbose:
print string.join(string.split(query))
cu.execute(query)
tup = cu.fetchall()
if not len(tup) or not tup:
print "Upon your request, the following query return NULL results\n", query
print "Please check that provided grade/time or dataVersionName exists in ES"
sys.exit(1)
for item in tup:
if not listOfSVIDs.count(item[0]):
listOfSVIDs.append(item[0])
else:
query = "SELECT svid FROM SpecificVersion WHERE svName='%s'" % svName
if verbose:
print string.join(string.split(query))
cu.execute(query)
tup = cu.fetchone()
if not tup:
print "Upon your request, the following query return NULL results\n", query
print "Please check that provided grade/time or dataVersionName exists in ES"
sys.exit(1)
listOfSVIDs.append(tup[0])
if not len(listOfSVIDs):
print "No matches in ES DB found for your request"
sys.exit(1)
for svid in listOfSVIDs:
modComment = string.replace(comment, "'", "\\'")
query = """INSERT INTO SpecificVersionComment (svid,CommentDate,Comment)
VALUES('%s','%s','%s')""" % (svid, cTime, modComment)
if verbose:
print string.join(string.split(query))
if dbType == "mysql":
cu.execute("BEGIN")
cu.execute(query)
if dbType == "mysql":
cu.execute("COMMIT")
else:
db.commit()
def ESBuilder(args):
"""ESBuilder is a main injection tool. It supports two types of DBs:
MySQL and SQLite. The injection can be done for variety of file formats:
evio, hddm, idxa. For option information and usage please use '-help' option.
For option description '--help'.
For specific injection types please use '-examples' option.
Please note, ESBuilder is a wrapper shell script around ESBuilder.py
module which does the work.
"""
localOpt = ["[ -add <dir or file or pattern of files> ]"]
localOpt.append("[ -grade <grade> ] [ -time <timeStamp> ]")
localOpt.append("[ -dataVersionName <name> ] [ -view <skim> ]")
localOpt.append("[ -listOfParents <dataVersionName's> ]")
localOpt.append("[ -output <dir> ] [ -HSMDir <HSM directory> ]")
localOpt.append("[ -dupRead <fileName> ] [ -skim ] [ -no-skim ]")
localOpt.append("[ -masterDB <name@host:port:socket or fileName> ]")
usage = es_init.helpMsg("ESBuilder", localOpt)
usageDescription = """
Option description:
* -grade: specifies the grade, e.g. "physics", "p2-unchecked"
* -add: adds data file(s) to the EventStore
You may specify: directory, file name or a list of files
For patterns use '*', e.g MC*tau*.pds
* -output: output location for storing key/location files
* -dataVersionName: specifies the data version name (aka svName)
-time: specifies the timeStamp, e.g. 20090227. If time is not provided
will try to append to existing grade/dataVersionName or use a
one day in a future as new timeStamp if no grade/dataVersionName
combination is found.
-view: specifies the view, e.g. "tau"
-listOfParents specifies list of parents for given injection,
e.g. while injecting p2-unchecked grade its parent is 'daq'.
-newDB: force the creation of a new EventStore
-sqlite use the SQLite version of EventStore
default sqlite.db, otherwise a fileName needs to be provided
-mysql use the MySQL version of EventStore. In order to access MySQL
you need either provide login/password through the -user/-password
options or create $HOME/.esdb.conf with user:password entry
-masterDB specifies host and db name of the master you want to use
-verbose: verbose mode, a lot of useful printout
-idleMode when this flag is specified, no key/location file will be
generated (useful once you have them and want reproduce DB
content). But content of DB will be updated. USE WITH CAUTION.
-delete delete a grade from EventStore. USE WITH CAUTION.
You need to provide the grade and the timeStamp.
-HSMDir specifies output HSM directory.
-logFile specifies the log file name. You may either provide a full file name
(including path) or 'stdout' or 'stderr' to redirect your log to
appropriate I/O stream. During injection an intermidiate files
esdb.log.YYYYMMDD_HHMMSS_PID will be created.
Once a job successfully finishes, the esdb.log.YYYYMMDD_HHMMSS_PID
is moved to your logFile, otherwise esdb.log.YYYYMMDD_HHMMSS_PID remains.
-profile perform internal profiling.
-dupRead in the case of duplicated records force to use this source
-skim force ESBuilder to use input files as a skim, i.e. find their parents
and build combined location file for all of them
-no-skim force ESBuilder to use input files as is
Please note: required parameters are marked with (*). All options can be
specified in any order. By default: view='all', EventStoreTMP DB is used and key/location
files are generated.
"""
examples = es_init.ESExamples()
userCommand = "ESBuilder.py"
optList, dictOpt = es_init.ESOptions(userCommand, args, usage, usageDescription)
dbName, dbHost, userName, userPass, dbPort, dbSocket = optList[0]
historyFile, logFile, verbose, profile = optList[1]
userCommand = optList[2]
# default values
grade = ""
timeS = gen_util.dayAhead()
oDir = ""
view = "all"
run = 0
file = ""
newDB = 0
delete = 0
genMode = 1
minRun = 0
maxRun = 1000000
localtime = time.strftime("%Y%m%d_%H%M%S", time.localtime())
uname = os.uname()
svName = ""
tempLogFile = "esdb.log.%s_%s" % (localtime, os.getpid())
fileList = []
listOfParents = []
oHSMDir = ""
dupRead = ""
skim = 0
noskim = 0
masterDBName = dbName
masterDBHost = dbHost
master = ""
masterDB = ""
masterDBPort = dbPort
masterDBSocket = dbSocket
# parse the rest of the options and form user's command
x = 1
doNotRead = 0
while x < len(args):
try:
if args[x] == "-newDB":
newDB = 1
x += 1
continue
if args[x] == "-HSMDir":
oHSMDir = args[x + 1]
checkArg([oHSMDir])
x += 2
continue
if args[x] == "-dupRead":
dupRead = args[x + 1]
checkArg([dupRead])
x += 2
continue
if args[x] == "-dataVersionName":
svName = args[x + 1]
checkArg([svName])
x += 2
continue
if args[x] == "-grade":
grade = args[x + 1]
checkArg([grade])
x += 2
continue
if args[x] == "-time":
timeS = args[x + 1]
checkArg([timeS])
x += 2
continue
if args[x] == "-output":
oDir = args[x + 1] + "/"
checkArg([oDir])
x += 2
continue
if args[x] == "-runRange":
minRun = int(args[x + 1])
maxRun = int(args[x + 2])
checkArg([minRun, maxRun])
x += 3
continue
if args[x] == "-listOfParents":
x += 1
while args[x][0] != "-":
newArg = args[x]
listOfParents.append(args[x])
x += 1
if len(args) == x:
break
checkArg(listOfParents)
continue
if args[x] == "-add":
file = os_path_util.formAbsolutePath(args[x + 1])
# first check if pattern is present
if len(args) > x + 2 and args[x + 2][0] != "-":
counter = 0
for idx in xrange(x + 1, len(args)):
newArg = args[idx]
if newArg[0] == "-":
break
counter += 1
if os.path.isfile(newArg):
fileList.append(os_path_util.formAbsolutePath(newArg))
x += counter + 1
continue
elif os.path.isdir(file):
dir = file + "/"
for f in os.listdir(dir):
if string.split(f, ".")[-1] != "pds":
continue
fileName = dir + f
fileList.append(os_path_util.formAbsolutePath(fileName))
x += 2
continue
elif os_path_util.isFile(file):
if file[-5:] == ".list":
tmpList = open(file).readlines()
for item in tmpList:
fileList.append(string.split(item)[0])
else:
fileList = [file]
x += 2
continue
# check if this file exists
else:
print "ESBuilder: no such file", file
raise
checkArg(fileList)
if args[x] == "-view":
view = args[x + 1]
checkArg([view])
x += 2
continue
if args[x] == "-idleMode":
genMode = 0
x += 1
continue
if args[x] == "-skim":
skim = 1
x += 1
continue
if args[x] == "-no-skim":
noskim = 1
x += 1
continue
if args[x] == "-masterDB":
masterDB = args[x + 1]
master = 1
checkArg([masterDB])
x += 2
continue
# if we reach here, that means we found unkown option
if dictOpt.has_key(args[x]):
x += dictOpt[args[x]]
else:
print "Option '%s' is not allowed" % args[x]
raise
except:
sys.exit(1)
### AUTHENTICATION???
# check that USER=pass2, otherwise exit
authUsers = ["gluex", "sdobbs"] ### CHECK
# check if USER environment is set up, otherwise use LOGNAME
env = os.environ
if not env.has_key("USER"):
os.environ["USER"] = env["LOGNAME"]
if not authUsers.count(os.environ["USER"]) and dbName == "EventStore" and string.find(dbHost, "hallddb") != -1:
print "ERROR: Injection to db='EventStore' should be done from official (gluex) account for %s DB\n" % dbName
print "For your own injection please use another db name"
sys.exit(1)
# check underlying OS, so far we only allow to inject from SunOS
# if os.environ["USER"]=="pass2" and uname[0]!="SunOS":
# print "ERROR: for pass2 account the EventStore injection should be done from SunOS\n"
# sys.exit(1)
#######################################
# form normalized abosulte paths
oDir = os_path_util.formAbsolutePath(oDir)
# check required parameters
if not len(grade):
print "ESBuilder requires to specify a grade, see -grade option"
sys.exit(1)
if string.find(grade, "unchecked") == -1 and view == "all": ### CHECK
print "ESBuilder only allow to inject 'unchecked' grades"
print " daq-unechecked, p2-unchecked, physics-unchecked"
print "Either specify different view or inject as unchecked grade"
print "Given grade='%s' view='%s'" % (grade, view)
sys.exit(1)
if not len(fileList):
print "ESBuilder requires to specify input file(s) with -add option"
sys.exit(1)
# check permissions and access to output dir
if not os.path.isdir(oDir):
print "Output directory '%s' doesn't exists" % oDir
print "ESBuilder requires to specify output dir to store key/location files, see -output option"
sys.exit(1)
if oDir and not os_path_util.checkPermission(oDir):
print "You don't have permission to write to output area '%s'" % oDir
sys.exit(1)
# check permission to write to HSM
if oHSMDir and not os.path.isdir(oHSMDir):
print "HSM directory '%s' does not exists" % oHSMDir
sys.exit(1)
if not os_path_util.checkPermission(oHSMDir):
print "You don't have permission to write to HSM location '%s'" % oHSMDir
sys.exit(1)
# check that all inputs are in place
for file in fileList:
if not os.path.isfile(file):
print "File '%s' does not exists" % file
sys.exit(1)
if dupRead and not os.path.isfile(dupRead):
print "File '%s' does not exists" % dupRead
sys.exit(1)
# connect to MySQL EventStoreDB
outputLog, globalLog = es_init.ESOutputLog(logFile)
db, dbType = es_init.ESDBConnector(dbHost, dbName, userName, userPass, "", dbPort, dbSocket)
es_init.ESInput(userCommand, outputLog, dbType)
# Be verbose
dbinfo = "\t grade\t'%s'\n\t timeStamp\t'%s'\n\t view\t\t'%s'\n" % (grade, timeS, view)
if newDB:
if verbose:
print "Creating new tables DB:"
print dbinfo
else:
if verbose:
print "Updating existing tables in DB:"
print dbinfo
if genMode == 0 and verbose:
print "\n\t ===> Process running in Idle mode"
# create instance of ESManager class
mydb = ESManager.ESManager(db, dbType, outputLog)
# set-up all parameters
mydb.setOutputDir(oDir)
mydb.setGenerateDB(newDB)
mydb.setSVName(svName)
mydb.setParents(listOfParents)
mydb.setGrade(grade)
mydb.setTimeStamp(timeS)
mydb.setView(view)
mydb.setMinRun(minRun)
mydb.setMaxRun(maxRun)
mydb.setVerboseLevel(verbose)
mydb.setReadDuplicatesSource(dupRead)
mydb.setSkimFlag(skim)
mydb.setNoSkimFlag(noskim)
mydb.setDBHost(dbHost)
mydb.setDBName(dbName)
mydb.setDBPort(dbPort)
mydb.setDBSocket(dbSocket)
# interpret the master option
if masterDB:
dbComponents = string.split(masterDB, "@")
if len(dbComponents) == 2:
masterDBName = dbComponents[0]
newComponents = string.split(dbComponents[1], ":")
masterDBHost = newComponents[0]
port = socket = ""
if len(newComponents) == 2:
port = newComponents[1]
elif len(newComponents) == 3:
socket = newComponents[2]
# masterDBHost,port,socket=string.split(dbComponents[1],":")
if port:
masterDBPort = port
if socket:
masterDBSocket = socket
else:
masterDBHost = dbComponents[0]
else:
login, adminInfo, cMasterName, cMasterHost, cMasterPort, cMasterSocket = esdb_auth.readConfigFile()
if cMasterHost:
masterDBHost = cMasterHost
masterDBName = cMasterName
masterDBPort = cMasterPort
masterDBSocket = cMasterSocket
mydb.setMasterDB(masterDBName, masterDBHost, masterDBPort, masterDBSocket)
# update DB using transaction
if delete:
status = mydb.deleteGrade(delGrade, delTime)
else: # for anything else
try:
status = mydb.updateDB(genMode, fileList, oHSMDir)
except:
print "ERROR: fail to process:"
for item in fileList:
print item
print "--------------- See traceback ----------------"
raise
# close connection to db
mydb.commit()
mydb.close()
returnStatus = es_init.ESOutput(status, userCommand, historyFile, outputLog, globalLog)
return returnStatus
def ESFixPath(args):
"""Fix paths in EventStoreDB. The CLEOc data path specifications:
/cleo/{detector,simulated}/{event,calibration}/{daq,pass2_version}/123400/123456/{specific_version_path}
"""
localOpt = ["[ -prefix <Site CLEOc data prefix, e.g. /cdat> ]"]
usage = es_init.helpMsg("ESFixPath",localOpt)
usageDescription=""
examples="""
"""
userCommand="ESFixPath.py"
optList, dictOpt = es_init.ESOptions(userCommand,args,usage,usageDescription,examples)
dbName,dbHost,userName,userPass,dbPort,dbSocket = optList[0]
historyFile,logFile,verbose,profile = optList[1]
userCommand = optList[2]
prefix = "/cdat/" ### CHECK
x = 1
while x < len(args):
try:
if string.lower(args[x]) == "-prefix":
prefix = args[x+1]
x+=2
continue
if dictOpt.has_key(args[x]):
x+=dictOpt[args[x]]
else:
print "Option '%s' is not allowed"%args[x]
raise
except:
sys.exit(1)
if prefix[0]!="/":
print "Prefix should start with /"
print usage
sys.exit(1)
# initialize log
outputLog, globalLog = es_init.ESOutputLog(logFile)
# connect to dbIn EventStore
db, dbType = es_init.ESDBConnector(dbHost,dbName,userName,userPass)
sql = sql_util.SQLUtil(db,dbType,outputLog)
tableList = sql.getTables()
# Initialize ESManager for dbOut
es_init.ESInput(userCommand,outputLog,dbType)
# lock all tables for write operation in dbOut
sql.lockTables()
# postpone all commits to DB, once job will finish we'll invoke commit.
sql.setCommitFlag(0)
sql.startTxn()
try:
# read all releases (from Solaris) which would be used to find out pass2_version
relList = os.listdir(RELPATH)
# start processing
print "Processing:"
sys.__stdout__.flush()
# we can replace retrieval of all files from DB, by using min/max fileId and then
# loop over file Ids.
# query="SELECT fileId,fileName FROM FileID"
# tup = sql.fetchAll(query)
# for item in tup:
# fileId = item[0]
# file = item[1]
query="SELECT MIN(fileId),MAX(fileId) FROM FileID"
tup = sql.fetchOne(query)
minId= long(tup[0])
maxId= long(tup[1])+1
for fileId in xrange(minId,maxId):
query = "SELECT fileName FROM FileID WHERE fileId='%s'"%fileId
tup = sql.fetchOne(query)
if not tup: continue
file = tup[0]
if not os_path_util.isFile(file):
print "Found non existing file",file
continue
# Open KeyFile table, locate fileId, graphid=>svName (with all parents)
# create a new link
keyFile= file
query ="SELECT run,graphid FROM KeyFile WHERE keyFileId='%s'"%fileId
tup = sql.fetchAll(query)
for item in tup:
run = item[0]
gid = item[1]
query="""SELECT svName FROM SpecificVersion,GraphPath
WHERE GraphPath.svid=SpecificVersion.svid
AND GraphPath.graphid='%s'"""%gid
tup = sql.fetchOne(query) # FIXME, there're many svid's assigned to gid
svName = tup[0]
dir,fileName = os.path.split(keyFile)
dList,idList,dict,dictId,graph=sql.getAllParents(svName)
parentList = newParentList(sql,run,dList)
if string.lower(svName)=='daq':
release = 'daq'
else:
release = getRelease(relList,svName)
newPath = formNewPath(prefix,run,release,svName,parentList)
newDir = os.path.join(newPath,'index')
if not os.path.isdir(newDir):
os.makedirs(newDir)
newFile = os.path.join(newDir,fileName)
print "Link (key)",newFile,"=>",keyFile
sys.__stdout__.flush()
if not os_path_util.isFile(newFile):
os.symlink(file,newFile)
# change db entry
query="UPDATE FileID SET fileName='%s' WHERE fileId='%s'"%(newFile,fileId)
sql.updateDBAndLog(query)
# Open Location table, locate fileId, graphid=>svName and open
# loc. file header to locate data files
locFile = file
query = "SELECT run,graphid FROM Location WHERE locationFileId='%s'"%fileId
tup = sql.fetchAll(query)
for item in tup:
run = item[0]
gid = item[1]
query="""SELECT svName FROM SpecificVersion,GraphPath
WHERE GraphPath.svid=SpecificVersion.svid
AND GraphPath.graphid='%s'"""%gid
tup = sql.fetchOne(query) # FIXME, there're many svid's assigned to gid
svName = tup[0]
dir,fileName = os.path.split(locFile)
dList,idList,dict,dictId,graph=sql.getAllParents(svName)
parentList = newParentList(sql,run,dList)
if string.lower(svName)=='daq':
release = 'daq'
else:
release = getRelease(relList,svName)
newPath = formNewPath(prefix,run,release,svName,parentList)
newDir = os.path.join(newPath,'index')
if not os.path.isdir(newDir):
os.makedirs(newDir)
newFile = os.path.join(newDir,fileName)
print "Link (loc)",newFile,"=>",locFile
sys.__stdout__.flush()
if not os_path_util.isFile(newFile):
os.symlink(locFile,newFile)
# change db entry
query="UPDATE FileID SET fileName='%s' WHERE fileId='%s'"%(newFile,fileId)
sql.updateDBAndLog(query)
# open loc. file header and get data file id's
query="SELECT fileName,fileId FROM FileID WHERE"
count=0
for id in file_util.getFileIds(locFile):
if not count:
query+= " fileId='%s'"%id
count =1
else:
query+= " OR fileId='%s'"%id
tup = sql.fetchAll(query)
for item in tup:
datFile = '%s'%item[0]
if not os_path_util.isFile(datFile):
print "Found non existing file",datFile
continue
fileId = item[1]
dir,fileName = os.path.split(datFile)
newDir = os.path.join(newPath,'data')
if not os.path.isdir(newDir):
os.makedirs(newDir)
newFile = os.path.join(newDir,fileName)
print "Link (dat)",newFile,"=>",datFile
sys.__stdout__.flush()
if not os_path_util.isFile(newFile):
os.symlink(datFile,newFile)
# change db entry
query="UPDATE FileID SET fileName='%s' WHERE fileId='%s'"%(newFile,fileId)
sql.updateDBAndLog(query)
except:
print "Caught an error during merging step."
gen_util.printExcept()
db.rollback()
return
# everything is ready for commit
sql.setCommitFlag(1)
sql.endTxn()
sql.commit()
sql.unlockTables()
sql.close()
returnStatus = es_init.ESOutput(1,userCommand,historyFile,outputLog,globalLog)
return returnStatus
try:
os.remove(fileIn)
if self.verbose:
print "File %s has been successfully moved"%fileIn
except:
gen_util.printExcept()
print "WARNING: cannot remove %s"%fileIn
return (self.ok, query, cQuery)
return (self.ok, query, cQuery)
return (self.error, query, cQuery) # error
#
# main
#
if __name__ == "__main__":
localOpt=["[ -move <fileIn> <fileOutDir> ]"]
usage=es_init.helpMsg("ESMove",localOpt)
usageDescription="""
Option description:
"""
examples = es_init.ESExamples()
userCommand="ESMove.py"
optList, dictOpt = es_init.ESOptions(userCommand,sys.argv,usage,usageDescription)
dbName,dbHost,userName,userPass,dbPort,dbSocket = optList[0]
historyFile,logFile,verbose,profile = optList[1]
userCommand = optList[2]
# default values
fileInList=[]
fileIn = ""
fileOut = ""
self.startTxn()
self.updateDBAndLog(query)
self.endTxn()
except:
pass # we insert duplicate it's fine
# delete all location files
if self.doDelete(locIdDict,"Location")!=self.ok:
return self.error
return self.ok
#
# main
#
if __name__ == "__main__":
localOpt=["[ -delete <grade> <timeStamp> ]"]
usage=es_init.helpMsg("ESDelete",localOpt)
usageDescription="""
Option description:
"""
examples = es_init.ESExamples()
userCommand="ESDelete.py"
optList, dictOpt = es_init.ESOptions(userCommand,sys.argv,usage,usageDescription)
dbName,dbHost,userName,userPass,dbPort,dbSocket = optList[0]
historyFile,logFile,verbose,profile = optList[1]
userCommand = optList[2]
# default values
delGrade = ""
delTime = ""
else:
self.printDBContent(dbTable)
return self.ok
#
# main
#
if __name__ == "__main__":
localOpt = [
"[ -info ] [ -runList <minRun> <maxRun> ] [ -time <YYYYDDMM> ]"
]
localOpt.append(
"[ -findFileForRun <run> ] [ -showDepend <dataVersionName> ]")
localOpt.append("[ -dbTable <dbTable> ] [ -tableSchema <tableName>]")
usage = es_init.helpMsg("ESDump", localOpt)
usageDescription = """
Option description:
"""
examples = es_init.ESExamples()
userCommand = "ESDump.py"
optList, dictOpt = es_init.ESOptions(userCommand, sys.argv, usage,
usageDescription)
dbName, dbHost, userName, userPass, dbPort, dbSocket = optList[0]
historyFile, logFile, verbose, profile = optList[1]
userCommand = optList[2]
usageDescription = """
ESDump provides usefull information about data store in EventStore.
Default ESDump print content of group EventStore at local time
def dumpTable(self,dbTable):
"""Prints table content. It uses sql_util module to do the job."""
if dbTable=="all":
for x in self.dbNames:
self.printDBContent(x)
else:
self.printDBContent(dbTable)
return self.ok
#
# main
#
if __name__ == "__main__":
localOpt=["[ -info ] [ -runList <minRun> <maxRun> ] [ -time <YYYYDDMM> ]"]
localOpt.append("[ -findFileForRun <run> ] [ -showDepend <dataVersionName> ]")
localOpt.append("[ -dbTable <dbTable> ] [ -tableSchema <tableName>]")
usage=es_init.helpMsg("ESDump",localOpt)
usageDescription="""
Option description:
"""
examples = es_init.ESExamples()
userCommand="ESDump.py"
optList, dictOpt = es_init.ESOptions(userCommand,sys.argv,usage,usageDescription)
dbName,dbHost,userName,userPass,dbPort,dbSocket = optList[0]
historyFile,logFile,verbose,profile = optList[1]
userCommand = optList[2]
usageDescription="""
ESDump provides usefull information about data store in EventStore.
Default ESDump print content of group EventStore at local time
(MySQL server at Cornell)
tup = self.fetchAll(q)
for x in tup:
print x[0]
def process(self):
self.lookupKeyFiles()
self.lookupLocationFiles()
return self.ok
#
# main
#
if __name__ == "__main__":
localOpt = ["[ -graphid gid ]"]
usage = es_init.helpMsg("ESLookupData", localOpt)
usageDescription = """
Option description:
"""
examples = ""
userCommand = "ESLookupData.py"
optList, dictOpt = es_init.ESOptions(userCommand, sys.argv, usage,
usageDescription)
dbName, dbHost, userName, userPass, dbPort, dbSocket = optList[0]
historyFile, logFile, verbose, profile = optList[1]
userCommand = optList[2]
# parse the rest of the options and form user's command
x = 1
counter = 0
while x < len(sys.argv):
try:
self.updateDBAndLog(query)
self.endTxn()
except:
pass # we insert duplicate it's fine
# delete all location files
if self.doDelete(locIdDict, "Location") != self.ok:
return self.error
return self.ok
#
# main
#
if __name__ == "__main__":
localOpt = ["[ -delete <grade> <timeStamp> ]"]
usage = es_init.helpMsg("ESDelete", localOpt)
usageDescription = """
Option description:
"""
examples = es_init.ESExamples()
userCommand = "ESDelete.py"
optList, dictOpt = es_init.ESOptions(userCommand, sys.argv, usage,
usageDescription)
dbName, dbHost, userName, userPass, dbPort, dbSocket = optList[0]
historyFile, logFile, verbose, profile = optList[1]
userCommand = optList[2]
# default values
delGrade = ""
query = "SELECT fileName FROM FileID,KeyFile WHERE graphid='%d' AND view='%s' AND run='%d' AND FileID.fileId=KeyFile.keyFileId"%(graphID,view,run)
if uid>0:
query += " AND uid='%d'"%uid
self.cu.execute(query)
#
print " Key filename = %s" % self.cu.fetchone()
return self.ok
#
# main
#
if __name__ == "__main__":
# localOpt=["[ -noHeader ][ -runList <minRun> <maxRun> ] [ -time <YYYYDDMM> ]"]
localOpt=["[ -noHeader ]"]
usage=es_init.helpMsg("ESQuery",localOpt)
usageDescription="""
ESQuery provides a user-friendly interface to get information from EventStore databases.
Command list with brief description:
grades - list of grades in DB
timestamps <grade> - list of timestamps for <grade>
skims <grade> [-time <YYYYDDMM>] - list of skims for <grade>
runs <grades> - list of runs for <grade>
actualDate <grade> [-time <YYYYDDMM>] - more recent date in DB to given time
verions <grade> [-time <YYYYDDMM>] - data version info for <grade>
graphVerions <grade> [-time <YYYYDDMM>] - graph version info for <grade>
Development commands:
if self.verbose:
print "File %s has been successfully moved" % fileIn
except:
gen_util.printExcept()
print "WARNING: cannot remove %s" % fileIn
return (self.ok, query, cQuery)
return (self.ok, query, cQuery)
return (self.error, query, cQuery) # error
#
# main
#
if __name__ == "__main__":
localOpt = ["[ -move <fileIn> <fileOutDir> ]"]
usage = es_init.helpMsg("ESMove", localOpt)
usageDescription = """
Option description:
"""
examples = es_init.ESExamples()
userCommand = "ESMove.py"
optList, dictOpt = es_init.ESOptions(userCommand, sys.argv, usage,
usageDescription)
dbName, dbHost, userName, userPass, dbPort, dbSocket = optList[0]
historyFile, logFile, verbose, profile = optList[1]
userCommand = optList[2]
# default values
fileInList = []
fileIn = ""
for idx in xrange(0,len(fidList),2):
pdsID = (fidList[idx+1]<<32)|fidList[idx]
q="""select fileName from FileID where fileid=%s"""%pdsID
tup = self.fetchAll(q)
for x in tup:
print x[0]
def process(self):
self.lookupKeyFiles()
self.lookupLocationFiles()
return self.ok
#
# main
#
if __name__ == "__main__":
localOpt=["[ -graphid gid ]"]
usage=es_init.helpMsg("ESLookupData",localOpt)
usageDescription="""
Option description:
"""
examples = ""
userCommand="ESLookupData.py"
optList, dictOpt = es_init.ESOptions(userCommand,sys.argv,usage,usageDescription)
dbName,dbHost,userName,userPass,dbPort,dbSocket = optList[0]
historyFile,logFile,verbose,profile = optList[1]
userCommand = optList[2]
# parse the rest of the options and form user's command
x = 1
counter = 0
while x < len(sys.argv):
try:
if sys.argv[x] == "-graphid":