def findLatestTimeStamp(self,iGrade):
        """Search for latest timeStamp in Version table, it will assign a day ahead
	for non existing grade."""
	query = "SELECT MAX(timeStamp) FROM Version WHERE grade='%s'"%iGrade
	if self.verbose:
	   print query
	self.updateDBAndLog(query)
	tup = self.fetchOne(query)
	if tup and tup[0]!=None:
	   tStamp= tup[0]
	else:
	   # no timeStamp found for given grade, we'll use one day ahead
	   tStamp = gen_util.dayAhead()
	   print "No valid timeStamp found for '%s'. We'll use %s"%(iGrade,tStamp)
	if self.verbose:
	   print "Use timeStamp:",tStamp
	return tStamp
示例#2
0
def ESBuilder(args):
    """ESBuilder is a main injection tool. It supports two types of DBs:
    MySQL and SQLite. The injection can be done for variety of file formats:
    evio, hddm, idxa. For option information and usage please use '-help' option.
    For option description '--help'.
    For specific injection types please use '-examples' option.

    Please note, ESBuilder is a wrapper shell script around ESBuilder.py
    module which does the work.
    """
    localOpt = ["[ -add <dir or file or pattern of files> ]"]
    localOpt.append("[ -grade <grade> ] [ -time <timeStamp> ]")
    localOpt.append("[ -dataVersionName <name> ]  [ -view <skim> ]")
    localOpt.append("[ -listOfParents <dataVersionName's> ]")
    localOpt.append("[ -output <dir> ] [ -HSMDir <HSM directory> ]")
    localOpt.append("[ -dupRead <fileName> ] [ -skim ] [ -no-skim ]")
    localOpt.append("[ -masterDB <[email protected]:port:socket or fileName> ]")
    usage = es_init.helpMsg("ESBuilder", localOpt)
    usageDescription = """
Option description:
*   -grade:   specifies the grade, e.g. "physics", "p2-unchecked"
*   -add:     adds data file(s) to the EventStore
	      You may specify: directory, file name or a list of files
	      For patterns use '*', e.g MC*tau*.pds
*   -output:  output location for storing key/location files
*   -dataVersionName: specifies the data version name (aka svName)

    -time:    specifies the timeStamp, e.g. 20090227. If time is not provided 
              will try to append to existing grade/dataVersionName or use a 
	      one day in a future as new timeStamp if no grade/dataVersionName
	      combination is found.
    -view:    specifies the view, e.g. "tau"
    -listOfParents specifies list of parents for given injection,
	      e.g. while injecting p2-unchecked grade its parent is 'daq'.
    -newDB:   force the creation of a new EventStore
    -sqlite   use the SQLite version of EventStore
	      default sqlite.db, otherwise a fileName needs to be provided
    -mysql    use the MySQL version of EventStore. In order to access MySQL
	      you need either provide login/password through the -user/-password
	      options or create $HOME/.esdb.conf with user:password entry
    -masterDB specifies host and db name of the master you want to use
    -verbose: verbose mode, a lot of useful printout
    -idleMode when this flag is specified, no key/location file will be
	      generated (useful once you have them and want reproduce DB
	      content). But content of DB will be updated. USE WITH CAUTION.
    -delete   delete a grade from EventStore. USE WITH CAUTION.
	      You need to provide the grade and the timeStamp.
    -HSMDir   specifies output HSM directory.
    -logFile  specifies the log file name. You may either provide a full file name 
              (including path) or 'stdout' or 'stderr' to redirect your log to
	      appropriate I/O stream. During injection an intermidiate files
	      esdb.log.YYYYMMDD_HHMMSS_PID will be created.
	      Once a job successfully finishes, the esdb.log.YYYYMMDD_HHMMSS_PID 
	      is moved to your logFile, otherwise esdb.log.YYYYMMDD_HHMMSS_PID remains.
    -profile  perform internal profiling.
    -dupRead  in the case of duplicated records force to use this source
    -skim     force ESBuilder to use input files as a skim, i.e. find their parents
              and build combined location file for all of them
    -no-skim  force ESBuilder to use input files as is

Please note: required parameters are marked with (*). All options can be
specified in any order. By default: view='all', EventStoreTMP DB is used and key/location
files are generated.

    """

    examples = es_init.ESExamples()
    userCommand = "ESBuilder.py"
    optList, dictOpt = es_init.ESOptions(userCommand, args, usage, usageDescription)
    dbName, dbHost, userName, userPass, dbPort, dbSocket = optList[0]
    historyFile, logFile, verbose, profile = optList[1]
    userCommand = optList[2]

    # default values
    grade = ""
    timeS = gen_util.dayAhead()
    oDir = ""
    view = "all"
    run = 0
    file = ""
    newDB = 0
    delete = 0
    genMode = 1
    minRun = 0
    maxRun = 1000000
    localtime = time.strftime("%Y%m%d_%H%M%S", time.localtime())
    uname = os.uname()
    svName = ""
    tempLogFile = "esdb.log.%s_%s" % (localtime, os.getpid())
    fileList = []
    listOfParents = []
    oHSMDir = ""
    dupRead = ""
    skim = 0
    noskim = 0
    masterDBName = dbName
    masterDBHost = dbHost
    master = ""
    masterDB = ""
    masterDBPort = dbPort
    masterDBSocket = dbSocket
    # parse the rest of the options and form user's command
    x = 1
    doNotRead = 0
    while x < len(args):
        try:
            if args[x] == "-newDB":
                newDB = 1
                x += 1
                continue
            if args[x] == "-HSMDir":
                oHSMDir = args[x + 1]
                checkArg([oHSMDir])
                x += 2
                continue
            if args[x] == "-dupRead":
                dupRead = args[x + 1]
                checkArg([dupRead])
                x += 2
                continue
            if args[x] == "-dataVersionName":
                svName = args[x + 1]
                checkArg([svName])
                x += 2
                continue
            if args[x] == "-grade":
                grade = args[x + 1]
                checkArg([grade])
                x += 2
                continue
            if args[x] == "-time":
                timeS = args[x + 1]
                checkArg([timeS])
                x += 2
                continue
            if args[x] == "-output":
                oDir = args[x + 1] + "/"
                checkArg([oDir])
                x += 2
                continue
            if args[x] == "-runRange":
                minRun = int(args[x + 1])
                maxRun = int(args[x + 2])
                checkArg([minRun, maxRun])
                x += 3
                continue
            if args[x] == "-listOfParents":
                x += 1
                while args[x][0] != "-":
                    newArg = args[x]
                    listOfParents.append(args[x])
                    x += 1
                    if len(args) == x:
                        break
                checkArg(listOfParents)
                continue
            if args[x] == "-add":
                file = os_path_util.formAbsolutePath(args[x + 1])
                # first check if pattern is present
                if len(args) > x + 2 and args[x + 2][0] != "-":
                    counter = 0
                    for idx in xrange(x + 1, len(args)):
                        newArg = args[idx]
                        if newArg[0] == "-":
                            break
                        counter += 1
                        if os.path.isfile(newArg):
                            fileList.append(os_path_util.formAbsolutePath(newArg))
                    x += counter + 1
                    continue
                elif os.path.isdir(file):
                    dir = file + "/"
                    for f in os.listdir(dir):
                        if string.split(f, ".")[-1] != "pds":
                            continue
                        fileName = dir + f
                        fileList.append(os_path_util.formAbsolutePath(fileName))
                    x += 2
                    continue
                elif os_path_util.isFile(file):
                    if file[-5:] == ".list":
                        tmpList = open(file).readlines()
                        for item in tmpList:
                            fileList.append(string.split(item)[0])
                    else:
                        fileList = [file]
                    x += 2
                    continue
                    # check if this file exists
                else:
                    print "ESBuilder: no such file", file
                    raise
                checkArg(fileList)
            if args[x] == "-view":
                view = args[x + 1]
                checkArg([view])
                x += 2
                continue
            if args[x] == "-idleMode":
                genMode = 0
                x += 1
                continue
            if args[x] == "-skim":
                skim = 1
                x += 1
                continue
            if args[x] == "-no-skim":
                noskim = 1
                x += 1
                continue
            if args[x] == "-masterDB":
                masterDB = args[x + 1]
                master = 1
                checkArg([masterDB])
                x += 2
                continue
                # if we reach here, that means we found unkown option
            if dictOpt.has_key(args[x]):
                x += dictOpt[args[x]]
            else:
                print "Option '%s' is not allowed" % args[x]
                raise
        except:
            sys.exit(1)

            ### AUTHENTICATION???
            # check that USER=pass2, otherwise exit
    authUsers = ["gluex", "sdobbs"]  ### CHECK
    # check if USER environment is set up, otherwise use LOGNAME
    env = os.environ
    if not env.has_key("USER"):
        os.environ["USER"] = env["LOGNAME"]
    if not authUsers.count(os.environ["USER"]) and dbName == "EventStore" and string.find(dbHost, "hallddb") != -1:
        print "ERROR: Injection to db='EventStore' should be done from official (gluex) account for %s DB\n" % dbName
        print "For your own injection please use another db name"
        sys.exit(1)
    # check underlying OS, so far we only allow to inject from SunOS
    # if os.environ["USER"]=="pass2" and uname[0]!="SunOS":
    #   print "ERROR: for pass2 account the EventStore injection should be done from SunOS\n"
    #   sys.exit(1)
    #######################################

    # form normalized abosulte paths
    oDir = os_path_util.formAbsolutePath(oDir)

    # check required parameters
    if not len(grade):
        print "ESBuilder requires to specify a grade, see -grade option"
        sys.exit(1)
    if string.find(grade, "unchecked") == -1 and view == "all":  ### CHECK
        print "ESBuilder only allow to inject 'unchecked' grades"
        print "  daq-unechecked, p2-unchecked, physics-unchecked"
        print "Either specify different view or inject as unchecked grade"
        print "Given grade='%s' view='%s'" % (grade, view)
        sys.exit(1)
    if not len(fileList):
        print "ESBuilder requires to specify input file(s) with -add option"
        sys.exit(1)

    # check permissions and access to output dir
    if not os.path.isdir(oDir):
        print "Output directory '%s' doesn't exists" % oDir
        print "ESBuilder requires to specify output dir to store key/location files, see -output option"
        sys.exit(1)
    if oDir and not os_path_util.checkPermission(oDir):
        print "You don't have permission to write to output area '%s'" % oDir
        sys.exit(1)

    # check permission to write to HSM
    if oHSMDir and not os.path.isdir(oHSMDir):
        print "HSM directory '%s' does not exists" % oHSMDir
        sys.exit(1)
        if not os_path_util.checkPermission(oHSMDir):
            print "You don't have permission to write to HSM location '%s'" % oHSMDir
            sys.exit(1)

    # check that all inputs are in place
    for file in fileList:
        if not os.path.isfile(file):
            print "File '%s' does not exists" % file
            sys.exit(1)
    if dupRead and not os.path.isfile(dupRead):
        print "File '%s' does not exists" % dupRead
        sys.exit(1)

    # connect to MySQL EventStoreDB
    outputLog, globalLog = es_init.ESOutputLog(logFile)
    db, dbType = es_init.ESDBConnector(dbHost, dbName, userName, userPass, "", dbPort, dbSocket)
    es_init.ESInput(userCommand, outputLog, dbType)

    # Be verbose
    dbinfo = "\t grade\t'%s'\n\t timeStamp\t'%s'\n\t view\t\t'%s'\n" % (grade, timeS, view)
    if newDB:
        if verbose:
            print "Creating new tables DB:"
            print dbinfo
    else:
        if verbose:
            print "Updating existing tables in DB:"
            print dbinfo
    if genMode == 0 and verbose:
        print "\n\t ===> Process running in Idle mode"

    # create instance of ESManager class
    mydb = ESManager.ESManager(db, dbType, outputLog)
    # set-up all parameters
    mydb.setOutputDir(oDir)
    mydb.setGenerateDB(newDB)
    mydb.setSVName(svName)
    mydb.setParents(listOfParents)
    mydb.setGrade(grade)
    mydb.setTimeStamp(timeS)
    mydb.setView(view)
    mydb.setMinRun(minRun)
    mydb.setMaxRun(maxRun)
    mydb.setVerboseLevel(verbose)
    mydb.setReadDuplicatesSource(dupRead)
    mydb.setSkimFlag(skim)
    mydb.setNoSkimFlag(noskim)
    mydb.setDBHost(dbHost)
    mydb.setDBName(dbName)
    mydb.setDBPort(dbPort)
    mydb.setDBSocket(dbSocket)

    # interpret the master option
    if masterDB:
        dbComponents = string.split(masterDB, "@")
        if len(dbComponents) == 2:
            masterDBName = dbComponents[0]
            newComponents = string.split(dbComponents[1], ":")
            masterDBHost = newComponents[0]
            port = socket = ""
            if len(newComponents) == 2:
                port = newComponents[1]
            elif len(newComponents) == 3:
                socket = newComponents[2]
            #           masterDBHost,port,socket=string.split(dbComponents[1],":")
            if port:
                masterDBPort = port
            if socket:
                masterDBSocket = socket
        else:
            masterDBHost = dbComponents[0]
    else:
        login, adminInfo, cMasterName, cMasterHost, cMasterPort, cMasterSocket = esdb_auth.readConfigFile()
        if cMasterHost:
            masterDBHost = cMasterHost
            masterDBName = cMasterName
            masterDBPort = cMasterPort
            masterDBSocket = cMasterSocket
    mydb.setMasterDB(masterDBName, masterDBHost, masterDBPort, masterDBSocket)

    # update DB using transaction
    if delete:
        status = mydb.deleteGrade(delGrade, delTime)
    else:  # for anything else
        try:
            status = mydb.updateDB(genMode, fileList, oHSMDir)
        except:
            print "ERROR: fail to process:"
            for item in fileList:
                print item
            print "--------------- See traceback ----------------"
            raise

    # close connection to db
    mydb.commit()
    mydb.close()

    returnStatus = es_init.ESOutput(status, userCommand, historyFile, outputLog, globalLog)
    return returnStatus