def runJob( self ): if self.preexisting: log = "> " + self.validation.name + " is already validated." print log return log general = self.__config.getGeneral() log = "" for script in self.__scripts: name = os.path.splitext( os.path.basename( script) )[0] ValidationJob.jobCount += 1 if self.__commandLineOptions.dryRun: print "%s would run: %s"%( name, os.path.basename( script) ) continue log = "> Validating "+name print "> Validating "+name if self.validation.jobmode == "interactive": log += getCommandOutput2( script ) ValidationJob.interactCount += 1 elif self.validation.jobmode.split(",")[0] == "lxBatch": repMap = { "commands": self.validation.jobmode.split(",")[1], "logDir": general["logdir"], "jobName": name, "script": script, "bsub": "/afs/cern.ch/cms/caf/scripts/cmsbsub" } bsubOut=getCommandOutput2("%(bsub)s %(commands)s " "-J %(jobName)s " "-o %(logDir)s/%(jobName)s.stdout " "-e %(logDir)s/%(jobName)s.stderr " "%(script)s"%repMap) #Attention: here it is assumed that bsub returns a string #containing a job id like <123456789> ValidationJob.batchJobIds.append(bsubOut.split("<")[1].split(">")[0]) log+=bsubOut ValidationJob.batchCount += 1 elif self.validation.jobmode.split( "," )[0] == "crab": os.chdir( general["logdir"] ) crabName = "crab." + os.path.basename( script )[:-3] theCrab = crabWrapper.CrabWrapper() options = { "-create": "", "-cfg": crabName + ".cfg", "-submit": "" } try: theCrab.run( options ) except AllInOneError, e: print "crab:", str(e).split("\n")[0] exit(1) ValidationJob.crabCount += 1 else: raise AllInOneError, ("Unknown 'jobmode'!\n" "Please change this parameter either in " "the [general] or in the [" + self.__valType + ":" + self.__valName + "] section to one of the following " "values:\n" "\tinteractive\n\tlxBatch, -q <queue>\n" "\tcrab, -q <queue>")
def runJob(self): general = self.__config.getGeneral() log = "" for script in self.__scripts: name = os.path.splitext(os.path.basename(script))[0] if self.__commandLineOptions.dryRun: print "%s would run: %s" % (name, os.path.basename(script)) continue log = "> Validating " + name print "> Validating " + name if self.validation.jobmode == "interactive": log += getCommandOutput2(script) elif self.validation.jobmode.split(",")[0] == "lxBatch": repMap = { "commands": self.validation.jobmode.split(",")[1], "logDir": general["logdir"], "jobName": name, "script": script, "bsub": "/afs/cern.ch/cms/caf/scripts/cmsbsub" } log += getCommandOutput2( "%(bsub)s %(commands)s -J %(jobName)s " "-o %(logDir)s/%(jobName)s.stdout -e " "%(logDir)s/%(jobName)s.stderr " "%(script)s" % repMap) elif self.validation.jobmode.split(",")[0] == "crab": os.chdir(general["logdir"]) crabName = "crab." + os.path.basename(script)[:-3] theCrab = crabWrapper.CrabWrapper() options = { "-create": "", "-cfg": crabName + ".cfg", "-submit": "" } try: theCrab.run(options) except AllInOneError, e: print "crab:", str(e).split("\n")[0] exit(1) else: raise AllInOneError, ("Unknown 'jobmode'!\n" "Please change this parameter either in " "the [general] or in the [" + self.__valType + ":" + self.__valName + "] section to one of the following " "values:\n" "\tinteractive\n\tlxBatch, -q <queue>\n" "\tcrab, -q <queue>")
def main(argv=None): if argv == None: argv = sys.argv[1:] optParser = optparse.OptionParser() optParser.description = """All-in-one Alignment Validation. This will run various validation procedures either on batch queues or interactively. If no name is given (-N parameter) a name containing time and date is created automatically. To merge the outcome of all validation procedures run TkAlMerge.sh in your validation's directory. """ optParser.add_option( "-n", "--dryRun", dest="dryRun", action="store_true", default=False, help= "create all scripts and cfg File but do not start jobs (default=False)" ) optParser.add_option( "--getImages", dest="getImages", action="store_true", default=True, help="get all Images created during the process (default= True)") defaultConfig = "TkAlConfig.ini" optParser.add_option( "-c", "--config", dest="config", default=defaultConfig, help= "configuration to use (default TkAlConfig.ini) this can be a comma-seperated list of all .ini file you want to merge", metavar="CONFIG") optParser.add_option( "-N", "--Name", dest="Name", help="Name of this validation (default: alignmentValidation_DATE_TIME)", metavar="NAME") optParser.add_option( "-r", "--restrictTo", dest="restrictTo", help= "restrict validations to given modes (comma seperated) (default: no restriction)", metavar="RESTRICTTO") optParser.add_option("-s", "--status", dest="crabStatus", action="store_true", default=False, help="get the status of the crab jobs", metavar="STATUS") optParser.add_option("-d", "--debug", dest="debugMode", action="store_true", default=False, help="run the tool to get full traceback of errors", metavar="DEBUG") optParser.add_option( "-m", "--autoMerge", dest="autoMerge", action="store_true", default=False, help= "submit TkAlMerge.sh to run automatically when all jobs have finished (default=False)." " Works only for batch jobs") optParser.add_option( "--mergeOfflineParallel", dest="mergeOfflineParallel", action="store_true", default=False, help= "Enable parallel merging of offline data. Best used with -m option. Only works with lxBatch-jobmode", metavar="MERGE_PARALLEL") (options, args) = optParser.parse_args(argv) if not options.restrictTo == None: options.restrictTo = options.restrictTo.split(",") options.config = [ os.path.abspath( iniFile ) for iniFile in \ options.config.split( "," ) ] config = BetterConfigParser() outputIniFileSet = set(config.read(options.config)) failedIniFiles = [ iniFile for iniFile in options.config if iniFile not in outputIniFileSet ] # Check for missing ini file if options.config == [os.path.abspath(defaultConfig)]: if ( not options.crabStatus ) and \ ( not os.path.exists( defaultConfig ) ): raise AllInOneError("Default 'ini' file '%s' not found!\n" "You can specify another name with the " "command line option '-c'/'--config'." % (defaultConfig)) else: for iniFile in failedIniFiles: if not os.path.exists(iniFile): raise AllInOneError("'%s' does not exist. Please check for " "typos in the filename passed to the " "'-c'/'--config' option!" % (iniFile)) else: raise AllInOneError(("'%s' does exist, but parsing of the " "content failed!") % iniFile) # get the job name if options.Name == None: if not options.crabStatus: options.Name = "alignmentValidation_%s" % ( datetime.datetime.now().strftime("%y%m%d_%H%M%S")) else: existingValDirs = fnmatch.filter( os.walk('.').next()[1], "alignmentValidation_*") if len(existingValDirs) > 0: options.Name = existingValDirs[-1] else: print("Cannot guess last working directory!") print("Please use the parameter '-N' or '--Name' to specify " "the task for which you want a status report.") return 1 # set output path outPath = os.path.abspath(options.Name) # Check status of submitted jobs and return if options.crabStatus: os.chdir(outPath) crabLogDirs = fnmatch.filter(os.walk('.').next()[1], "crab.*") if len(crabLogDirs) == 0: print("Found no crab tasks for job name '%s'" % (options.Name)) return 1 theCrab = crabWrapper.CrabWrapper() for crabLogDir in crabLogDirs: print() print("*" + "=" * 78 + "*") print("| Status report and output retrieval for:" + " " * (77 - len("Status report and output retrieval for:")) + "|") taskName = crabLogDir.replace("crab.", "") print("| " + taskName + " " * (77 - len(taskName)) + "|") print("*" + "=" * 78 + "*") print() crabOptions = {"-getoutput": "", "-c": crabLogDir} try: theCrab.run(crabOptions) except AllInOneError as e: print("crab: No output retrieved for this task.") crabOptions = {"-status": "", "-c": crabLogDir} theCrab.run(crabOptions) return general = config.getGeneral() config.set("internals", "workdir", os.path.join(general["workdir"], options.Name)) config.set("internals", "scriptsdir", outPath) config.set("general", "datadir", os.path.join(general["datadir"], options.Name)) config.set("general", "logdir", os.path.join(general["logdir"], options.Name)) config.set( "general", "eosdir", os.path.join("AlignmentValidation", general["eosdir"], options.Name)) if not os.path.exists(outPath): os.makedirs(outPath) elif not os.path.isdir(outPath): raise AllInOneError( "the file %s is in the way rename the Job or move it away" % outPath) # replace default templates by the ones specified in the "alternateTemplates" section loadTemplates(config) #save backup configuration file backupConfigFile = open(os.path.join(outPath, "usedConfiguration.ini"), "w") config.write(backupConfigFile) #copy proxy, if there is one try: proxyexists = int(getCommandOutput2("voms-proxy-info --timeleft")) > 10 except RuntimeError: proxyexists = False if proxyexists: shutil.copyfile( getCommandOutput2("voms-proxy-info --path").strip(), os.path.join(outPath, ".user_proxy")) validations = [] for validation in config.items("validation"): alignmentList = [validation[1]] validationsToAdd = [(validation[0],alignment) \ for alignment in alignmentList] validations.extend(validationsToAdd) jobs = [ ValidationJob( validation, config, options) \ for validation in validations ] for job in jobs: if job.needsproxy and not proxyexists: raise AllInOneError( "At least one job needs a grid proxy, please init one.") map(lambda job: job.createJob(), jobs) validations = [job.getValidation() for job in jobs] if options.mergeOfflineParallel: parallelMergeObjects = createMergeScript( outPath, validations, options)['parallelMergeObjects'] else: createMergeScript(outPath, validations, options) print() map(lambda job: job.runJob(), jobs) if options.autoMerge and ValidationJob.jobCount == ValidationJob.batchCount and config.getGeneral( )["jobmode"].split(",")[0] == "lxBatch": print("> Automatically merging jobs when they have ended") # if everything is done as batch job, also submit TkAlMerge.sh to be run # after the jobs have finished #if parallel merge scripts: manage dependencies if options.mergeOfflineParallel and parallelMergeObjects != {}: initID = parallelMergeObjects["init"].runJob(config).split( "<")[1].split(">")[0] parallelIDs = [] for parallelMergeScript in parallelMergeObjects["parallel"]: parallelMergeScript.addDependency(initID) for job in jobs: if isinstance( job.validation, OfflineValidation ) and "TkAlMerge" + job.validation.alignmentToValidate.name == parallelMergeScript.name: parallelMergeScript.addDependency(job.JobId) parallelIDs.append( parallelMergeScript.runJob(config).split("<")[1].split(">") [0]) parallelMergeObjects["continue"].addDependency(parallelIDs) parallelMergeObjects["continue"].addDependency( ValidationJob.batchJobIds) parallelMergeObjects["continue"].runJob(config) else: repMap = { "commands": config.getGeneral()["jobmode"].split(",")[1], "jobName": "TkAlMerge", "logDir": config.getGeneral()["logdir"], "script": "TkAlMerge.sh", "bsub": "/afs/cern.ch/cms/caf/scripts/cmsbsub", "conditions": '"' + " && ".join([ "ended(" + jobId + ")" for jobId in ValidationJob.batchJobIds ]) + '"' } for ext in ("stdout", "stderr", "stdout.gz", "stderr.gz"): oldlog = "%(logDir)s/%(jobName)s." % repMap + ext if os.path.exists(oldlog): os.remove(oldlog) #issue job getCommandOutput2("%(bsub)s %(commands)s " "-o %(logDir)s/%(jobName)s.stdout " "-e %(logDir)s/%(jobName)s.stderr " "-w %(conditions)s " "%(logDir)s/%(script)s" % repMap)
def main(argv=None): if argv == None: argv = sys.argv[1:] optParser = optparse.OptionParser() optParser.description = """All-in-one Alignment Validation. This will run various validation procedures either on batch queues or interactviely. If no name is given (-N parameter) a name containing time and date is created automatically. To merge the outcome of all validation procedures run TkAlMerge.sh in your validation's directory. """ optParser.add_option( "-n", "--dryRun", dest="dryRun", action="store_true", default=False, help= "create all scripts and cfg File but do not start jobs (default=False)" ) optParser.add_option( "--getImages", dest="getImages", action="store_true", default=False, help="get all Images created during the process (default= False)") defaultConfig = "TkAlConfig.ini" optParser.add_option( "-c", "--config", dest="config", default=defaultConfig, help= "configuration to use (default TkAlConfig.ini) this can be a comma-seperated list of all .ini file you want to merge", metavar="CONFIG") optParser.add_option( "-N", "--Name", dest="Name", help="Name of this validation (default: alignmentValidation_DATE_TIME)", metavar="NAME") optParser.add_option( "-r", "--restrictTo", dest="restrictTo", help= "restrict validations to given modes (comma seperated) (default: no restriction)", metavar="RESTRICTTO") optParser.add_option("-s", "--status", dest="crabStatus", action="store_true", default=False, help="get the status of the crab jobs", metavar="STATUS") optParser.add_option("-d", "--debug", dest="debugMode", action="store_true", default=False, help="run the tool to get full traceback of errors", metavar="DEBUG") optParser.add_option( "-m", "--autoMerge", dest="autoMerge", action="store_true", default=False, help= "submit TkAlMerge.sh to run automatically when all jobs have finished (default=False)." " Works only for batch jobs") (options, args) = optParser.parse_args(argv) if not options.restrictTo == None: options.restrictTo = options.restrictTo.split(",") options.config = [ os.path.abspath( iniFile ) for iniFile in \ options.config.split( "," ) ] config = BetterConfigParser() outputIniFileSet = set(config.read(options.config)) failedIniFiles = [ iniFile for iniFile in options.config if iniFile not in outputIniFileSet ] # Check for missing ini file if options.config == [os.path.abspath(defaultConfig)]: if ( not options.crabStatus ) and \ ( not os.path.exists( defaultConfig ) ): raise AllInOneError, ("Default 'ini' file '%s' not found!\n" "You can specify another name with the " "command line option '-c'/'--config'." % (defaultConfig)) else: for iniFile in failedIniFiles: if not os.path.exists(iniFile): raise AllInOneError, ("'%s' does not exist. Please check for " "typos in the filename passed to the " "'-c'/'--config' option!" % (iniFile)) else: raise AllInOneError, ("'%s' does exist, but parsing of the " "content failed!") # get the job name if options.Name == None: if not options.crabStatus: options.Name = "alignmentValidation_%s" % ( datetime.datetime.now().strftime("%y%m%d_%H%M%S")) else: existingValDirs = fnmatch.filter( os.walk('.').next()[1], "alignmentValidation_*") if len(existingValDirs) > 0: options.Name = existingValDirs[-1] else: print "Cannot guess last working directory!" print( "Please use the parameter '-N' or '--Name' to specify " "the task for which you want a status report.") return 1 # set output path outPath = os.path.abspath(options.Name) # Check status of submitted jobs and return if options.crabStatus: os.chdir(outPath) crabLogDirs = fnmatch.filter(os.walk('.').next()[1], "crab.*") if len(crabLogDirs) == 0: print "Found no crab tasks for job name '%s'" % (options.Name) return 1 theCrab = crabWrapper.CrabWrapper() for crabLogDir in crabLogDirs: print print "*" + "=" * 78 + "*" print("| Status report and output retrieval for:" + " " * (77 - len("Status report and output retrieval for:")) + "|") taskName = crabLogDir.replace("crab.", "") print "| " + taskName + " " * (77 - len(taskName)) + "|" print "*" + "=" * 78 + "*" print crabOptions = {"-getoutput": "", "-c": crabLogDir} try: theCrab.run(crabOptions) except AllInOneError, e: print "crab: No output retrieved for this task." crabOptions = {"-status": "", "-c": crabLogDir} theCrab.run(crabOptions) return