def runJob( self ):
if self.preexisting:
log = "> " + self.validation.name + " is already validated."
print log
return log
general = self.__config.getGeneral()
log = ""
for script in self.__scripts:
name = os.path.splitext( os.path.basename( script) )[0]
ValidationJob.jobCount += 1
if self.__commandLineOptions.dryRun:
print "%s would run: %s"%( name, os.path.basename( script) )
continue
log = "> Validating "+name
print "> Validating "+name
if self.validation.jobmode == "interactive":
log += getCommandOutput2( script )
ValidationJob.interactCount += 1
elif self.validation.jobmode.split(",")[0] == "lxBatch":
repMap = {
"commands": self.validation.jobmode.split(",")[1],
"logDir": general["logdir"],
"jobName": name,
"script": script,
"bsub": "/afs/cern.ch/cms/caf/scripts/cmsbsub"
}
bsubOut=getCommandOutput2("%(bsub)s %(commands)s "
"-J %(jobName)s "
"-o %(logDir)s/%(jobName)s.stdout "
"-e %(logDir)s/%(jobName)s.stderr "
"%(script)s"%repMap)
#Attention: here it is assumed that bsub returns a string
#containing a job id like <123456789>
ValidationJob.batchJobIds.append(bsubOut.split("<")[1].split(">")[0])
log+=bsubOut
ValidationJob.batchCount += 1
elif self.validation.jobmode.split( "," )[0] == "crab":
os.chdir( general["logdir"] )
crabName = "crab." + os.path.basename( script )[:-3]
theCrab = crabWrapper.CrabWrapper()
options = { "-create": "",
"-cfg": crabName + ".cfg",
"-submit": "" }
try:
theCrab.run( options )
except AllInOneError, e:
print "crab:", str(e).split("\n")[0]
exit(1)
ValidationJob.crabCount += 1
else:
raise AllInOneError, ("Unknown 'jobmode'!\n"
"Please change this parameter either in "
"the [general] or in the ["
+ self.__valType + ":" + self.__valName
+ "] section to one of the following "
"values:\n"
"\tinteractive\n\tlxBatch, -q <queue>\n"
"\tcrab, -q <queue>")
def runJob(self):
general = self.__config.getGeneral()
log = ""
for script in self.__scripts:
name = os.path.splitext(os.path.basename(script))[0]
if self.__commandLineOptions.dryRun:
print "%s would run: %s" % (name, os.path.basename(script))
continue
log = "> Validating " + name
print "> Validating " + name
if self.validation.jobmode == "interactive":
log += getCommandOutput2(script)
elif self.validation.jobmode.split(",")[0] == "lxBatch":
repMap = {
"commands": self.validation.jobmode.split(",")[1],
"logDir": general["logdir"],
"jobName": name,
"script": script,
"bsub": "/afs/cern.ch/cms/caf/scripts/cmsbsub"
}
log += getCommandOutput2(
"%(bsub)s %(commands)s -J %(jobName)s "
"-o %(logDir)s/%(jobName)s.stdout -e "
"%(logDir)s/%(jobName)s.stderr "
"%(script)s" % repMap)
elif self.validation.jobmode.split(",")[0] == "crab":
os.chdir(general["logdir"])
crabName = "crab." + os.path.basename(script)[:-3]
theCrab = crabWrapper.CrabWrapper()
options = {
"-create": "",
"-cfg": crabName + ".cfg",
"-submit": ""
}
try:
theCrab.run(options)
except AllInOneError, e:
print "crab:", str(e).split("\n")[0]
exit(1)
else:
raise AllInOneError, ("Unknown 'jobmode'!\n"
"Please change this parameter either in "
"the [general] or in the [" +
self.__valType + ":" + self.__valName +
"] section to one of the following "
"values:\n"
"\tinteractive\n\tlxBatch, -q <queue>\n"
"\tcrab, -q <queue>")
def main(argv=None):
if argv == None:
argv = sys.argv[1:]
optParser = optparse.OptionParser()
optParser.description = """All-in-one Alignment Validation.
This will run various validation procedures either on batch queues or interactively.
If no name is given (-N parameter) a name containing time and date is created automatically.
To merge the outcome of all validation procedures run TkAlMerge.sh in your validation's directory.
"""
optParser.add_option(
"-n",
"--dryRun",
dest="dryRun",
action="store_true",
default=False,
help=
"create all scripts and cfg File but do not start jobs (default=False)"
)
optParser.add_option(
"--getImages",
dest="getImages",
action="store_true",
default=True,
help="get all Images created during the process (default= True)")
defaultConfig = "TkAlConfig.ini"
optParser.add_option(
"-c",
"--config",
dest="config",
default=defaultConfig,
help=
"configuration to use (default TkAlConfig.ini) this can be a comma-seperated list of all .ini file you want to merge",
metavar="CONFIG")
optParser.add_option(
"-N",
"--Name",
dest="Name",
help="Name of this validation (default: alignmentValidation_DATE_TIME)",
metavar="NAME")
optParser.add_option(
"-r",
"--restrictTo",
dest="restrictTo",
help=
"restrict validations to given modes (comma seperated) (default: no restriction)",
metavar="RESTRICTTO")
optParser.add_option("-s",
"--status",
dest="crabStatus",
action="store_true",
default=False,
help="get the status of the crab jobs",
metavar="STATUS")
optParser.add_option("-d",
"--debug",
dest="debugMode",
action="store_true",
default=False,
help="run the tool to get full traceback of errors",
metavar="DEBUG")
optParser.add_option(
"-m",
"--autoMerge",
dest="autoMerge",
action="store_true",
default=False,
help=
"submit TkAlMerge.sh to run automatically when all jobs have finished (default=False)."
" Works only for batch jobs")
optParser.add_option(
"--mergeOfflineParallel",
dest="mergeOfflineParallel",
action="store_true",
default=False,
help=
"Enable parallel merging of offline data. Best used with -m option. Only works with lxBatch-jobmode",
metavar="MERGE_PARALLEL")
(options, args) = optParser.parse_args(argv)
if not options.restrictTo == None:
options.restrictTo = options.restrictTo.split(",")
options.config = [ os.path.abspath( iniFile ) for iniFile in \
options.config.split( "," ) ]
config = BetterConfigParser()
outputIniFileSet = set(config.read(options.config))
failedIniFiles = [
iniFile for iniFile in options.config
if iniFile not in outputIniFileSet
]
# Check for missing ini file
if options.config == [os.path.abspath(defaultConfig)]:
if ( not options.crabStatus ) and \
( not os.path.exists( defaultConfig ) ):
raise AllInOneError("Default 'ini' file '%s' not found!\n"
"You can specify another name with the "
"command line option '-c'/'--config'." %
(defaultConfig))
else:
for iniFile in failedIniFiles:
if not os.path.exists(iniFile):
raise AllInOneError("'%s' does not exist. Please check for "
"typos in the filename passed to the "
"'-c'/'--config' option!" % (iniFile))
else:
raise AllInOneError(("'%s' does exist, but parsing of the "
"content failed!") % iniFile)
# get the job name
if options.Name == None:
if not options.crabStatus:
options.Name = "alignmentValidation_%s" % (
datetime.datetime.now().strftime("%y%m%d_%H%M%S"))
else:
existingValDirs = fnmatch.filter(
os.walk('.').next()[1], "alignmentValidation_*")
if len(existingValDirs) > 0:
options.Name = existingValDirs[-1]
else:
print("Cannot guess last working directory!")
print("Please use the parameter '-N' or '--Name' to specify "
"the task for which you want a status report.")
return 1
# set output path
outPath = os.path.abspath(options.Name)
# Check status of submitted jobs and return
if options.crabStatus:
os.chdir(outPath)
crabLogDirs = fnmatch.filter(os.walk('.').next()[1], "crab.*")
if len(crabLogDirs) == 0:
print("Found no crab tasks for job name '%s'" % (options.Name))
return 1
theCrab = crabWrapper.CrabWrapper()
for crabLogDir in crabLogDirs:
print()
print("*" + "=" * 78 + "*")
print("| Status report and output retrieval for:" + " " *
(77 - len("Status report and output retrieval for:")) + "|")
taskName = crabLogDir.replace("crab.", "")
print("| " + taskName + " " * (77 - len(taskName)) + "|")
print("*" + "=" * 78 + "*")
print()
crabOptions = {"-getoutput": "", "-c": crabLogDir}
try:
theCrab.run(crabOptions)
except AllInOneError as e:
print("crab: No output retrieved for this task.")
crabOptions = {"-status": "", "-c": crabLogDir}
theCrab.run(crabOptions)
return
general = config.getGeneral()
config.set("internals", "workdir",
os.path.join(general["workdir"], options.Name))
config.set("internals", "scriptsdir", outPath)
config.set("general", "datadir",
os.path.join(general["datadir"], options.Name))
config.set("general", "logdir",
os.path.join(general["logdir"], options.Name))
config.set(
"general", "eosdir",
os.path.join("AlignmentValidation", general["eosdir"], options.Name))
if not os.path.exists(outPath):
os.makedirs(outPath)
elif not os.path.isdir(outPath):
raise AllInOneError(
"the file %s is in the way rename the Job or move it away" %
outPath)
# replace default templates by the ones specified in the "alternateTemplates" section
loadTemplates(config)
#save backup configuration file
backupConfigFile = open(os.path.join(outPath, "usedConfiguration.ini"),
"w")
config.write(backupConfigFile)
#copy proxy, if there is one
try:
proxyexists = int(getCommandOutput2("voms-proxy-info --timeleft")) > 10
except RuntimeError:
proxyexists = False
if proxyexists:
shutil.copyfile(
getCommandOutput2("voms-proxy-info --path").strip(),
os.path.join(outPath, ".user_proxy"))
validations = []
for validation in config.items("validation"):
alignmentList = [validation[1]]
validationsToAdd = [(validation[0],alignment) \
for alignment in alignmentList]
validations.extend(validationsToAdd)
jobs = [ ValidationJob( validation, config, options) \
for validation in validations ]
for job in jobs:
if job.needsproxy and not proxyexists:
raise AllInOneError(
"At least one job needs a grid proxy, please init one.")
map(lambda job: job.createJob(), jobs)
validations = [job.getValidation() for job in jobs]
if options.mergeOfflineParallel:
parallelMergeObjects = createMergeScript(
outPath, validations, options)['parallelMergeObjects']
else:
createMergeScript(outPath, validations, options)
print()
map(lambda job: job.runJob(), jobs)
if options.autoMerge and ValidationJob.jobCount == ValidationJob.batchCount and config.getGeneral(
)["jobmode"].split(",")[0] == "lxBatch":
print("> Automatically merging jobs when they have ended")
# if everything is done as batch job, also submit TkAlMerge.sh to be run
# after the jobs have finished
#if parallel merge scripts: manage dependencies
if options.mergeOfflineParallel and parallelMergeObjects != {}:
initID = parallelMergeObjects["init"].runJob(config).split(
"<")[1].split(">")[0]
parallelIDs = []
for parallelMergeScript in parallelMergeObjects["parallel"]:
parallelMergeScript.addDependency(initID)
for job in jobs:
if isinstance(
job.validation, OfflineValidation
) and "TkAlMerge" + job.validation.alignmentToValidate.name == parallelMergeScript.name:
parallelMergeScript.addDependency(job.JobId)
parallelIDs.append(
parallelMergeScript.runJob(config).split("<")[1].split(">")
[0])
parallelMergeObjects["continue"].addDependency(parallelIDs)
parallelMergeObjects["continue"].addDependency(
ValidationJob.batchJobIds)
parallelMergeObjects["continue"].runJob(config)
else:
repMap = {
"commands":
config.getGeneral()["jobmode"].split(",")[1],
"jobName":
"TkAlMerge",
"logDir":
config.getGeneral()["logdir"],
"script":
"TkAlMerge.sh",
"bsub":
"/afs/cern.ch/cms/caf/scripts/cmsbsub",
"conditions":
'"' + " && ".join([
"ended(" + jobId + ")"
for jobId in ValidationJob.batchJobIds
]) + '"'
}
for ext in ("stdout", "stderr", "stdout.gz", "stderr.gz"):
oldlog = "%(logDir)s/%(jobName)s." % repMap + ext
if os.path.exists(oldlog):
os.remove(oldlog)
#issue job
getCommandOutput2("%(bsub)s %(commands)s "
"-o %(logDir)s/%(jobName)s.stdout "
"-e %(logDir)s/%(jobName)s.stderr "
"-w %(conditions)s "
"%(logDir)s/%(script)s" % repMap)
def main(argv=None):
if argv == None:
argv = sys.argv[1:]
optParser = optparse.OptionParser()
optParser.description = """All-in-one Alignment Validation.
This will run various validation procedures either on batch queues or interactviely.
If no name is given (-N parameter) a name containing time and date is created automatically.
To merge the outcome of all validation procedures run TkAlMerge.sh in your validation's directory.
"""
optParser.add_option(
"-n",
"--dryRun",
dest="dryRun",
action="store_true",
default=False,
help=
"create all scripts and cfg File but do not start jobs (default=False)"
)
optParser.add_option(
"--getImages",
dest="getImages",
action="store_true",
default=False,
help="get all Images created during the process (default= False)")
defaultConfig = "TkAlConfig.ini"
optParser.add_option(
"-c",
"--config",
dest="config",
default=defaultConfig,
help=
"configuration to use (default TkAlConfig.ini) this can be a comma-seperated list of all .ini file you want to merge",
metavar="CONFIG")
optParser.add_option(
"-N",
"--Name",
dest="Name",
help="Name of this validation (default: alignmentValidation_DATE_TIME)",
metavar="NAME")
optParser.add_option(
"-r",
"--restrictTo",
dest="restrictTo",
help=
"restrict validations to given modes (comma seperated) (default: no restriction)",
metavar="RESTRICTTO")
optParser.add_option("-s",
"--status",
dest="crabStatus",
action="store_true",
default=False,
help="get the status of the crab jobs",
metavar="STATUS")
optParser.add_option("-d",
"--debug",
dest="debugMode",
action="store_true",
default=False,
help="run the tool to get full traceback of errors",
metavar="DEBUG")
optParser.add_option(
"-m",
"--autoMerge",
dest="autoMerge",
action="store_true",
default=False,
help=
"submit TkAlMerge.sh to run automatically when all jobs have finished (default=False)."
" Works only for batch jobs")
(options, args) = optParser.parse_args(argv)
if not options.restrictTo == None:
options.restrictTo = options.restrictTo.split(",")
options.config = [ os.path.abspath( iniFile ) for iniFile in \
options.config.split( "," ) ]
config = BetterConfigParser()
outputIniFileSet = set(config.read(options.config))
failedIniFiles = [
iniFile for iniFile in options.config
if iniFile not in outputIniFileSet
]
# Check for missing ini file
if options.config == [os.path.abspath(defaultConfig)]:
if ( not options.crabStatus ) and \
( not os.path.exists( defaultConfig ) ):
raise AllInOneError, ("Default 'ini' file '%s' not found!\n"
"You can specify another name with the "
"command line option '-c'/'--config'." %
(defaultConfig))
else:
for iniFile in failedIniFiles:
if not os.path.exists(iniFile):
raise AllInOneError, ("'%s' does not exist. Please check for "
"typos in the filename passed to the "
"'-c'/'--config' option!" % (iniFile))
else:
raise AllInOneError, ("'%s' does exist, but parsing of the "
"content failed!")
# get the job name
if options.Name == None:
if not options.crabStatus:
options.Name = "alignmentValidation_%s" % (
datetime.datetime.now().strftime("%y%m%d_%H%M%S"))
else:
existingValDirs = fnmatch.filter(
os.walk('.').next()[1], "alignmentValidation_*")
if len(existingValDirs) > 0:
options.Name = existingValDirs[-1]
else:
print "Cannot guess last working directory!"
print(
"Please use the parameter '-N' or '--Name' to specify "
"the task for which you want a status report.")
return 1
# set output path
outPath = os.path.abspath(options.Name)
# Check status of submitted jobs and return
if options.crabStatus:
os.chdir(outPath)
crabLogDirs = fnmatch.filter(os.walk('.').next()[1], "crab.*")
if len(crabLogDirs) == 0:
print "Found no crab tasks for job name '%s'" % (options.Name)
return 1
theCrab = crabWrapper.CrabWrapper()
for crabLogDir in crabLogDirs:
print
print "*" + "=" * 78 + "*"
print("| Status report and output retrieval for:" + " " *
(77 - len("Status report and output retrieval for:")) + "|")
taskName = crabLogDir.replace("crab.", "")
print "| " + taskName + " " * (77 - len(taskName)) + "|"
print "*" + "=" * 78 + "*"
print
crabOptions = {"-getoutput": "", "-c": crabLogDir}
try:
theCrab.run(crabOptions)
except AllInOneError, e:
print "crab: No output retrieved for this task."
crabOptions = {"-status": "", "-c": crabLogDir}
theCrab.run(crabOptions)
return
