def runJob( self ):
if self.preexisting:
log = "> " + self.validation.name + " is already validated."
print log
return log
general = self.__config.getGeneral()
log = ""
for script in self.__scripts:
name = os.path.splitext( os.path.basename( script) )[0]
ValidationJob.jobCount += 1
if self.__commandLineOptions.dryRun:
print "%s would run: %s"%( name, os.path.basename( script) )
continue
log = "> Validating "+name
print "> Validating "+name
if self.validation.jobmode == "interactive":
log += getCommandOutput2( script )
ValidationJob.interactCount += 1
elif self.validation.jobmode.split(",")[0] == "lxBatch":
repMap = {
"commands": self.validation.jobmode.split(",")[1],
"logDir": general["logdir"],
"jobName": name,
"script": script,
"bsub": "/afs/cern.ch/cms/caf/scripts/cmsbsub"
}
bsubOut=getCommandOutput2("%(bsub)s %(commands)s "
"-J %(jobName)s "
"-o %(logDir)s/%(jobName)s.stdout "
"-e %(logDir)s/%(jobName)s.stderr "
"%(script)s"%repMap)
#Attention: here it is assumed that bsub returns a string
#containing a job id like <123456789>
ValidationJob.batchJobIds.append(bsubOut.split("<")[1].split(">")[0])
log+=bsubOut
ValidationJob.batchCount += 1
elif self.validation.jobmode.split( "," )[0] == "crab":
os.chdir( general["logdir"] )
crabName = "crab." + os.path.basename( script )[:-3]
theCrab = crabWrapper.CrabWrapper()
options = { "-create": "",
"-cfg": crabName + ".cfg",
"-submit": "" }
try:
theCrab.run( options )
except AllInOneError, e:
print "crab:", str(e).split("\n")[0]
exit(1)
ValidationJob.crabCount += 1
else:
raise AllInOneError, ("Unknown 'jobmode'!\n"
"Please change this parameter either in "
"the [general] or in the ["
+ self.__valType + ":" + self.__valName
+ "] section to one of the following "
"values:\n"
"\tinteractive\n\tlxBatch, -q <queue>\n"
"\tcrab, -q <queue>")
def runJob( self ):
general = self.__config.getGeneral()
log = ""
for script in self.__scripts:
name = os.path.splitext( os.path.basename( script) )[0]
if self.__commandLineOptions.dryRun:
print "%s would run: %s"%( name, os.path.basename( script) )
continue
log = "> Validating "+name
print "> Validating "+name
if self.validation.jobmode == "interactive":
log += getCommandOutput2( script )
elif self.validation.jobmode.split(",")[0] == "lxBatch":
repMap = {
"commands": self.validation.jobmode.split(",")[1],
"logDir": general["logdir"],
"jobName": name,
"script": script,
"bsub": "/afs/cern.ch/cms/caf/scripts/cmsbsub"
}
log+=getCommandOutput2("%(bsub)s %(commands)s -J %(jobName)s "
"-o %(logDir)s/%(jobName)s.stdout -e "
"%(logDir)s/%(jobName)s.stderr "
"%(script)s"%repMap)
elif self.validation.jobmode.split( "," )[0] == "crab":
os.chdir( general["logdir"] )
crabName = "crab." + os.path.basename( script )[:-3]
theCrab = crabWrapper.CrabWrapper()
options = { "-create": "",
"-cfg": crabName + ".cfg",
"-submit": "" }
try:
theCrab.run( options )
except AllInOneError, e:
print "crab:", str(e).split("\n")[0]
exit(1)
else:
raise AllInOneError, ("Unknown 'jobmode'!\n"
"Please change this parameter either in "
"the [general] or in the ["
+ self.__valType + ":" + self.__valName
+ "] section to one of the following "
"values:\n"
"\tinteractive\n\tlxBatch, -q <queue>\n"
"\tcrab, -q <queue>")
def runJob(self, config):
repMap = {
"commands": config.getGeneral()["jobmode"].split(",")[1],
"jobName": self.name,
"logDir": config.getGeneral()["logdir"],
"script": self.path,
"bsub": "/afs/cern.ch/cms/caf/scripts/cmsbsub",
"conditions": '"' + " && ".join(["ended(" + jobId + ")" for jobId in self.dependencies]) + '"'
}
return getCommandOutput2("%(bsub)s %(commands)s "
"-J %(jobName)s "
"-o %(logDir)s/%(jobName)s.stdout "
"-e %(logDir)s/%(jobName)s.stderr "
"-w %(conditions)s "
"%(script)s"%repMap)
print "> Automatically merging jobs when they have ended"
repMap = {
"commands": config.getGeneral()["jobmode"].split(",")[1],
"jobName": "TkAlMerge",
"logDir": config.getGeneral()["logdir"],
"script": "TkAlMerge.sh",
"bsub": "/afs/cern.ch/cms/caf/scripts/cmsbsub",
"conditions": '"' + " && ".join(["ended(" + jobId + ")" for jobId in ValidationJob.batchJobIds]) + '"'
}
for ext in ("stdout", "stderr", "stdout.gz", "stderr.gz"):
oldlog = "%(logDir)s/%(jobName)s."%repMap + ext
if os.path.exists(oldlog):
os.remove(oldlog)
getCommandOutput2("%(bsub)s %(commands)s "
"-o %(logDir)s/%(jobName)s.stdout "
"-e %(logDir)s/%(jobName)s.stderr "
"-w %(conditions)s "
"%(logDir)s/%(script)s"%repMap)
if __name__ == "__main__":
# main(["-n","-N","test","-c","defaultCRAFTValidation.ini,latestObjects.ini","--getImages"])
if "-d" in sys.argv[1:] or "--debug" in sys.argv[1:]:
main()
else:
try:
main()
except AllInOneError, e:
print "\nAll-In-One Tool:", str(e)
exit(1)
def main(argv = None):
if argv == None:
argv = sys.argv[1:]
optParser = optparse.OptionParser()
optParser.description = """All-in-one Alignment Validation.
This will run various validation procedures either on batch queues or interactively.
If no name is given (-N parameter) a name containing time and date is created automatically.
To merge the outcome of all validation procedures run TkAlMerge.sh in your validation's directory.
"""
optParser.add_option("-n", "--dryRun", dest="dryRun", action="store_true", default=False,
help="create all scripts and cfg File but do not start jobs (default=False)")
optParser.add_option( "--getImages", dest="getImages", action="store_true", default=True,
help="get all Images created during the process (default= True)")
defaultConfig = "TkAlConfig.ini"
optParser.add_option("-c", "--config", dest="config", default = defaultConfig,
help="configuration to use (default TkAlConfig.ini) this can be a comma-seperated list of all .ini file you want to merge", metavar="CONFIG")
optParser.add_option("-N", "--Name", dest="Name",
help="Name of this validation (default: alignmentValidation_DATE_TIME)", metavar="NAME")
optParser.add_option("-r", "--restrictTo", dest="restrictTo",
help="restrict validations to given modes (comma seperated) (default: no restriction)", metavar="RESTRICTTO")
optParser.add_option("-s", "--status", dest="crabStatus", action="store_true", default = False,
help="get the status of the crab jobs", metavar="STATUS")
optParser.add_option("-d", "--debug", dest="debugMode", action="store_true",
default = False,
help="run the tool to get full traceback of errors",
metavar="DEBUG")
optParser.add_option("-m", "--autoMerge", dest="autoMerge", action="store_true", default = False,
help="submit TkAlMerge.sh to run automatically when all jobs have finished (default=False)."
" Works only for batch jobs")
(options, args) = optParser.parse_args(argv)
if not options.restrictTo == None:
options.restrictTo = options.restrictTo.split(",")
options.config = [ os.path.abspath( iniFile ) for iniFile in \
options.config.split( "," ) ]
config = BetterConfigParser()
outputIniFileSet = set( config.read( options.config ) )
failedIniFiles = [ iniFile for iniFile in options.config if iniFile not in outputIniFileSet ]
# Check for missing ini file
if options.config == [ os.path.abspath( defaultConfig ) ]:
if ( not options.crabStatus ) and \
( not os.path.exists( defaultConfig ) ):
raise AllInOneError( "Default 'ini' file '%s' not found!\n"
"You can specify another name with the "
"command line option '-c'/'--config'."
%( defaultConfig ))
else:
for iniFile in failedIniFiles:
if not os.path.exists( iniFile ):
raise AllInOneError( "'%s' does not exist. Please check for "
"typos in the filename passed to the "
"'-c'/'--config' option!"
%( iniFile ))
else:
raise AllInOneError(( "'%s' does exist, but parsing of the "
"content failed!" ) % iniFile)
# get the job name
if options.Name == None:
if not options.crabStatus:
options.Name = "alignmentValidation_%s"%(datetime.datetime.now().strftime("%y%m%d_%H%M%S"))
else:
existingValDirs = fnmatch.filter( os.walk( '.' ).next()[1],
"alignmentValidation_*" )
if len( existingValDirs ) > 0:
options.Name = existingValDirs[-1]
else:
print "Cannot guess last working directory!"
print ( "Please use the parameter '-N' or '--Name' to specify "
"the task for which you want a status report." )
return 1
# set output path
outPath = os.path.abspath( options.Name )
# Check status of submitted jobs and return
if options.crabStatus:
os.chdir( outPath )
crabLogDirs = fnmatch.filter( os.walk('.').next()[1], "crab.*" )
if len( crabLogDirs ) == 0:
print "Found no crab tasks for job name '%s'"%( options.Name )
return 1
theCrab = crabWrapper.CrabWrapper()
for crabLogDir in crabLogDirs:
print
print "*" + "=" * 78 + "*"
print ( "| Status report and output retrieval for:"
+ " " * (77 - len( "Status report and output retrieval for:" ) )
+ "|" )
taskName = crabLogDir.replace( "crab.", "" )
print "| " + taskName + " " * (77 - len( taskName ) ) + "|"
print "*" + "=" * 78 + "*"
print
crabOptions = { "-getoutput":"",
"-c": crabLogDir }
try:
theCrab.run( crabOptions )
except AllInOneError as e:
print "crab: No output retrieved for this task."
crabOptions = { "-status": "",
"-c": crabLogDir }
theCrab.run( crabOptions )
return
general = config.getGeneral()
config.set("internals","workdir",os.path.join(general["workdir"],options.Name) )
config.set("internals","scriptsdir",outPath)
config.set("general","datadir",os.path.join(general["datadir"],options.Name) )
config.set("general","logdir",os.path.join(general["logdir"],options.Name) )
config.set("general","eosdir",os.path.join("AlignmentValidation", general["eosdir"], options.Name) )
if not os.path.exists( outPath ):
os.makedirs( outPath )
elif not os.path.isdir( outPath ):
raise AllInOneError("the file %s is in the way rename the Job or move it away"%outPath)
# replace default templates by the ones specified in the "alternateTemplates" section
loadTemplates( config )
#save backup configuration file
backupConfigFile = open( os.path.join( outPath, "usedConfiguration.ini" ) , "w" )
config.write( backupConfigFile )
#copy proxy, if there is one
try:
proxyexists = int(getCommandOutput2("voms-proxy-info --timeleft")) > 10
except RuntimeError:
proxyexists = False
if proxyexists:
shutil.copyfile(getCommandOutput2("voms-proxy-info --path").strip(), os.path.join(outPath, ".user_proxy"))
validations = []
for validation in config.items("validation"):
alignmentList = [validation[1]]
validationsToAdd = [(validation[0],alignment) \
for alignment in alignmentList]
validations.extend(validationsToAdd)
jobs = [ ValidationJob( validation, config, options) \
for validation in validations ]
for job in jobs:
if job.needsproxy and not proxyexists:
raise AllInOneError("At least one job needs a grid proxy, please init one.")
map( lambda job: job.createJob(), jobs )
validations = [ job.getValidation() for job in jobs ]
createMergeScript(outPath, validations)
print
map( lambda job: job.runJob(), jobs )
if options.autoMerge:
# if everything is done as batch job, also submit TkAlMerge.sh to be run
# after the jobs have finished
if ValidationJob.jobCount == ValidationJob.batchCount and config.getGeneral()["jobmode"].split(",")[0] == "lxBatch":
print "> Automatically merging jobs when they have ended"
repMap = {
"commands": config.getGeneral()["jobmode"].split(",")[1],
"jobName": "TkAlMerge",
"logDir": config.getGeneral()["logdir"],
"script": "TkAlMerge.sh",
"bsub": "/afs/cern.ch/cms/caf/scripts/cmsbsub",
"conditions": '"' + " && ".join(["ended(" + jobId + ")" for jobId in ValidationJob.batchJobIds]) + '"'
}
for ext in ("stdout", "stderr", "stdout.gz", "stderr.gz"):
oldlog = "%(logDir)s/%(jobName)s."%repMap + ext
if os.path.exists(oldlog):
os.remove(oldlog)
getCommandOutput2("%(bsub)s %(commands)s "
"-o %(logDir)s/%(jobName)s.stdout "
"-e %(logDir)s/%(jobName)s.stderr "
"-w %(conditions)s "
"%(logDir)s/%(script)s"%repMap)
def runJob(self):
if self.__preexisting:
if self.validation.jobid:
self.batchJobIds.append(self.validation.jobid)
log = "> " + self.validation.name + " is already validated."
print log
return log
general = self.__config.getGeneral()
log = ""
for script in self.__scripts:
name = os.path.splitext(os.path.basename(script))[0]
ValidationJob.jobCount += 1
if self.__commandLineOptions.dryRun:
print "%s would run: %s" % (name, os.path.basename(script))
continue
log = "> Validating " + name
print "> Validating " + name
if self.validation.jobmode == "interactive":
log += getCommandOutput2(script)
ValidationJob.interactCount += 1
elif self.validation.jobmode.split(",")[0] == "lxBatch":
repMap = {
"commands": self.validation.jobmode.split(",")[1],
"logDir": general["logdir"],
"jobName": name,
"script": script,
"bsub": "/afs/cern.ch/cms/caf/scripts/cmsbsub"
}
for ext in ("stdout", "stderr", "stdout.gz", "stderr.gz"):
oldlog = "%(logDir)s/%(jobName)s." % repMap + ext
if os.path.exists(oldlog):
os.remove(oldlog)
bsubOut = getCommandOutput2("%(bsub)s %(commands)s "
"-J %(jobName)s "
"-o %(logDir)s/%(jobName)s.stdout "
"-e %(logDir)s/%(jobName)s.stderr "
"%(script)s" % repMap)
#Attention: here it is assumed that bsub returns a string
#containing a job id like <123456789>
ValidationJob.batchJobIds.append(
bsubOut.split("<")[1].split(">")[0])
log += bsubOut
ValidationJob.batchCount += 1
elif self.validation.jobmode.split(",")[0] == "crab":
os.chdir(general["logdir"])
crabName = "crab." + os.path.basename(script)[:-3]
theCrab = crabWrapper.CrabWrapper()
options = {
"-create": "",
"-cfg": crabName + ".cfg",
"-submit": ""
}
try:
theCrab.run(options)
except AllInOneError as e:
print "crab:", str(e).split("\n")[0]
exit(1)
ValidationJob.crabCount += 1
else:
raise AllInOneError("Unknown 'jobmode'!\n"
"Please change this parameter either in "
"the [general] or in the [" +
self.__valType + ":" + self.__valName +
"] section to one of the following "
"values:\n"
"\tinteractive\n\tlxBatch, -q <queue>\n"
"\tcrab, -q <queue>")
return log
if options.autoMerge:
# if everything is done as batch job, also submit TkAlMerge.sh to be run
# after the jobs have finished
if ValidationJob.jobCount == ValidationJob.batchCount and config.getGeneral()["jobmode"].split(",")[0] == "lxBatch":
print "> Automatically merging jobs when they have ended"
repMap = {
"commands": config.getGeneral()["jobmode"].split(",")[1],
"jobName": "TkAlMerge",
"logDir": config.getGeneral()["logdir"],
"script": "TkAlMerge.sh",
"bsub": "/afs/cern.ch/cms/caf/scripts/cmsbsub",
"conditions": '"' + " && ".join(["ended(" + jobId + ")" for jobId in ValidationJob.batchJobIds]) + '"'
}
getCommandOutput2("%(bsub)s %(commands)s "
"-o %(logDir)s/%(jobName)s.stdout "
"-e %(logDir)s/%(jobName)s.stderr "
"-w %(conditions)s "
"%(logDir)s/%(script)s"%repMap)
if __name__ == "__main__":
# main(["-n","-N","test","-c","defaultCRAFTValidation.ini,latestObjects.ini","--getImages"])
if "-d" in sys.argv[1:] or "--debug" in sys.argv[1:]:
main()
else:
try:
main()
except AllInOneError, e:
print "\nAll-In-One Tool:", str(e)
exit(1)
def main(argv=None):
if argv == None:
argv = sys.argv[1:]
optParser = optparse.OptionParser()
optParser.description = """All-in-one Alignment Validation.
This will run various validation procedures either on batch queues or interactively.
If no name is given (-N parameter) a name containing time and date is created automatically.
To merge the outcome of all validation procedures run TkAlMerge.sh in your validation's directory.
"""
optParser.add_option(
"-n",
"--dryRun",
dest="dryRun",
action="store_true",
default=False,
help=
"create all scripts and cfg File but do not start jobs (default=False)"
)
optParser.add_option(
"--getImages",
dest="getImages",
action="store_true",
default=True,
help="get all Images created during the process (default= True)")
defaultConfig = "TkAlConfig.ini"
optParser.add_option(
"-c",
"--config",
dest="config",
default=defaultConfig,
help=
"configuration to use (default TkAlConfig.ini) this can be a comma-seperated list of all .ini file you want to merge",
metavar="CONFIG")
optParser.add_option(
"-N",
"--Name",
dest="Name",
help="Name of this validation (default: alignmentValidation_DATE_TIME)",
metavar="NAME")
optParser.add_option(
"-r",
"--restrictTo",
dest="restrictTo",
help=
"restrict validations to given modes (comma seperated) (default: no restriction)",
metavar="RESTRICTTO")
optParser.add_option("-s",
"--status",
dest="crabStatus",
action="store_true",
default=False,
help="get the status of the crab jobs",
metavar="STATUS")
optParser.add_option("-d",
"--debug",
dest="debugMode",
action="store_true",
default=False,
help="run the tool to get full traceback of errors",
metavar="DEBUG")
optParser.add_option(
"-m",
"--autoMerge",
dest="autoMerge",
action="store_true",
default=False,
help=
"submit TkAlMerge.sh to run automatically when all jobs have finished (default=False)."
" Works only for batch jobs")
(options, args) = optParser.parse_args(argv)
if not options.restrictTo == None:
options.restrictTo = options.restrictTo.split(",")
options.config = [ os.path.abspath( iniFile ) for iniFile in \
options.config.split( "," ) ]
config = BetterConfigParser()
outputIniFileSet = set(config.read(options.config))
failedIniFiles = [
iniFile for iniFile in options.config
if iniFile not in outputIniFileSet
]
# Check for missing ini file
if options.config == [os.path.abspath(defaultConfig)]:
if ( not options.crabStatus ) and \
( not os.path.exists( defaultConfig ) ):
raise AllInOneError("Default 'ini' file '%s' not found!\n"
"You can specify another name with the "
"command line option '-c'/'--config'." %
(defaultConfig))
else:
for iniFile in failedIniFiles:
if not os.path.exists(iniFile):
raise AllInOneError("'%s' does not exist. Please check for "
"typos in the filename passed to the "
"'-c'/'--config' option!" % (iniFile))
else:
raise AllInOneError(("'%s' does exist, but parsing of the "
"content failed!") % iniFile)
# get the job name
if options.Name == None:
if not options.crabStatus:
options.Name = "alignmentValidation_%s" % (
datetime.datetime.now().strftime("%y%m%d_%H%M%S"))
else:
existingValDirs = fnmatch.filter(
os.walk('.').next()[1], "alignmentValidation_*")
if len(existingValDirs) > 0:
options.Name = existingValDirs[-1]
else:
print "Cannot guess last working directory!"
print(
"Please use the parameter '-N' or '--Name' to specify "
"the task for which you want a status report.")
return 1
# set output path
outPath = os.path.abspath(options.Name)
# Check status of submitted jobs and return
if options.crabStatus:
os.chdir(outPath)
crabLogDirs = fnmatch.filter(os.walk('.').next()[1], "crab.*")
if len(crabLogDirs) == 0:
print "Found no crab tasks for job name '%s'" % (options.Name)
return 1
theCrab = crabWrapper.CrabWrapper()
for crabLogDir in crabLogDirs:
print
print "*" + "=" * 78 + "*"
print("| Status report and output retrieval for:" + " " *
(77 - len("Status report and output retrieval for:")) + "|")
taskName = crabLogDir.replace("crab.", "")
print "| " + taskName + " " * (77 - len(taskName)) + "|"
print "*" + "=" * 78 + "*"
print
crabOptions = {"-getoutput": "", "-c": crabLogDir}
try:
theCrab.run(crabOptions)
except AllInOneError as e:
print "crab: No output retrieved for this task."
crabOptions = {"-status": "", "-c": crabLogDir}
theCrab.run(crabOptions)
return
general = config.getGeneral()
config.set("internals", "workdir",
os.path.join(general["workdir"], options.Name))
config.set("internals", "scriptsdir", outPath)
config.set("general", "datadir",
os.path.join(general["datadir"], options.Name))
config.set("general", "logdir",
os.path.join(general["logdir"], options.Name))
config.set(
"general", "eosdir",
os.path.join("AlignmentValidation", general["eosdir"], options.Name))
if not os.path.exists(outPath):
os.makedirs(outPath)
elif not os.path.isdir(outPath):
raise AllInOneError(
"the file %s is in the way rename the Job or move it away" %
outPath)
# replace default templates by the ones specified in the "alternateTemplates" section
loadTemplates(config)
#save backup configuration file
backupConfigFile = open(os.path.join(outPath, "usedConfiguration.ini"),
"w")
config.write(backupConfigFile)
#copy proxy, if there is one
try:
proxyexists = int(getCommandOutput2("voms-proxy-info --timeleft")) > 10
except RuntimeError:
proxyexists = False
if proxyexists:
shutil.copyfile(
getCommandOutput2("voms-proxy-info --path").strip(),
os.path.join(outPath, ".user_proxy"))
validations = []
for validation in config.items("validation"):
alignmentList = [validation[1]]
validationsToAdd = [(validation[0],alignment) \
for alignment in alignmentList]
validations.extend(validationsToAdd)
jobs = [ ValidationJob( validation, config, options) \
for validation in validations ]
for job in jobs:
if job.needsproxy and not proxyexists:
raise AllInOneError(
"At least one job needs a grid proxy, please init one.")
map(lambda job: job.createJob(), jobs)
validations = [job.getValidation() for job in jobs]
createMergeScript(outPath, validations)
print
map(lambda job: job.runJob(), jobs)
if options.autoMerge:
# if everything is done as batch job, also submit TkAlMerge.sh to be run
# after the jobs have finished
if ValidationJob.jobCount == ValidationJob.batchCount and config.getGeneral(
)["jobmode"].split(",")[0] == "lxBatch":
print "> Automatically merging jobs when they have ended"
repMap = {
"commands":
config.getGeneral()["jobmode"].split(",")[1],
"jobName":
"TkAlMerge",
"logDir":
config.getGeneral()["logdir"],
"script":
"TkAlMerge.sh",
"bsub":
"/afs/cern.ch/cms/caf/scripts/cmsbsub",
"conditions":
'"' + " && ".join([
"ended(" + jobId + ")"
for jobId in ValidationJob.batchJobIds
]) + '"'
}
for ext in ("stdout", "stderr", "stdout.gz", "stderr.gz"):
oldlog = "%(logDir)s/%(jobName)s." % repMap + ext
if os.path.exists(oldlog):
os.remove(oldlog)
getCommandOutput2("%(bsub)s %(commands)s "
"-o %(logDir)s/%(jobName)s.stdout "
"-e %(logDir)s/%(jobName)s.stderr "
"-w %(conditions)s "
"%(logDir)s/%(script)s" % repMap)
def main(argv=None):
if argv == None:
argv = sys.argv[1:]
optParser = optparse.OptionParser()
optParser.description = """All-in-one Alignment Validation.
This will run various validation procedures either on batch queues or interactively.
If no name is given (-N parameter) a name containing time and date is created automatically.
To merge the outcome of all validation procedures run TkAlMerge.sh in your validation's directory.
"""
optParser.add_option(
"-n",
"--dryRun",
dest="dryRun",
action="store_true",
default=False,
help=
"create all scripts and cfg File but do not start jobs (default=False)"
)
optParser.add_option(
"--getImages",
dest="getImages",
action="store_true",
default=True,
help="get all Images created during the process (default= True)")
defaultConfig = "TkAlConfig.ini"
optParser.add_option(
"-c",
"--config",
dest="config",
default=defaultConfig,
help=
"configuration to use (default TkAlConfig.ini) this can be a comma-seperated list of all .ini file you want to merge",
metavar="CONFIG")
optParser.add_option(
"-N",
"--Name",
dest="Name",
help="Name of this validation (default: alignmentValidation_DATE_TIME)",
metavar="NAME")
optParser.add_option(
"-r",
"--restrictTo",
dest="restrictTo",
help=
"restrict validations to given modes (comma seperated) (default: no restriction)",
metavar="RESTRICTTO")
optParser.add_option("-d",
"--debug",
dest="debugMode",
action="store_true",
default=False,
help="run the tool to get full traceback of errors",
metavar="DEBUG")
(options, args) = optParser.parse_args(argv)
if not options.restrictTo == None:
options.restrictTo = options.restrictTo.split(",")
options.config = [ os.path.abspath( iniFile ) for iniFile in \
options.config.split( "," )]
config = BetterConfigParser()
outputIniFileSet = set(config.read(options.config))
failedIniFiles = [
iniFile for iniFile in options.config
if iniFile not in outputIniFileSet
]
# Check for missing ini file
if options.config == [os.path.abspath(defaultConfig)]:
if (not os.path.exists(defaultConfig)):
raise AllInOneError("Default 'ini' file '%s' not found!\n"
"You can specify another name with the "
"command line option '-c'/'--config'." %
(defaultConfig))
else:
for iniFile in failedIniFiles:
if not os.path.exists(iniFile):
raise AllInOneError("'%s' does not exist. Please check for "
"typos in the filename passed to the "
"'-c'/'--config' option!" % (iniFile))
else:
raise AllInOneError(("'%s' does exist, but parsing of the "
"content failed!") % iniFile)
# get the job name
if options.Name == None:
existingValDirs = fnmatch.filter(
os.walk('.').next()[1], "alignmentValidation_*")
if len(existingValDirs) > 0:
options.Name = existingValDirs[-1]
else:
print("Cannot guess last working directory!")
print("Please use the parameter '-N' or '--Name' to specify "
"the task for which you want a status report.")
return 1
# set output path
outPath = os.path.abspath(options.Name)
general = config.getGeneral()
config.set("internals", "workdir",
os.path.join(general["workdir"], options.Name))
config.set("internals", "scriptsdir", outPath)
config.set("general", "datadir",
os.path.join(general["datadir"], options.Name))
config.set("general", "logdir",
os.path.join(general["logdir"], options.Name))
config.set(
"general", "eosdir",
os.path.join("AlignmentValidation", general["eosdir"], options.Name))
if not os.path.exists(outPath):
os.makedirs(outPath)
elif not os.path.isdir(outPath):
raise AllInOneError(
"the file %s is in the way rename the Job or move it away" %
outPath)
# replace default templates by the ones specified in the "alternateTemplates" section
loadTemplates(config)
#save backup configuration file
backupConfigFile = open(os.path.join(outPath, "usedConfiguration.ini"),
"w")
config.write(backupConfigFile)
#copy proxy, if there is one
try:
proxyexists = int(getCommandOutput2("voms-proxy-info --timeleft")) > 10
except RuntimeError:
proxyexists = False
if proxyexists:
shutil.copyfile(
getCommandOutput2("voms-proxy-info --path").strip(),
os.path.join(outPath, ".user_proxy"))
validations = []
jobs = []
for validation in config.items("validation"):
alignmentList = [validation[1]]
validationsToAdd = [(validation[0],alignment) \
for alignment in alignmentList]
validations.extend(validationsToAdd)
for validation in validations:
job = ValidationJobMultiIOV(validation, config, options, outPath,
len(validations))
if (job.optionMultiIOV == True):
jobs.extend(job)
else:
jobs.extend(ValidationJob(validation, config, options, 1))
for job in jobs:
if job.needsproxy and not proxyexists:
raise AllInOneError(
"At least one job needs a grid proxy, please init one.")
map(lambda job: job.createJob(), jobs)
validations = [job.getValidation() for job in jobs]
validations = flatten(validations)
createMergeScript(outPath, validations, options)
map(lambda job: job.runJob(), jobs)
ValidationJobMultiIOV.runCondorJobs(outPath)
