def main(argv=None): if argv == None: argv = sys.argv[1:] optParser = optparse.OptionParser() optParser.description = """All-in-one Alignment Validation. This will run various validation procedures either on batch queues or interactively. If no name is given (-N parameter) a name containing time and date is created automatically. To merge the outcome of all validation procedures run TkAlMerge.sh in your validation's directory. """ optParser.add_option( "-n", "--dryRun", dest="dryRun", action="store_true", default=False, help= "create all scripts and cfg File but do not start jobs (default=False)" ) optParser.add_option( "--getImages", dest="getImages", action="store_true", default=True, help="get all Images created during the process (default= True)") defaultConfig = "TkAlConfig.ini" optParser.add_option( "-c", "--config", dest="config", default=defaultConfig, help= "configuration to use (default TkAlConfig.ini) this can be a comma-seperated list of all .ini file you want to merge", metavar="CONFIG") optParser.add_option( "-N", "--Name", dest="Name", help="Name of this validation (default: alignmentValidation_DATE_TIME)", metavar="NAME") optParser.add_option( "-r", "--restrictTo", dest="restrictTo", help= "restrict validations to given modes (comma seperated) (default: no restriction)", metavar="RESTRICTTO") optParser.add_option("-s", "--status", dest="crabStatus", action="store_true", default=False, help="get the status of the crab jobs", metavar="STATUS") optParser.add_option("-d", "--debug", dest="debugMode", action="store_true", default=False, help="run the tool to get full traceback of errors", metavar="DEBUG") optParser.add_option( "-m", "--autoMerge", dest="autoMerge", action="store_true", default=False, help= "submit TkAlMerge.sh to run automatically when all jobs have finished (default=False)." " Works only for batch jobs") optParser.add_option( "--mergeOfflineParallel", dest="mergeOfflineParallel", action="store_true", default=False, help= "Enable parallel merging of offline data. Best used with -m option. Only works with lxBatch-jobmode", metavar="MERGE_PARALLEL") (options, args) = optParser.parse_args(argv) if not options.restrictTo == None: options.restrictTo = options.restrictTo.split(",") options.config = [ os.path.abspath( iniFile ) for iniFile in \ options.config.split( "," ) ] config = BetterConfigParser() outputIniFileSet = set(config.read(options.config)) failedIniFiles = [ iniFile for iniFile in options.config if iniFile not in outputIniFileSet ] # Check for missing ini file if options.config == [os.path.abspath(defaultConfig)]: if ( not options.crabStatus ) and \ ( not os.path.exists( defaultConfig ) ): raise AllInOneError("Default 'ini' file '%s' not found!\n" "You can specify another name with the " "command line option '-c'/'--config'." % (defaultConfig)) else: for iniFile in failedIniFiles: if not os.path.exists(iniFile): raise AllInOneError("'%s' does not exist. Please check for " "typos in the filename passed to the " "'-c'/'--config' option!" % (iniFile)) else: raise AllInOneError(("'%s' does exist, but parsing of the " "content failed!") % iniFile) # get the job name if options.Name == None: if not options.crabStatus: options.Name = "alignmentValidation_%s" % ( datetime.datetime.now().strftime("%y%m%d_%H%M%S")) else: existingValDirs = fnmatch.filter( os.walk('.').next()[1], "alignmentValidation_*") if len(existingValDirs) > 0: options.Name = existingValDirs[-1] else: print("Cannot guess last working directory!") print("Please use the parameter '-N' or '--Name' to specify " "the task for which you want a status report.") return 1 # set output path outPath = os.path.abspath(options.Name) # Check status of submitted jobs and return if options.crabStatus: os.chdir(outPath) crabLogDirs = fnmatch.filter(os.walk('.').next()[1], "crab.*") if len(crabLogDirs) == 0: print("Found no crab tasks for job name '%s'" % (options.Name)) return 1 theCrab = crabWrapper.CrabWrapper() for crabLogDir in crabLogDirs: print() print("*" + "=" * 78 + "*") print("| Status report and output retrieval for:" + " " * (77 - len("Status report and output retrieval for:")) + "|") taskName = crabLogDir.replace("crab.", "") print("| " + taskName + " " * (77 - len(taskName)) + "|") print("*" + "=" * 78 + "*") print() crabOptions = {"-getoutput": "", "-c": crabLogDir} try: theCrab.run(crabOptions) except AllInOneError as e: print("crab: No output retrieved for this task.") crabOptions = {"-status": "", "-c": crabLogDir} theCrab.run(crabOptions) return general = config.getGeneral() config.set("internals", "workdir", os.path.join(general["workdir"], options.Name)) config.set("internals", "scriptsdir", outPath) config.set("general", "datadir", os.path.join(general["datadir"], options.Name)) config.set("general", "logdir", os.path.join(general["logdir"], options.Name)) config.set( "general", "eosdir", os.path.join("AlignmentValidation", general["eosdir"], options.Name)) if not os.path.exists(outPath): os.makedirs(outPath) elif not os.path.isdir(outPath): raise AllInOneError( "the file %s is in the way rename the Job or move it away" % outPath) # replace default templates by the ones specified in the "alternateTemplates" section loadTemplates(config) #save backup configuration file backupConfigFile = open(os.path.join(outPath, "usedConfiguration.ini"), "w") config.write(backupConfigFile) #copy proxy, if there is one try: proxyexists = int(getCommandOutput2("voms-proxy-info --timeleft")) > 10 except RuntimeError: proxyexists = False if proxyexists: shutil.copyfile( getCommandOutput2("voms-proxy-info --path").strip(), os.path.join(outPath, ".user_proxy")) validations = [] for validation in config.items("validation"): alignmentList = [validation[1]] validationsToAdd = [(validation[0],alignment) \ for alignment in alignmentList] validations.extend(validationsToAdd) jobs = [ ValidationJob( validation, config, options) \ for validation in validations ] for job in jobs: if job.needsproxy and not proxyexists: raise AllInOneError( "At least one job needs a grid proxy, please init one.") map(lambda job: job.createJob(), jobs) validations = [job.getValidation() for job in jobs] if options.mergeOfflineParallel: parallelMergeObjects = createMergeScript( outPath, validations, options)['parallelMergeObjects'] else: createMergeScript(outPath, validations, options) print() map(lambda job: job.runJob(), jobs) if options.autoMerge and ValidationJob.jobCount == ValidationJob.batchCount and config.getGeneral( )["jobmode"].split(",")[0] == "lxBatch": print("> Automatically merging jobs when they have ended") # if everything is done as batch job, also submit TkAlMerge.sh to be run # after the jobs have finished #if parallel merge scripts: manage dependencies if options.mergeOfflineParallel and parallelMergeObjects != {}: initID = parallelMergeObjects["init"].runJob(config).split( "<")[1].split(">")[0] parallelIDs = [] for parallelMergeScript in parallelMergeObjects["parallel"]: parallelMergeScript.addDependency(initID) for job in jobs: if isinstance( job.validation, OfflineValidation ) and "TkAlMerge" + job.validation.alignmentToValidate.name == parallelMergeScript.name: parallelMergeScript.addDependency(job.JobId) parallelIDs.append( parallelMergeScript.runJob(config).split("<")[1].split(">") [0]) parallelMergeObjects["continue"].addDependency(parallelIDs) parallelMergeObjects["continue"].addDependency( ValidationJob.batchJobIds) parallelMergeObjects["continue"].runJob(config) else: repMap = { "commands": config.getGeneral()["jobmode"].split(",")[1], "jobName": "TkAlMerge", "logDir": config.getGeneral()["logdir"], "script": "TkAlMerge.sh", "bsub": "/afs/cern.ch/cms/caf/scripts/cmsbsub", "conditions": '"' + " && ".join([ "ended(" + jobId + ")" for jobId in ValidationJob.batchJobIds ]) + '"' } for ext in ("stdout", "stderr", "stdout.gz", "stderr.gz"): oldlog = "%(logDir)s/%(jobName)s." % repMap + ext if os.path.exists(oldlog): os.remove(oldlog) #issue job getCommandOutput2("%(bsub)s %(commands)s " "-o %(logDir)s/%(jobName)s.stdout " "-e %(logDir)s/%(jobName)s.stderr " "-w %(conditions)s " "%(logDir)s/%(script)s" % repMap)
def main(argv=None): if argv == None: argv = sys.argv[1:] optParser = optparse.OptionParser() optParser.description = """All-in-one Alignment Validation. This will run various validation procedures either on batch queues or interactively. If no name is given (-N parameter) a name containing time and date is created automatically. To merge the outcome of all validation procedures run TkAlMerge.sh in your validation's directory. """ optParser.add_option( "-n", "--dryRun", dest="dryRun", action="store_true", default=False, help= "create all scripts and cfg File but do not start jobs (default=False)" ) optParser.add_option( "--getImages", dest="getImages", action="store_true", default=True, help="get all Images created during the process (default= True)") defaultConfig = "TkAlConfig.ini" optParser.add_option( "-c", "--config", dest="config", default=defaultConfig, help= "configuration to use (default TkAlConfig.ini) this can be a comma-seperated list of all .ini file you want to merge", metavar="CONFIG") optParser.add_option( "-N", "--Name", dest="Name", help="Name of this validation (default: alignmentValidation_DATE_TIME)", metavar="NAME") optParser.add_option( "-r", "--restrictTo", dest="restrictTo", help= "restrict validations to given modes (comma seperated) (default: no restriction)", metavar="RESTRICTTO") optParser.add_option("-d", "--debug", dest="debugMode", action="store_true", default=False, help="run the tool to get full traceback of errors", metavar="DEBUG") (options, args) = optParser.parse_args(argv) if not options.restrictTo == None: options.restrictTo = options.restrictTo.split(",") options.config = [ os.path.abspath( iniFile ) for iniFile in \ options.config.split( "," )] config = BetterConfigParser() outputIniFileSet = set(config.read(options.config)) failedIniFiles = [ iniFile for iniFile in options.config if iniFile not in outputIniFileSet ] # Check for missing ini file if options.config == [os.path.abspath(defaultConfig)]: if (not os.path.exists(defaultConfig)): raise AllInOneError("Default 'ini' file '%s' not found!\n" "You can specify another name with the " "command line option '-c'/'--config'." % (defaultConfig)) else: for iniFile in failedIniFiles: if not os.path.exists(iniFile): raise AllInOneError("'%s' does not exist. Please check for " "typos in the filename passed to the " "'-c'/'--config' option!" % (iniFile)) else: raise AllInOneError(("'%s' does exist, but parsing of the " "content failed!") % iniFile) # get the job name if options.Name == None: existingValDirs = fnmatch.filter( os.walk('.').next()[1], "alignmentValidation_*") if len(existingValDirs) > 0: options.Name = existingValDirs[-1] else: print("Cannot guess last working directory!") print("Please use the parameter '-N' or '--Name' to specify " "the task for which you want a status report.") return 1 # set output path outPath = os.path.abspath(options.Name) general = config.getGeneral() config.set("internals", "workdir", os.path.join(general["workdir"], options.Name)) config.set("internals", "scriptsdir", outPath) config.set("general", "datadir", os.path.join(general["datadir"], options.Name)) config.set("general", "logdir", os.path.join(general["logdir"], options.Name)) config.set( "general", "eosdir", os.path.join("AlignmentValidation", general["eosdir"], options.Name)) if not os.path.exists(outPath): os.makedirs(outPath) elif not os.path.isdir(outPath): raise AllInOneError( "the file %s is in the way rename the Job or move it away" % outPath) # replace default templates by the ones specified in the "alternateTemplates" section loadTemplates(config) #save backup configuration file backupConfigFile = open(os.path.join(outPath, "usedConfiguration.ini"), "w") config.write(backupConfigFile) #copy proxy, if there is one try: proxyexists = int(getCommandOutput2("voms-proxy-info --timeleft")) > 10 except RuntimeError: proxyexists = False if proxyexists: shutil.copyfile( getCommandOutput2("voms-proxy-info --path").strip(), os.path.join(outPath, ".user_proxy")) validations = [] jobs = [] for validation in config.items("validation"): validation = validation[0].split("-") alignmentList = [validation[1]] validationsToAdd = [(validation[0],alignment) \ for alignment in alignmentList] validations.extend(validationsToAdd) for validation in validations: job = ValidationJobMultiIOV(validation, config, options, outPath, len(validations)) if (job.optionMultiIOV == True): jobs.extend(job) else: jobs.extend(ValidationJob(validation, config, options, 1)) for job in jobs: if job.needsproxy and not proxyexists: raise AllInOneError( "At least one job needs a grid proxy, please init one.") lmap(lambda job: job.createJob(), jobs) validations = [job.getValidation() for job in jobs] validations = flatten(validations) createMergeScript(outPath, validations, options) lmap(lambda job: job.runJob(), jobs) if options.dryRun: pass else: ValidationJobMultiIOV.runCondorJobs(outPath)
def main(argv = None): if argv == None: argv = sys.argv[1:] optParser = optparse.OptionParser() optParser.description = """All-in-one Alignment Validation. This will run various validation procedures either on batch queues or interactively. If no name is given (-N parameter) a name containing time and date is created automatically. To merge the outcome of all validation procedures run TkAlMerge.sh in your validation's directory. """ optParser.add_option("-n", "--dryRun", dest="dryRun", action="store_true", default=False, help="create all scripts and cfg File but do not start jobs (default=False)") optParser.add_option( "--getImages", dest="getImages", action="store_true", default=True, help="get all Images created during the process (default= True)") defaultConfig = "TkAlConfig.ini" optParser.add_option("-c", "--config", dest="config", default = defaultConfig, help="configuration to use (default TkAlConfig.ini) this can be a comma-seperated list of all .ini file you want to merge", metavar="CONFIG") optParser.add_option("-N", "--Name", dest="Name", help="Name of this validation (default: alignmentValidation_DATE_TIME)", metavar="NAME") optParser.add_option("-r", "--restrictTo", dest="restrictTo", help="restrict validations to given modes (comma seperated) (default: no restriction)", metavar="RESTRICTTO") optParser.add_option("-s", "--status", dest="crabStatus", action="store_true", default = False, help="get the status of the crab jobs", metavar="STATUS") optParser.add_option("-d", "--debug", dest="debugMode", action="store_true", default = False, help="run the tool to get full traceback of errors", metavar="DEBUG") optParser.add_option("-m", "--autoMerge", dest="autoMerge", action="store_true", default = False, help="submit TkAlMerge.sh to run automatically when all jobs have finished (default=False)." " Works only for batch jobs") (options, args) = optParser.parse_args(argv) if not options.restrictTo == None: options.restrictTo = options.restrictTo.split(",") options.config = [ os.path.abspath( iniFile ) for iniFile in \ options.config.split( "," ) ] config = BetterConfigParser() outputIniFileSet = set( config.read( options.config ) ) failedIniFiles = [ iniFile for iniFile in options.config if iniFile not in outputIniFileSet ] # Check for missing ini file if options.config == [ os.path.abspath( defaultConfig ) ]: if ( not options.crabStatus ) and \ ( not os.path.exists( defaultConfig ) ): raise AllInOneError( "Default 'ini' file '%s' not found!\n" "You can specify another name with the " "command line option '-c'/'--config'." %( defaultConfig )) else: for iniFile in failedIniFiles: if not os.path.exists( iniFile ): raise AllInOneError( "'%s' does not exist. Please check for " "typos in the filename passed to the " "'-c'/'--config' option!" %( iniFile )) else: raise AllInOneError(( "'%s' does exist, but parsing of the " "content failed!" ) % iniFile) # get the job name if options.Name == None: if not options.crabStatus: options.Name = "alignmentValidation_%s"%(datetime.datetime.now().strftime("%y%m%d_%H%M%S")) else: existingValDirs = fnmatch.filter( os.walk( '.' ).next()[1], "alignmentValidation_*" ) if len( existingValDirs ) > 0: options.Name = existingValDirs[-1] else: print "Cannot guess last working directory!" print ( "Please use the parameter '-N' or '--Name' to specify " "the task for which you want a status report." ) return 1 # set output path outPath = os.path.abspath( options.Name ) # Check status of submitted jobs and return if options.crabStatus: os.chdir( outPath ) crabLogDirs = fnmatch.filter( os.walk('.').next()[1], "crab.*" ) if len( crabLogDirs ) == 0: print "Found no crab tasks for job name '%s'"%( options.Name ) return 1 theCrab = crabWrapper.CrabWrapper() for crabLogDir in crabLogDirs: print print "*" + "=" * 78 + "*" print ( "| Status report and output retrieval for:" + " " * (77 - len( "Status report and output retrieval for:" ) ) + "|" ) taskName = crabLogDir.replace( "crab.", "" ) print "| " + taskName + " " * (77 - len( taskName ) ) + "|" print "*" + "=" * 78 + "*" print crabOptions = { "-getoutput":"", "-c": crabLogDir } try: theCrab.run( crabOptions ) except AllInOneError as e: print "crab: No output retrieved for this task." crabOptions = { "-status": "", "-c": crabLogDir } theCrab.run( crabOptions ) return general = config.getGeneral() config.set("internals","workdir",os.path.join(general["workdir"],options.Name) ) config.set("internals","scriptsdir",outPath) config.set("general","datadir",os.path.join(general["datadir"],options.Name) ) config.set("general","logdir",os.path.join(general["logdir"],options.Name) ) config.set("general","eosdir",os.path.join("AlignmentValidation", general["eosdir"], options.Name) ) if not os.path.exists( outPath ): os.makedirs( outPath ) elif not os.path.isdir( outPath ): raise AllInOneError("the file %s is in the way rename the Job or move it away"%outPath) # replace default templates by the ones specified in the "alternateTemplates" section loadTemplates( config ) #save backup configuration file backupConfigFile = open( os.path.join( outPath, "usedConfiguration.ini" ) , "w" ) config.write( backupConfigFile ) #copy proxy, if there is one try: proxyexists = int(getCommandOutput2("voms-proxy-info --timeleft")) > 10 except RuntimeError: proxyexists = False if proxyexists: shutil.copyfile(getCommandOutput2("voms-proxy-info --path").strip(), os.path.join(outPath, ".user_proxy")) validations = [] for validation in config.items("validation"): alignmentList = [validation[1]] validationsToAdd = [(validation[0],alignment) \ for alignment in alignmentList] validations.extend(validationsToAdd) jobs = [ ValidationJob( validation, config, options) \ for validation in validations ] for job in jobs: if job.needsproxy and not proxyexists: raise AllInOneError("At least one job needs a grid proxy, please init one.") map( lambda job: job.createJob(), jobs ) validations = [ job.getValidation() for job in jobs ] createMergeScript(outPath, validations) print map( lambda job: job.runJob(), jobs ) if options.autoMerge: # if everything is done as batch job, also submit TkAlMerge.sh to be run # after the jobs have finished if ValidationJob.jobCount == ValidationJob.batchCount and config.getGeneral()["jobmode"].split(",")[0] == "lxBatch": print "> Automatically merging jobs when they have ended" repMap = { "commands": config.getGeneral()["jobmode"].split(",")[1], "jobName": "TkAlMerge", "logDir": config.getGeneral()["logdir"], "script": "TkAlMerge.sh", "bsub": "/afs/cern.ch/cms/caf/scripts/cmsbsub", "conditions": '"' + " && ".join(["ended(" + jobId + ")" for jobId in ValidationJob.batchJobIds]) + '"' } for ext in ("stdout", "stderr", "stdout.gz", "stderr.gz"): oldlog = "%(logDir)s/%(jobName)s."%repMap + ext if os.path.exists(oldlog): os.remove(oldlog) getCommandOutput2("%(bsub)s %(commands)s " "-o %(logDir)s/%(jobName)s.stdout " "-e %(logDir)s/%(jobName)s.stderr " "-w %(conditions)s " "%(logDir)s/%(script)s"%repMap)