if __name__ == "__main__": parser = OptionParser(usage="%prog -i|--id <jobid> -l|--list <list of workflows>") parser.add_option("-i", "--id", dest="jobid", help="Job Id e.g. 1of3", default="1of1") parser.add_option("-l", "--list", dest="workflow", help="List of workflows to run e.g. 1.0,2.0,3.0", type=str, default=None) parser.add_option("-f", "--force",dest="force", help="Force running of workflows without checking the server for previous run", action="store_true", default=False) parser.add_option("-d", "--das-cache", dest="das_cache", help="Das cache file", type=str, default=None) opts, args = parser.parse_args() if len(args) > 0: parser.error("Too many/few arguments") if not opts.workflow: parser.error("Missing -l|--list <workflows> argument.") if (not environ.has_key("CMSSW_VERSION")) or (not environ.has_key("CMSSW_BASE")) or (not environ.has_key("SCRAM_ARCH")): print "ERROR: Unable to file the release environment, please make sure you have set the cmssw environment before calling this script" exit(1) thrds = cmsRunProcessCount if isThreaded(environ["CMSSW_VERSION"],environ["SCRAM_ARCH"]): print "Treaded IB Found" thrds=int(MachineMemoryGB/5) elif "fc22_" in environ["SCRAM_ARCH"]: print "FC22 IB Found" thrds=int(MachineMemoryGB/4) else: print "Normal IB Found" if thrds>cmsRunProcessCount: thrds=cmsRunProcessCount matrix = PyRelValsThread(thrds, environ["CMSSW_BASE"]+"/pyRelval", opts.jobid) matrix.setArgs(GetMatrixOptions(environ["CMSSW_VERSION"],environ["SCRAM_ARCH"], opts.das_cache)) matrix.run_workflows(opts.workflow.split(","),LogUpdater(environ["CMSSW_BASE"]),opts.force)
if isThreaded(cmssw_ver,arch): print "Threaded IB Found" thrds=int(MachineMemoryGB/4.5) if thrds==0: thrds=1 elif "fc24_ppc64le_" in arch: print "FC22 IB Found" thrds=int(MachineMemoryGB/4) elif "fc24_ppc64le_" in arch: print "CentOS 7.2 + PPC64LE Found" thrds=int(MachineMemoryGB/3) else: print "Normal IB Found" if thrds>cmsRunProcessCount: thrds=cmsRunProcessCount known_errs = get_known_errors(cmssw_ver, arch, "relvals") matrix = PyRelValsThread(thrds, environ["CMSSW_BASE"]+"/pyRelval", opts.jobid) matrix.setArgs(GetMatrixOptions(cmssw_ver,arch)) #print matrix.args #print GetMatrixOptions(cmssw_ver,arch) #print matrix.args['rest'] wfs = opts.workflow.split(",") for wf in wfs: print 'flow is :', wf #matrix.args['rest'] = FixWFArgs(cmssw_ver,arch,wf,matrix.args['rest']).replace(' -t 4','') #print fixed_args #matrix.args['rest'] = fixed_args.replace(' -t 4','') #print matrix.args['rest'] #print matrix.args['w'] runStep1Only(matrix.basedir, wf, matrix.args['rest']+" "+matrix.args['w']) wf_folder = [f for f in os.listdir(matrix.basedir) if f.find(wf+'_') is not -1][0] print 'wf is:', wf, 'folder is', os.path.join(matrix.basedir,wf_folder)
shell=True) e = waitpid(p.pid, 0)[1] print("Time took to create jobs:", int(time() - stime), "sec") system("touch " + cmssw_base + "/done." + opts.jobid) if logger: logger.updateRelValMatrixPartialLogs(cmssw_base, "done." + opts.jobid) exit(e) if isThreaded(cmssw_ver, arch): print("Threaded IB Found") thrds = int(MachineMemoryGB / 4.5) if thrds == 0: thrds = 1 elif "fc24_ppc64le_" in arch: print("FC22 IB Found") thrds = int(MachineMemoryGB / 4) elif "fc24_ppc64le_" in arch: print("CentOS 7.2 + PPC64LE Found") thrds = int(MachineMemoryGB / 3) else: print("Normal IB Found") if thrds > cmsRunProcessCount: thrds = cmsRunProcessCount known_errs = get_known_errors(cmssw_ver, arch, "relvals") matrix = PyRelValsThread(thrds, cmssw_base + "/pyRelval", opts.jobid) matrix.setArgs(GetMatrixOptions(cmssw_ver, arch)) matrix.run_workflows(opts.workflow.split(","), logger, opts.force, known_errors=known_errs)
if len(args) > 0: parser.error("Too many/few arguments") if not opts.workflow: parser.error("Missing -l|--list <workflows> argument.") if (not environ.has_key("CMSSW_VERSION")) or ( not environ.has_key("CMSSW_BASE")) or ( not environ.has_key("SCRAM_ARCH")): print "ERROR: Unable to file the release environment, please make sure you have set the cmssw environment before calling this script" exit(1) thrds = cmsRunProcessCount if isThreaded(environ["CMSSW_VERSION"], environ["SCRAM_ARCH"]): print "Threaded IB Found" thrds = int(MachineMemoryGB / 4.5) if thrds == 0: thrds = 1 elif "fc24_ppc64le_" in environ["SCRAM_ARCH"]: print "FC22 IB Found" thrds = int(MachineMemoryGB / 4) elif "fc24_ppc64le_" in environ["SCRAM_ARCH"]: print "CentOS 7.2 + PPC64LE Found" thrds = int(MachineMemoryGB / 3) else: print "Normal IB Found" if thrds > cmsRunProcessCount: thrds = cmsRunProcessCount matrix = PyRelValsThread(thrds, environ["CMSSW_BASE"] + "/pyRelval", opts.jobid) matrix.setArgs( GetMatrixOptions(environ["CMSSW_VERSION"], environ["SCRAM_ARCH"])) matrix.run_workflows(opts.workflow.split(","), LogUpdater(environ["CMSSW_BASE"]), opts.force)