def update_known_error(worflow, workflow_dir): known_errors = get_known_errors(os.environ["CMSSW_VERSION"], os.environ["SCRAM_ARCH"], "relvals") if worflow in known_errors: json.dump(known_errors[workflow], open("%s/known_error.json" % workflow_dir, "w")) return
shell=True) e = waitpid(p.pid, 0)[1] print("Time took to create jobs:", int(time() - stime), "sec") system("touch " + cmssw_base + "/done." + opts.jobid) if logger: logger.updateRelValMatrixPartialLogs(cmssw_base, "done." + opts.jobid) exit(e) if isThreaded(cmssw_ver, arch): print("Threaded IB Found") thrds = int(MachineMemoryGB / 4.5) if thrds == 0: thrds = 1 elif "fc24_ppc64le_" in arch: print("FC22 IB Found") thrds = int(MachineMemoryGB / 4) elif "fc24_ppc64le_" in arch: print("CentOS 7.2 + PPC64LE Found") thrds = int(MachineMemoryGB / 3) else: print("Normal IB Found") if thrds > cmsRunProcessCount: thrds = cmsRunProcessCount known_errs = get_known_errors(cmssw_ver, arch, "relvals") matrix = PyRelValsThread(thrds, cmssw_base + "/pyRelval", opts.jobid) matrix.setArgs(GetMatrixOptions(cmssw_ver, arch)) matrix.run_workflows(opts.workflow.split(","), logger, opts.force, known_errors=known_errs)
cmssw_ver = environ["CMSSW_VERSION"] arch = environ["SCRAM_ARCH"] if isThreaded(cmssw_ver,arch): print "Threaded IB Found" thrds=int(MachineMemoryGB/4.5) if thrds==0: thrds=1 elif "fc24_ppc64le_" in arch: print "FC22 IB Found" thrds=int(MachineMemoryGB/4) elif "fc24_ppc64le_" in arch: print "CentOS 7.2 + PPC64LE Found" thrds=int(MachineMemoryGB/3) else: print "Normal IB Found" if thrds>cmsRunProcessCount: thrds=cmsRunProcessCount known_errs = get_known_errors(cmssw_ver, arch, "relvals") matrix = PyRelValsThread(thrds, environ["CMSSW_BASE"]+"/pyRelval", opts.jobid) matrix.setArgs(GetMatrixOptions(cmssw_ver,arch)) #print matrix.args #print GetMatrixOptions(cmssw_ver,arch) #print matrix.args['rest'] wfs = opts.workflow.split(",") for wf in wfs: print 'flow is :', wf #matrix.args['rest'] = FixWFArgs(cmssw_ver,arch,wf,matrix.args['rest']).replace(' -t 4','') #print fixed_args #matrix.args['rest'] = fixed_args.replace(' -t 4','') #print matrix.args['rest'] #print matrix.args['w'] runStep1Only(matrix.basedir, wf, matrix.args['rest']+" "+matrix.args['w'])
''' here the program is tested ''' avg_mem = 0.90 * psutil.virtual_memory()[0] avg_cpu = 200 * cpu_count() wf_limit = 1000 #print psutil.virtual_memory()[] #exit(0) toProcessQueue = Queue.Queue() processedTasksQueue = Queue.Queue() getNextJobsEvent = Event() finishJobsEvent = Event() known_errors = get_known_errors(opts.release, opts.arch, 'relvals') #print known_errors jc = JobsConstructor(None, known_errors) matrixMap = jc.constructJobsMatrix(opts.release, opts.arch, opts.days, opts.page_size, wf_list, wf_limit, os.environ["CMSSW_BASE"] + "/pyRelval/") ''' up to here it constructs the jobs stats''' jm = JobsManager(matrixMap) jm.toProcessQueue = toProcessQueue jm.processedQueue = processedTasksQueue jm.availableMemory = avg_mem jm.availableCPU = avg_cpu jp = JobsProcessor(toProcessQueue, processedTasksQueue)
def update_known_error(worflow, workflow_dir): known_errors = get_known_errors(os.environ["CMSSW_VERSION"], os.environ["SCRAM_ARCH"], "relvals") if worflow in known_errors: json.dump(known_errors[workflow], open("%s/known_error.json" % workflow_dir,"w")) return
''' here the program is tested ''' avg_mem = 0.90*psutil.virtual_memory()[0] avg_cpu = 200*cpu_count() wf_limit = 1000 #print psutil.virtual_memory()[] #exit(0) toProcessQueue = Queue.Queue() processedTasksQueue = Queue.Queue() getNextJobsEvent = Event() finishJobsEvent = Event() known_errors = get_known_errors(opts.release, opts.arch, 'relvals') #print known_errors jc = JobsConstructor(None, known_errors) matrixMap =jc.constructJobsMatrix(opts.release, opts.arch, opts.days, opts.page_size, wf_list, wf_limit,os.environ["CMSSW_BASE"]+"/pyRelval/") ''' up to here it constructs the jobs stats''' jm = JobsManager(matrixMap) jm.toProcessQueue = toProcessQueue jm.processedQueue = processedTasksQueue jm.availableMemory = avg_mem jm.availableCPU = avg_cpu jp = JobsProcessor(toProcessQueue, processedTasksQueue) jp.allJobs = jm.jobs