def update_known_error(worflow, workflow_dir):
known_errors = get_known_errors(os.environ["CMSSW_VERSION"],
os.environ["SCRAM_ARCH"], "relvals")
if worflow in known_errors:
json.dump(known_errors[workflow],
open("%s/known_error.json" % workflow_dir, "w"))
return
shell=True)
e = waitpid(p.pid, 0)[1]
print("Time took to create jobs:", int(time() - stime), "sec")
system("touch " + cmssw_base + "/done." + opts.jobid)
if logger:
logger.updateRelValMatrixPartialLogs(cmssw_base,
"done." + opts.jobid)
exit(e)
if isThreaded(cmssw_ver, arch):
print("Threaded IB Found")
thrds = int(MachineMemoryGB / 4.5)
if thrds == 0: thrds = 1
elif "fc24_ppc64le_" in arch:
print("FC22 IB Found")
thrds = int(MachineMemoryGB / 4)
elif "fc24_ppc64le_" in arch:
print("CentOS 7.2 + PPC64LE Found")
thrds = int(MachineMemoryGB / 3)
else:
print("Normal IB Found")
if thrds > cmsRunProcessCount: thrds = cmsRunProcessCount
known_errs = get_known_errors(cmssw_ver, arch, "relvals")
matrix = PyRelValsThread(thrds, cmssw_base + "/pyRelval", opts.jobid)
matrix.setArgs(GetMatrixOptions(cmssw_ver, arch))
matrix.run_workflows(opts.workflow.split(","),
logger,
opts.force,
known_errors=known_errs)
cmssw_ver = environ["CMSSW_VERSION"]
arch = environ["SCRAM_ARCH"]
if isThreaded(cmssw_ver,arch):
print "Threaded IB Found"
thrds=int(MachineMemoryGB/4.5)
if thrds==0: thrds=1
elif "fc24_ppc64le_" in arch:
print "FC22 IB Found"
thrds=int(MachineMemoryGB/4)
elif "fc24_ppc64le_" in arch:
print "CentOS 7.2 + PPC64LE Found"
thrds=int(MachineMemoryGB/3)
else:
print "Normal IB Found"
if thrds>cmsRunProcessCount: thrds=cmsRunProcessCount
known_errs = get_known_errors(cmssw_ver, arch, "relvals")
matrix = PyRelValsThread(thrds, environ["CMSSW_BASE"]+"/pyRelval", opts.jobid)
matrix.setArgs(GetMatrixOptions(cmssw_ver,arch))
#print matrix.args
#print GetMatrixOptions(cmssw_ver,arch)
#print matrix.args['rest']
wfs = opts.workflow.split(",")
for wf in wfs:
print 'flow is :', wf
#matrix.args['rest'] = FixWFArgs(cmssw_ver,arch,wf,matrix.args['rest']).replace(' -t 4','')
#print fixed_args
#matrix.args['rest'] = fixed_args.replace(' -t 4','')
#print matrix.args['rest']
#print matrix.args['w']
runStep1Only(matrix.basedir, wf, matrix.args['rest']+" "+matrix.args['w'])
''' here the program is tested '''
avg_mem = 0.90 * psutil.virtual_memory()[0]
avg_cpu = 200 * cpu_count()
wf_limit = 1000
#print psutil.virtual_memory()[]
#exit(0)
toProcessQueue = Queue.Queue()
processedTasksQueue = Queue.Queue()
getNextJobsEvent = Event()
finishJobsEvent = Event()
known_errors = get_known_errors(opts.release, opts.arch, 'relvals')
#print known_errors
jc = JobsConstructor(None, known_errors)
matrixMap = jc.constructJobsMatrix(opts.release, opts.arch, opts.days,
opts.page_size, wf_list, wf_limit,
os.environ["CMSSW_BASE"] + "/pyRelval/")
''' up to here it constructs the jobs stats'''
jm = JobsManager(matrixMap)
jm.toProcessQueue = toProcessQueue
jm.processedQueue = processedTasksQueue
jm.availableMemory = avg_mem
jm.availableCPU = avg_cpu
jp = JobsProcessor(toProcessQueue, processedTasksQueue)
def update_known_error(worflow, workflow_dir):
known_errors = get_known_errors(os.environ["CMSSW_VERSION"], os.environ["SCRAM_ARCH"], "relvals")
if worflow in known_errors:
json.dump(known_errors[workflow], open("%s/known_error.json" % workflow_dir,"w"))
return
''' here the program is tested '''
avg_mem = 0.90*psutil.virtual_memory()[0]
avg_cpu = 200*cpu_count()
wf_limit = 1000
#print psutil.virtual_memory()[]
#exit(0)
toProcessQueue = Queue.Queue()
processedTasksQueue = Queue.Queue()
getNextJobsEvent = Event()
finishJobsEvent = Event()
known_errors = get_known_errors(opts.release, opts.arch, 'relvals')
#print known_errors
jc = JobsConstructor(None, known_errors)
matrixMap =jc.constructJobsMatrix(opts.release, opts.arch, opts.days, opts.page_size, wf_list, wf_limit,os.environ["CMSSW_BASE"]+"/pyRelval/")
''' up to here it constructs the jobs stats'''
jm = JobsManager(matrixMap)
jm.toProcessQueue = toProcessQueue
jm.processedQueue = processedTasksQueue
jm.availableMemory = avg_mem
jm.availableCPU = avg_cpu
jp = JobsProcessor(toProcessQueue, processedTasksQueue)
jp.allJobs = jm.jobs
