def runTest():
global nClients, nQueries, testType, resultTest, testDir, lfnListFile
resultTest = []
pp = ProcessPool(nClients)
testFunction = eval(testType)
for c in xrange(nClients):
pp.createAndQueueTask(testFunction, [nQueries],
callback=finalize,
exceptionCallback=doException)
pp.processAllResults(3600)
pp.finalize(0)
timeResult = []
for testTime, success, failure in resultTest:
#print testTime,success,failure
timeResult += testTime
averageTime, errorTime = doStats(timeResult)
rateResult = [nClients / t for t in timeResult]
averageRate, errorRate = doStats(rateResult)
if testDir:
print "\nTest results for clients %d, %s" % (nClients, testDir)
else:
print "\nTest results for clients %d, %s" % (nClients, lfnListFile)
print "Query time: %.2f +/- %.2f" % (averageTime, errorTime)
print "Query rate: %.2f +/- %.2f" % (averageRate, errorRate)
return ((averageTime, errorTime), (averageRate, errorRate))
def runTest():
global nClients, nQueries, testType, resultTest, testDir, lfnListFile
resultTest = []
pp = ProcessPool(nClients)
testFunction = eval(testType)
for c in xrange(nClients):
pp.createAndQueueTask(testFunction, [nQueries],
callback=finalize,
exceptionCallback=doException)
pp.processAllResults(3600)
pp.finalize(0)
timeResult = []
for testTime, success, failure in resultTest:
# print testTime,success,failure
timeResult += testTime
averageTime, errorTime = doStats(timeResult)
rateResult = [nClients / t for t in timeResult]
averageRate, errorRate = doStats(rateResult)
if testDir:
print "\nTest results for clients %d, %s" % (nClients, testDir)
else:
print "\nTest results for clients %d, %s" % (nClients, lfnListFile)
print "Query time: %.2f +/- %.2f" % (averageTime, errorTime)
print "Query rate: %.2f +/- %.2f" % (averageRate, errorRate)
return((averageTime, errorTime), (averageRate, errorRate))
pPool = ProcessPool(10,50,50)
pPool.daemonize()
# dirlist = ['prod/ilc/mc-dbd/generated','prod/ilc/mc-dbd/ild']
# dirlist= ['prod/ilc/mc-dbd/generated/500-TDR_ws/higgs']
# dirlist= ['prod/ilc/mc-dbd/generated/250-TDR_ws/higgs','prod/ilc/mc-dbd/generated/350-TDR_ws/higgs']
#dirlist= ['prod/ilc/mc-dbd/generated/250-TDR_ws']
#dirlist= ['prod/ilc/mc-dbd/generated/250-TDR_ws/1f',
# 'prod/ilc/mc-dbd/generated/250-TDR_ws/3f',
# 'prod/ilc/mc-dbd/generated/250-TDR_ws/aa_lowpt',
# 'prod/ilc/mc-dbd/generated/250-TDR_ws/aa_minijet']
#dirlist= ['prod/ilc/mc-dbd/generated/250-TDR_ws/aa_2f',
# 'prod/ilc/mc-dbd/generated/350-TDR_ws/3f',
# 'prod/ilc/mc-dbd/generated/350-TDR_ws/1f',
# 'prod/ilc/mc-dbd/generated/350-TDR_ws/aa_minijet']
lfcHosts = ['grid-lfc.desy.de']
for dir in dirlist:
path = "/ilc/%s" % (dir)
print "Queueing user", dir, pPool.getFreeSlots(),pPool.getNumWorkingProcesses(),pPool.hasPendingTasks(),pPool.getNumIdleProcesses(), lfcHosts[0]
result = pPool.createAndQueueTask( processDir,[path,True,lfcHosts[0]],callback=finalizeDirectory )
if not result['OK']:
print "Failed queueing", path
pPool.processAllResults()
print "LFC_to_DFC completed."
# 'lfc-lhcb.grid.sara.nl', # 'lfclhcb.pic.es', # 'lhcb-lfc.gridpp.rl.ac.uk'] lfcHosts = ['prod-lfc-lhcb-ro.cern.ch'] # path = "/lhcb/LHCb" path = '/lhcb/user/c/chaen' print "Queueing task for directory", path, lfcHosts[0] writerProc = Process( target = writer, args = ( 'lfc_dfc.out', writerQueue, stopFlag ) ) writerProc.start() result = pPool.createAndQueueTask( processDir, [path , writerQueue, False, lfcHosts[0]], callback = finalizeDirectory ) if not result['OK']: print "Failed queueing", path for i in range(20): pPool.processResults() time.sleep(1) pPool.processAllResults( timeout = 300 ) stopFlag.value = 1 writerQueue.put( "Exit" ) writerProc.join()
# 'lfc-lhcb-ro.in2p3.fr',
# 'lfc-lhcb.grid.sara.nl',
# 'lfclhcb.pic.es',
# 'lhcb-lfc.gridpp.rl.ac.uk']
lfcHosts = ['prod-lfc-lhcb-ro.cern.ch']
# path = "/lhcb/LHCb"
path = '/lhcb/user/c/chaen'
print("Queueing task for directory", path, lfcHosts[0])
writerProc = Process(target=writer,
args=('lfc_dfc.out', writerQueue, stopFlag))
writerProc.start()
result = pPool.createAndQueueTask(processDir,
[path, writerQueue, False, lfcHosts[0]],
callback=finalizeDirectory)
if not result['OK']:
print("Failed queueing", path)
for i in range(20):
pPool.processResults()
time.sleep(1)
pPool.processAllResults(timeout=300)
stopFlag.value = 1
writerQueue.put("Exit")
writerProc.join()
pPool = ProcessPool(30, 40, 0)
#pPool.daemonize()
lfcHosts = [
'lfc-lhcb-ro.cern.ch', 'lfc-lhcb-ro.cr.cnaf.infn.it',
'lhcb-lfc-fzk.gridka.de', 'lfc-lhcb-ro.in2p3.fr', 'lfc-lhcb.grid.sara.nl',
'lfclhcb.pic.es', 'lhcb-lfc.gridpp.rl.ac.uk'
]
#for user in users:
# path = "/lhcb/user/%s/%s" % (user[0],user)
# random.shuffle(lfcHosts)
# print "Queueing user", user, pPool.getFreeSlots(),pPool.getNumWorkingProcesses(),pPool.hasPendingTasks(),pPool.getNumIdleProcesses(), lfcHosts[0]
# result = pPool.createAndQueueTask( processDir,[path,True,lfcHosts[0]],callback=finalizeDirectory )
# if not result['OK']:
# print "Failed queueing", path
path = "/lhcb/LHCb"
print "Queueing task for directory", path, lfcHosts[0]
result = pPool.createAndQueueTask(processDir, [path, False, lfcHosts[0]],
callback=finalizeDirectory)
if not result['OK']:
print "Failed queueing", path
for i in range(20):
pPool.processResults()
time.sleep(1)
pPool.processAllResults()
