taskBufferIF = JediTaskBufferInterface()
taskBufferIF.setupInterface()
"""
from pandajedi.jediconfig import jedi_config
from pandajedi.jedicore import JediTaskBuffer
taskBufferIF = JediTaskBuffer.JediTaskBuffer(None)
from pandajedi.jediddm.DDMInterface import DDMInterface
ddmIF = DDMInterface()
ddmIF.setupInterface()
ddmIF = ddmIF.getInterface("atlas")
datasetName = "valid1.159025.ParticleGenerator_gamma_E100.recon.AOD.e3099_s2082_r6012_tid04635343_00"
lostFiles = set(["AOD.04635343._000012.pool.root.1", "AOD.04635343._000015.pool.root.1"])
def resetStatusForLostFileRecovery(datasetName, lostFiles):
# get jeditaskid
varMap = {}
varMap[":type1"] = "log"
varMap[":type2"] = "output"
varMap[":name1"] = datasetName
varMap[":name2"] = datasetName.split(":")[-1]
sqlGI = "SELECT jediTaskID,datasetID FROM {0}.JEDI_Datasets ".format(jedi_config.db.schemaJEDI)
sqlGI += "WHERE type IN (:type1,:type2) AND datasetName IN (:name1,:name2) "
res = taskBufferIF.querySQLS(sqlGI, varMap)
useResultCache=600)
tmpListList = tbIF.getTasksToBeProcessed_JEDI(None,vo,workQueue,
prodSourceLabel,
cloudName,nFiles=10,simTasks=[jediTaskID],
fullSimulation=True,
typicalNumFilesMap=typicalNumFilesMap,
simDatasets=datasetID)
taskSetupper = TaskSetupper(vo,prodSourceLabel)
taskSetupper.initializeMods(tbIF,ddmIF)
for dummyID,tmpList in tmpListList:
for taskSpec,cloudName,inputChunk in tmpList:
jobBroker = JobBroker(taskSpec.vo,taskSpec.prodSourceLabel)
tmpStat = jobBroker.initializeMods(ddmIF.getInterface(vo),tbIF)
jobBroker.setTestMode(taskSpec.vo,taskSpec.prodSourceLabel)
splitter = JobSplitter()
gen = JobGeneratorThread(None,threadPool,tbIF,ddmIF,siteMapper,False,taskSetupper,None,
None,None,None,brokerageLockIDs)
taskParamMap = None
if taskSpec.useLimitedSites():
tmpStat,taskParamMap = gen.readTaskParams(taskSpec,taskParamMap,tmpLog)
jobBroker.setLockID(taskSpec.vo,taskSpec.prodSourceLabel,123,0)
tmpStat,inputChunk = jobBroker.doBrokerage(taskSpec,cloudName,inputChunk,taskParamMap)
brokerageLockID = jobBroker.getBaseLockID(taskSpec.vo,taskSpec.prodSourceLabel)
if brokerageLockID != None:
brokerageLockIDs.append(brokerageLockID)
for brokeragelockID in brokerageLockIDs:
tbIF.unlockProcessWithPID_JEDI(taskSpec.vo,taskSpec.prodSourceLabel,workQueue.queue_id,
workQueue,
prodSourceLabel,
cloudName,
nFiles=10,
simTasks=[jediTaskID],
fullSimulation=True,
typicalNumFilesMap=typicalNumFilesMap,
simDatasets=datasetIDs)
taskSetupper = TaskSetupper(vo, prodSourceLabel)
taskSetupper.initializeMods(tbIF, ddmIF)
for dummyID, tmpList in tmpListList:
for taskSpec, cloudName, inputChunk in tmpList:
jobBroker = JobBroker(taskSpec.vo, taskSpec.prodSourceLabel)
tmpStat = jobBroker.initializeMods(ddmIF.getInterface(vo), tbIF)
splitter = JobSplitter()
gen = JobGeneratorThread(None, threadPool, tbIF, ddmIF, siteMapper,
False, taskSetupper, None, None, 'dummy',
None, None)
taskParamMap = None
if taskSpec.useLimitedSites():
tmpStat, taskParamMap = gen.readTaskParams(taskSpec, taskParamMap,
tmpLog)
tmpStat, inputChunk = jobBroker.doBrokerage(taskSpec, cloudName,
inputChunk, taskParamMap)
#tmpStat,subChunks = splitter.doSplit(taskSpec,inputChunk,siteMapper)
taskBufferIF = JediTaskBufferInterface()
taskBufferIF.setupInterface()
"""
from pandajedi.jediconfig import jedi_config
from pandajedi.jedicore import JediTaskBuffer
taskBufferIF = JediTaskBuffer.JediTaskBuffer(None)
from pandajedi.jediddm.DDMInterface import DDMInterface
ddmIF = DDMInterface()
ddmIF.setupInterface()
ddmIF = ddmIF.getInterface('atlas')
datasetName = 'valid1.159025.ParticleGenerator_gamma_E100.recon.AOD.e3099_s2082_r6012_tid04635343_00'
lostFiles = set(
['AOD.04635343._000012.pool.root.1', 'AOD.04635343._000015.pool.root.1'])
def resetStatusForLostFileRecovery(datasetName, lostFiles):
# get jeditaskid
varMap = {}
varMap[':type1'] = 'log'
varMap[':type2'] = 'output'
varMap[':name1'] = datasetName
varMap[':name2'] = datasetName.split(':')[-1]
sqlGI = 'SELECT jediTaskID,datasetID FROM {0}.JEDI_Datasets '.format(
jedi_config.db.schemaJEDI)
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface
tbIF = JediTaskBufferInterface()
tbIF.setupInterface()
siteMapper = tbIF.getSiteMapper()
from pandajedi.jediddm.DDMInterface import DDMInterface
ddmIF = DDMInterface()
ddmIF.setupInterface()
atlasIF = ddmIF.getInterface('atlas')
from pandajedi.jedibrokerage.AtlasProdJobBroker import AtlasProdJobBroker
br = AtlasProdJobBroker(atlasIF,tbIF,siteMapper)
st,taskSpec = tbIF.getTaskWithID_JEDI(1)
print br.doBrokerage(taskSpec,'US',
#{'mc12_8TeV.167751.Sherpa_CT10_ZeeMassiveCBPt0_CVetoBVeto.recon.AOD.e1585_a159_a171_tid01225039_00':['AOD.01225039._002187.pool.root.1','AOD.01225039._002627.pool.root.1']})
{'data11_7TeV.00190256.physics_JetTauEtmiss.merge.NTUP_SMWZ.f408_m1007_p1035_tid01048572_00':['NTUP_SMWZ.01048572._000522.root.1']})