class Tier0PluginTest(unittest.TestCase):
def setUp(self):
"""
_setUp_
Setup the test environment
"""
self.testInit = TestInit(__file__)
self.testInit.setDatabaseConnection()
self.testInit.setSchema(["WMCore.WMBS"])
self.wmstatsCouchDB = 'wmstats_plugin_t'
self.testInit.setupCouch(self.wmstatsCouchDB, 'WMStats')
self.testDir = self.testInit.generateWorkDir()
self.wmstatsWriter = WMStatsWriter(os.environ['COUCHURL'],
self.wmstatsCouchDB)
self.stateMap = {}
self.orderedStates = []
self.plugin = None
return
def tearDown(self):
"""
_tearDown_
Clear databases and delete files
"""
self.testInit.tearDownCouch()
self.testInit.clearDatabase()
self.testInit.delWorkDir()
return
def setupRepackWorkflow(self):
"""
_setupRepackWorkflow_
Populate WMBS with a repack-like workflow,
every subscription must be unfinished at first
"""
workflowName = 'Repack_Run481516_StreamZ'
mergeTasks = [
'RepackMergewrite_QuadElectron_RAW',
'RepackMergewrite_TriPhoton_RAW',
'RepackMergewrite_SingleNeutrino_RAW'
]
self.stateMap = {'Merge': [], 'Processing Done': []}
self.orderedStates = ['Merge', 'Processing Done']
# Populate WMStats
self.wmstatsWriter.insertGenericRequest({'_id': workflowName})
self.wmstatsWriter.updateRequestStatus(workflowName, 'Closed')
# Create a wmspec in disk
workload = newWorkload(workflowName)
repackTask = workload.newTask('Repack')
for task in mergeTasks:
repackTask.addTask(task)
repackTask.addTask('RepackCleanupUnmergedwrite_QuadElectron_RAW')
specPath = os.path.join(self.testDir, 'Repack.pkl')
workload.save(specPath)
# Populate WMBS
topFileset = Fileset(name='TestStreamerFileset')
topFileset.create()
options = {
'spec': specPath,
'owner': 'ItsAMeMario',
'name': workflowName,
'wfType': 'tier0'
}
topLevelWorkflow = Workflow(task='/%s/Repack' % workflowName,
**options)
topLevelWorkflow.create()
topLevelSub = Subscription(topFileset, topLevelWorkflow)
topLevelSub.create()
self.stateMap['Merge'].append(topFileset)
for task in mergeTasks:
mergeWorkflow = Workflow(task='/%s/Repack/%s' %
(workflowName, task),
**options)
mergeWorkflow.create()
unmergedFileset = Fileset(name='TestUnmergedFileset%s' % task)
unmergedFileset.create()
mergeSub = Subscription(unmergedFileset, mergeWorkflow)
mergeSub.create()
self.stateMap['Processing Done'].append(unmergedFileset)
cleanupWorkflow = Workflow(
task=
'/Repack_Run481516_StreamZ/Repack/RepackCleanupUnmergedwrite_QuadElectron_RAW',
**options)
cleanupWorkflow.create()
unmergedFileset = Fileset(name='TestUnmergedFilesetToCleanup')
unmergedFileset.create()
cleanupSub = Subscription(unmergedFileset, cleanupWorkflow)
cleanupSub.create()
return
def setupExpressWorkflow(self):
"""
_setupExpressWorkflow_
Populate WMBS with a express-like workflow,
every subscription must be unfinished at first
"""
workflowName = 'Express_Run481516_StreamZFast'
secondLevelTasks = [
'ExpressMergewrite_StreamZFast_DQM',
'ExpressMergewrite_ExpressPhysics_FEVT',
'ExpressAlcaSkimwrite_StreamZFast_ALCARECO',
'ExpressCleanupUnmergedwrite_StreamZFast_DQM',
'ExpressCleanupUnmergedwrite_ExpressPhysics_FEVT',
'ExpressCleanupUnmergedwrite_StreamZFast_ALCARECO'
]
alcaHarvestTask = 'ExpressAlcaSkimwrite_StreamZFast_ALCARECOAlcaHarvestALCARECOStreamPromptCalibProd'
dqmHarvestTask = 'ExpressMergewrite_StreamZFast_DQMDQMHarvestMerged'
self.stateMap = {'Merge': [], 'Harvesting': [], 'Processing Done': []}
self.orderedStates = ['Merge', 'Harvesting', 'Processing Done']
# Populate WMStats
self.wmstatsWriter.insertGenericRequest({'_id': workflowName})
self.wmstatsWriter.updateRequestStatus(workflowName, 'Closed')
# Create a wmspec in disk
workload = newWorkload(workflowName)
expressTask = workload.newTask('Express')
for task in secondLevelTasks:
secondLevelTask = expressTask.addTask(task)
if task == 'ExpressAlcaSkimwrite_StreamZFast_ALCARECO':
secondLevelTask.addTask(alcaHarvestTask)
elif task == 'ExpressMergewrite_StreamZFast_DQM':
secondLevelTask.addTask(dqmHarvestTask)
specPath = os.path.join(self.testDir, 'Express.pkl')
workload.save(specPath)
# Populate WMBS
sharedFileset = Fileset(name='TestFileset')
sharedFileset.create()
sharedFileset.markOpen(False)
options = {
'spec': specPath,
'owner': 'ItsAMeMario',
'name': workflowName,
'wfType': 'tier0'
}
topLevelWorkflow = Workflow(task='/%s/Express' % workflowName,
**options)
topLevelWorkflow.create()
topLevelSub = Subscription(sharedFileset, topLevelWorkflow)
topLevelSub.create()
self.stateMap['Merge'].append(topLevelSub)
for task in filter(lambda x: not x.count('CleanupUnmerged'),
secondLevelTasks):
secondLevelWorkflow = Workflow(task='/%s/Express/%s' %
(workflowName, task),
**options)
secondLevelWorkflow.create()
mergeSub = Subscription(sharedFileset, secondLevelWorkflow)
mergeSub.create()
self.stateMap['Harvesting'].append(mergeSub)
for (parent, child) in [
('ExpressAlcaSkimwrite_StreamZFast_ALCARECO', alcaHarvestTask),
('ExpressMergewrite_StreamZFast_DQM', dqmHarvestTask)
]:
harvestingWorkflow = Workflow(task='/%s/Express/%s/%s' %
(workflowName, parent, child),
**options)
harvestingWorkflow.create()
harvestingSub = Subscription(sharedFileset, harvestingWorkflow)
harvestingSub.create()
self.stateMap['Processing Done'].append(harvestingSub)
return
def setupPromptRecoWorkflow(self):
"""
_setupPromptRecoWorkflow_
Populate WMBS with a real PromptReco workflow,
every subscription must be unfinished at first
"""
# Populate disk and WMBS
testArguments = getTestArguments()
workflowName = 'PromptReco_Run195360_Cosmics'
workload = promptrecoWorkload(workflowName, testArguments)
wmbsHelper = WMBSHelper(workload,
'Reco',
'SomeBlock',
cachepath=self.testDir)
wmbsHelper.createTopLevelFileset()
wmbsHelper.createSubscription(wmbsHelper.topLevelTask,
wmbsHelper.topLevelFileset)
self.stateMap = {
'AlcaSkim': [],
'Merge': [],
'Harvesting': [],
'Processing Done': []
}
self.orderedStates = [
'AlcaSkim', 'Merge', 'Harvesting', 'Processing Done'
]
# Populate WMStats
self.wmstatsWriter.insertGenericRequest({'_id': workflowName})
self.wmstatsWriter.updateRequestStatus(workflowName, 'Closed')
topLevelTask = '/%s/Reco' % workflowName
alcaSkimTask = '%s/AlcaSkim' % topLevelTask
mergeTasks = [
'%s/AlcaSkim/AlcaSkimMergeALCARECOStreamHcalCalHOCosmics',
'%s/AlcaSkim/AlcaSkimMergeALCARECOStreamTkAlCosmics0T',
'%s/AlcaSkim/AlcaSkimMergeALCARECOStreamMuAlGlobalCosmics',
'%s/RecoMergewrite_AOD', '%s/RecoMergewrite_DQM',
'%s/RecoMergewrite_RECO'
]
harvestingTask = '%s/RecoMergewrite_DQM/RecoMergewrite_DQMDQMHarvestMerged' % topLevelTask
self.stateMap['AlcaSkim'].append(wmbsHelper.topLevelSubscription)
alcaSkimWorkflow = Workflow(name=workflowName, task=alcaSkimTask)
alcaSkimWorkflow.load()
alcarecoFileset = Fileset(
name='/PromptReco_Run195360_Cosmics/Reco/unmerged-write_ALCARECO')
alcarecoFileset.load()
alcaSkimSub = Subscription(alcarecoFileset, alcaSkimWorkflow)
alcaSkimSub.load()
self.stateMap['Merge'].append(alcaSkimSub)
for task in mergeTasks:
mergeTask = task % topLevelTask
mergeWorkflow = Workflow(name=workflowName, task=mergeTask)
mergeWorkflow.load()
if 'AlcaSkim' in mergeTask:
stream = mergeTask.split('/')[-1][13:]
unmergedFileset = Fileset(name='%s/unmerged-%s' %
(alcaSkimTask, stream))
unmergedFileset.load()
else:
dataTier = mergeTask.split('/')[-1].split('_')[-1]
unmergedFileset = Fileset(name='%s/unmerged-write_%s' %
(topLevelTask, dataTier))
unmergedFileset.load()
mergeSub = Subscription(unmergedFileset, mergeWorkflow)
mergeSub.load()
self.stateMap['Harvesting'].append(mergeSub)
harvestingWorkflow = Workflow(name=workflowName, task=harvestingTask)
harvestingWorkflow.load()
harvestingFileset = Fileset(
name=
'/PromptReco_Run195360_Cosmics/Reco/RecoMergewrite_DQM/merged-Merged'
)
harvestingFileset.load()
harvestingSub = Subscription(harvestingFileset, harvestingWorkflow)
harvestingSub.load()
self.stateMap['Processing Done'].append(harvestingSub)
return
def verifyStateTransitions(self,
transitionMethod='markFinished',
transitionTrigger=True):
"""
_verifyStateTransitions_
Utility method which goes through the list of states in self.orderedStates and
finishes the tasks that demand a state transition in each step. This according
to the defined transition method and trigger.
It verifies that the request document in WMStats is moving according to the transitions
"""
for idx in range(0, len(self.orderedStates) * 2):
nextState = self.orderedStates[idx / 2]
if (idx / 2) == 0:
currentState = 'Closed'
else:
currentState = self.orderedStates[idx / 2 - 1]
if idx % 2 == 0:
for transitionObject in self.stateMap[nextState][:-1]:
method = getattr(transitionObject, transitionMethod)
method(transitionTrigger)
self.plugin([], self.wmstatsWriter, self.wmstatsWriter)
currentStateWorkflows = self.wmstatsWriter.workflowsByStatus(
[currentState], stale=False)
nextStateWorkflows = self.wmstatsWriter.workflowsByStatus(
[nextState], stale=False)
self.assertEqual(
len(currentStateWorkflows), 1,
'Workflow moved incorrectly from %s' % currentState)
self.assertEqual(
len(nextStateWorkflows), 0,
'Workflow moved incorrectly to %s' % nextState)
else:
transitionObject = self.stateMap[nextState][-1]
method = getattr(transitionObject, transitionMethod)
method(transitionTrigger)
self.plugin([], self.wmstatsWriter, self.wmstatsWriter)
currentStateWorkflows = self.wmstatsWriter.workflowsByStatus(
[currentState], stale=False)
nextStateWorkflows = self.wmstatsWriter.workflowsByStatus(
[nextState], stale=False)
self.assertEqual(
len(currentStateWorkflows), 0,
'Workflow did not move correctly from %s' % currentState)
self.assertEqual(
len(nextStateWorkflows), 1,
'Workflow did not move correctly to %s' % nextState)
return
def testA_RepackStates(self):
"""
_testA_RepackStates_
Setup an environment with a Repack workflow
and traverse through the different states.
Check that the transitions are sane.
"""
# Set the environment
self.setupRepackWorkflow()
self.plugin = Tier0Plugin()
# Verify the transitions
self.verifyStateTransitions('markOpen', False)
return
def testB_ExpressStates(self):
"""
_testB_ExpressStates_
Setup an environment with a Express workflow
and traverse through the different states.
Check that the transitions are sane.
"""
# Set the environment
self.setupExpressWorkflow()
self.plugin = Tier0Plugin()
# Verify the transitions
self.verifyStateTransitions()
return
def testC_PromptRecoStates(self):
"""
_testC_PromptRecoStates_
Setup an environment with a PromptReco workflow
and traverse through the different states.
Check that the transitions are sane.
"""
# Set the environment
self.setupPromptRecoWorkflow()
self.plugin = Tier0Plugin()
# Verify the transitions
self.verifyStateTransitions()
return
class Tier0PluginTest(unittest.TestCase):
def setUp(self):
"""
_setUp_
Setup the test environment
"""
self.testInit = TestInit(__file__)
self.testInit.setDatabaseConnection()
self.testInit.setSchema(["WMCore.WMBS"])
self.wmstatsCouchDB = 'wmstats_plugin_t'
self.testInit.setupCouch(self.wmstatsCouchDB, 'WMStats')
self.testDir = self.testInit.generateWorkDir()
self.wmstatsWriter = WMStatsWriter(os.environ['COUCHURL'], self.wmstatsCouchDB)
self.stateMap = {}
self.orderedStates = []
self.plugin = None
return
def tearDown(self):
"""
_tearDown_
Clear databases and delete files
"""
self.testInit.tearDownCouch()
self.testInit.clearDatabase()
self.testInit.delWorkDir()
return
def setupRepackWorkflow(self):
"""
_setupRepackWorkflow_
Populate WMBS with a repack-like workflow,
every subscription must be unfinished at first
"""
workflowName = 'Repack_Run481516_StreamZ'
mergeTasks = ['RepackMergewrite_QuadElectron_RAW', 'RepackMergewrite_TriPhoton_RAW',
'RepackMergewrite_SingleNeutrino_RAW']
self.stateMap = {'Merge' : [],
'Processing Done' : []}
self.orderedStates = ['Merge', 'Processing Done']
# Populate WMStats
self.wmstatsWriter.insertGenericRequest({'_id' : workflowName})
self.wmstatsWriter.updateRequestStatus(workflowName, 'Closed')
# Create a wmspec in disk
workload = newWorkload(workflowName)
repackTask = workload.newTask('Repack')
for task in mergeTasks:
repackTask.addTask(task)
repackTask.addTask('RepackCleanupUnmergedwrite_QuadElectron_RAW')
specPath = os.path.join(self.testDir, 'Repack.pkl')
workload.save(specPath)
# Populate WMBS
topFileset = Fileset(name = 'TestStreamerFileset')
topFileset.create()
options = {'spec' : specPath, 'owner' : 'ItsAMeMario',
'name' : workflowName, 'wfType' : 'tier0'}
topLevelWorkflow = Workflow(task = '/%s/Repack' % workflowName,
**options)
topLevelWorkflow.create()
topLevelSub = Subscription(topFileset, topLevelWorkflow)
topLevelSub.create()
self.stateMap['Merge'].append(topFileset)
for task in mergeTasks:
mergeWorkflow = Workflow(task = '/%s/Repack/%s' % (workflowName, task), **options)
mergeWorkflow.create()
unmergedFileset = Fileset(name = 'TestUnmergedFileset%s' % task)
unmergedFileset.create()
mergeSub = Subscription(unmergedFileset, mergeWorkflow)
mergeSub.create()
self.stateMap['Processing Done'].append(unmergedFileset)
cleanupWorkflow = Workflow(task = '/Repack_Run481516_StreamZ/Repack/RepackCleanupUnmergedwrite_QuadElectron_RAW',
**options)
cleanupWorkflow.create()
unmergedFileset = Fileset(name = 'TestUnmergedFilesetToCleanup')
unmergedFileset.create()
cleanupSub = Subscription(unmergedFileset, cleanupWorkflow)
cleanupSub.create()
return
def setupExpressWorkflow(self):
"""
_setupExpressWorkflow_
Populate WMBS with a express-like workflow,
every subscription must be unfinished at first
"""
workflowName = 'Express_Run481516_StreamZFast'
secondLevelTasks = ['ExpressMergewrite_StreamZFast_DQM', 'ExpressMergewrite_ExpressPhysics_FEVT',
'ExpressAlcaSkimwrite_StreamZFast_ALCARECO', 'ExpressCleanupUnmergedwrite_StreamZFast_DQM',
'ExpressCleanupUnmergedwrite_ExpressPhysics_FEVT', 'ExpressCleanupUnmergedwrite_StreamZFast_ALCARECO']
alcaHarvestTask = 'ExpressAlcaSkimwrite_StreamZFast_ALCARECOAlcaHarvestALCARECOStreamPromptCalibProd'
dqmHarvestTask = 'ExpressMergewrite_StreamZFast_DQMEndOfRunDQMHarvestMerged'
self.stateMap = {'Merge' : [],
'Harvesting' : [],
'Processing Done' : []}
self.orderedStates = ['Merge', 'Harvesting', 'Processing Done']
# Populate WMStats
self.wmstatsWriter.insertGenericRequest({'_id' : workflowName})
self.wmstatsWriter.updateRequestStatus(workflowName, 'Closed')
# Create a wmspec in disk
workload = newWorkload(workflowName)
expressTask = workload.newTask('Express')
for task in secondLevelTasks:
secondLevelTask = expressTask.addTask(task)
if task == 'ExpressAlcaSkimwrite_StreamZFast_ALCARECO':
secondLevelTask.addTask(alcaHarvestTask)
elif task == 'ExpressMergewrite_StreamZFast_DQM':
secondLevelTask.addTask(dqmHarvestTask)
specPath = os.path.join(self.testDir, 'Express.pkl')
workload.save(specPath)
# Populate WMBS
sharedFileset = Fileset(name = 'TestFileset')
sharedFileset.create()
sharedFileset.markOpen(False)
options = {'spec' : specPath, 'owner' : 'ItsAMeMario',
'name' : workflowName, 'wfType' : 'tier0'}
topLevelWorkflow = Workflow(task = '/%s/Express' % workflowName,
**options)
topLevelWorkflow.create()
topLevelSub = Subscription(sharedFileset, topLevelWorkflow)
topLevelSub.create()
self.stateMap['Merge'].append(topLevelSub)
for task in filter(lambda x : not x.count('CleanupUnmerged'), secondLevelTasks):
secondLevelWorkflow = Workflow(task = '/%s/Express/%s' % (workflowName, task), **options)
secondLevelWorkflow.create()
mergeSub = Subscription(sharedFileset, secondLevelWorkflow)
mergeSub.create()
self.stateMap['Harvesting'].append(mergeSub)
for (parent, child) in [('ExpressAlcaSkimwrite_StreamZFast_ALCARECO', alcaHarvestTask),
('ExpressMergewrite_StreamZFast_DQM', dqmHarvestTask)]:
harvestingWorkflow = Workflow(task = '/%s/Express/%s/%s' % (workflowName, parent, child),
**options)
harvestingWorkflow.create()
harvestingSub = Subscription(sharedFileset, harvestingWorkflow)
harvestingSub.create()
self.stateMap['Processing Done'].append(harvestingSub)
return
def setupPromptRecoWorkflow(self):
"""
_setupPromptRecoWorkflow_
Populate WMBS with a real PromptReco workflow,
every subscription must be unfinished at first
"""
# Populate disk and WMBS
testArguments = getTestArguments()
workflowName = 'PromptReco_Run195360_Cosmics'
workload = promptrecoWorkload(workflowName, testArguments)
wmbsHelper = WMBSHelper(workload, 'Reco', 'SomeBlock', cachepath = self.testDir)
wmbsHelper.createTopLevelFileset()
wmbsHelper._createSubscriptionsInWMBS(wmbsHelper.topLevelTask, wmbsHelper.topLevelFileset)
self.stateMap = {'AlcaSkim' : [],
'Merge' : [],
'Harvesting' : [],
'Processing Done' : []}
self.orderedStates = ['AlcaSkim', 'Merge', 'Harvesting', 'Processing Done']
# Populate WMStats
self.wmstatsWriter.insertGenericRequest({'_id' : workflowName})
self.wmstatsWriter.updateRequestStatus(workflowName, 'Closed')
topLevelTask = '/%s/Reco' % workflowName
alcaSkimTask = '%s/AlcaSkim' % topLevelTask
mergeTasks = ['%s/AlcaSkim/AlcaSkimMergeALCARECOStreamHcalCalHOCosmics',
'%s/AlcaSkim/AlcaSkimMergeALCARECOStreamTkAlCosmics0T',
'%s/AlcaSkim/AlcaSkimMergeALCARECOStreamMuAlGlobalCosmics',
'%s/RecoMergewrite_AOD',
'%s/RecoMergewrite_DQM',
'%s/RecoMergewrite_RECO']
harvestingTask = '%s/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged' % topLevelTask
self.stateMap['AlcaSkim'].append(wmbsHelper.topLevelSubscription)
alcaSkimWorkflow = Workflow(name = workflowName, task = alcaSkimTask)
alcaSkimWorkflow.load()
alcarecoFileset = Fileset(name = '/PromptReco_Run195360_Cosmics/Reco/unmerged-write_ALCARECO')
alcarecoFileset.load()
alcaSkimSub = Subscription(alcarecoFileset, alcaSkimWorkflow)
alcaSkimSub.load()
self.stateMap['Merge'].append(alcaSkimSub)
for task in mergeTasks:
mergeTask = task % topLevelTask
mergeWorkflow = Workflow(name = workflowName, task = mergeTask)
mergeWorkflow.load()
if 'AlcaSkim' in mergeTask:
stream = mergeTask.split('/')[-1][13:]
unmergedFileset = Fileset(name = '%s/unmerged-%s' % (alcaSkimTask, stream))
unmergedFileset.load()
else:
dataTier = mergeTask.split('/')[-1].split('_')[-1]
unmergedFileset = Fileset(name = '%s/unmerged-write_%s' % (topLevelTask, dataTier))
unmergedFileset.load()
mergeSub = Subscription(unmergedFileset, mergeWorkflow)
mergeSub.load()
self.stateMap['Harvesting'].append(mergeSub)
harvestingWorkflow = Workflow(name = workflowName, task = harvestingTask)
harvestingWorkflow.load()
harvestingFileset = Fileset(name = '/PromptReco_Run195360_Cosmics/Reco/RecoMergewrite_DQM/merged-Merged')
harvestingFileset.load()
harvestingSub = Subscription(harvestingFileset, harvestingWorkflow)
harvestingSub.load()
self.stateMap['Processing Done'].append(harvestingSub)
return
def verifyStateTransitions(self, transitionMethod = 'markFinished', transitionTrigger = True):
"""
_verifyStateTransitions_
Utility method which goes through the list of states in self.orderedStates and
finishes the tasks that demand a state transition in each step. This according
to the defined transition method and trigger.
It verifies that the request document in WMStats is moving according to the transitions
"""
for idx in range(0, len(self.orderedStates) * 2):
nextState = self.orderedStates[idx / 2]
if (idx / 2) == 0:
currentState = 'Closed'
else:
currentState = self.orderedStates[idx / 2 - 1]
if idx % 2 == 0:
for transitionObject in self.stateMap[nextState][:-1]:
method = getattr(transitionObject, transitionMethod)
method(transitionTrigger)
self.plugin([], self.wmstatsWriter, self.wmstatsWriter)
currentStateWorkflows = self.wmstatsWriter.workflowsByStatus([currentState], stale = False)
nextStateWorkflows = self.wmstatsWriter.workflowsByStatus([nextState], stale = False)
self.assertEqual(len(currentStateWorkflows), 1, 'Workflow moved incorrectly from %s' % currentState)
self.assertEqual(len(nextStateWorkflows), 0, 'Workflow moved incorrectly to %s' % nextState)
else:
transitionObject = self.stateMap[nextState][-1]
method = getattr(transitionObject, transitionMethod)
method(transitionTrigger)
self.plugin([], self.wmstatsWriter, self.wmstatsWriter)
currentStateWorkflows = self.wmstatsWriter.workflowsByStatus([currentState], stale = False)
nextStateWorkflows = self.wmstatsWriter.workflowsByStatus([nextState], stale = False)
self.assertEqual(len(currentStateWorkflows), 0, 'Workflow did not move correctly from %s' % currentState)
self.assertEqual(len(nextStateWorkflows), 1, 'Workflow did not move correctly to %s' % nextState)
return
def testA_RepackStates(self):
"""
_testA_RepackStates_
Setup an environment with a Repack workflow
and traverse through the different states.
Check that the transitions are sane.
"""
# Set the environment
self.setupRepackWorkflow()
self.plugin = Tier0Plugin()
# Verify the transitions
self.verifyStateTransitions('markOpen', False)
return
def testB_ExpressStates(self):
"""
_testB_ExpressStates_
Setup an environment with a Express workflow
and traverse through the different states.
Check that the transitions are sane.
"""
# Set the environment
self.setupExpressWorkflow()
self.plugin = Tier0Plugin()
# Verify the transitions
self.verifyStateTransitions()
return
def testC_PromptRecoStates(self):
"""
_testC_PromptRecoStates_
Setup an environment with a PromptReco workflow
and traverse through the different states.
Check that the transitions are sane.
"""
# Set the environment
self.setupPromptRecoWorkflow()
self.plugin = Tier0Plugin()
# Verify the transitions
self.verifyStateTransitions()
return
class Tier0FeederPoller(BaseWorkerThread):
def __init__(self, config):
"""
_init_
"""
BaseWorkerThread.__init__(self)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package = "T0.WMBS",
logger = logging,
dbinterface = myThread.dbi)
self.tier0ConfigFile = config.Tier0Feeder.tier0ConfigFile
self.specDirectory = config.Tier0Feeder.specDirectory
self.dropboxuser = config.Tier0Feeder.dropboxuser
self.dropboxpass = config.Tier0Feeder.dropboxpass
self.transferSystemBaseDir = getattr(config.Tier0Feeder, "transferSystemBaseDir", None)
if self.transferSystemBaseDir != None:
if not os.path.exists(self.transferSystemBaseDir):
self.transferSystemBaseDir = None
self.dqmUploadProxy = config.WMBSService.proxy
self.localSummaryCouchDB = WMStatsWriter(config.AnalyticsDataCollector.localWMStatsURL)
hltConfConnectUrl = config.HLTConfDatabase.connectUrl
dbFactoryHltConf = DBFactory(logging, dburl = hltConfConnectUrl, options = {})
dbInterfaceHltConf = dbFactoryHltConf.connect()
daoFactoryHltConf = DAOFactory(package = "T0.WMBS",
logger = logging,
dbinterface = dbInterfaceHltConf)
self.getHLTConfigDAO = daoFactoryHltConf(classname = "RunConfig.GetHLTConfig")
storageManagerConnectUrl = config.StorageManagerDatabase.connectUrl
dbFactoryStorageManager = DBFactory(logging, dburl = storageManagerConnectUrl, options = {})
self.dbInterfaceStorageManager = dbFactoryStorageManager.connect()
self.getExpressReadyRunsDAO = None
if hasattr(config, "PopConLogDatabase"):
popConLogConnectUrl = getattr(config.PopConLogDatabase, "connectUrl", None)
if popConLogConnectUrl != None:
dbFactoryPopConLog = DBFactory(logging, dburl = popConLogConnectUrl, options = {})
dbInterfacePopConLog = dbFactoryPopConLog.connect()
daoFactoryPopConLog = DAOFactory(package = "T0.WMBS",
logger = logging,
dbinterface = dbInterfacePopConLog)
self.getExpressReadyRunsDAO = daoFactoryPopConLog(classname = "Tier0Feeder.GetExpressReadyRuns")
return
def algorithm(self, parameters = None):
"""
_algorithm_
"""
logging.debug("Running Tier0Feeder algorithm...")
myThread = threading.currentThread()
findNewRunsDAO = self.daoFactory(classname = "Tier0Feeder.FindNewRuns")
findNewRunStreamsDAO = self.daoFactory(classname = "Tier0Feeder.FindNewRunStreams")
findNewExpressRunsDAO = self.daoFactory(classname = "Tier0Feeder.FindNewExpressRuns")
releaseExpressDAO = self.daoFactory(classname = "Tier0Feeder.ReleaseExpress")
feedStreamersDAO = self.daoFactory(classname = "Tier0Feeder.FeedStreamers")
markRepackInjectedDAO = self.daoFactory(classname = "Tier0Feeder.MarkRepackInjected")
tier0Config = None
try:
tier0Config = loadConfigurationFile(self.tier0ConfigFile)
except:
# usually happens when there are syntax errors in the configuration
logging.exception("Cannot load Tier0 configuration file, not configuring new runs and run/streams")
# only configure new runs and run/streams if we have a valid Tier0 configuration
if tier0Config != None:
#
# find new runs, setup global run settings and stream/dataset/trigger mapping
#
runHltkeys = findNewRunsDAO.execute(transaction = False)
for run, hltkey in sorted(runHltkeys.items()):
hltConfig = None
# local runs have no hltkey and are configured differently
if hltkey != None:
# retrieve HLT configuration and make sure it's usable
try:
hltConfig = self.getHLTConfigDAO.execute(hltkey, transaction = False)
if hltConfig['process'] == None or len(hltConfig['mapping']) == 0:
raise RuntimeError, "HLTConfDB query returned no process or mapping"
except:
logging.exception("Can't retrieve hltkey %s for run %d" % (hltkey, run))
continue
try:
RunConfigAPI.configureRun(tier0Config, run, hltConfig)
except:
logging.exception("Can't configure for run %d" % (run))
#
# find unconfigured run/stream with data
# populate RunConfig, setup workflows/filesets/subscriptions
#
runStreams = findNewRunStreamsDAO.execute(transaction = False)
for run in sorted(runStreams.keys()):
for stream in sorted(runStreams[run]):
try:
RunConfigAPI.configureRunStream(tier0Config,
run, stream,
self.specDirectory,
self.dqmUploadProxy)
except:
logging.exception("Can't configure for run %d and stream %s" % (run, stream))
#
# end runs which are active and have ended according to the EoR StorageManager records
#
RunLumiCloseoutAPI.endRuns(self.dbInterfaceStorageManager)
#
# release runs for Express
#
runs = findNewExpressRunsDAO.execute(transaction = False)
if len(runs) > 0:
binds = []
for run in runs:
binds.append( { 'RUN' : run } )
if self.getExpressReadyRunsDAO != None:
runs = self.getExpressReadyRunsDAO.execute(binds = binds, transaction = False)
if len(runs) > 0:
binds = []
for run in runs:
binds.append( { 'RUN' : run } )
releaseExpressDAO.execute(binds = binds, transaction = False)
#
# release runs for PromptReco
#
RunConfigAPI.releasePromptReco(tier0Config,
self.specDirectory,
self.dqmUploadProxy)
#
# check if all datasets for a stream had their PromptReco released
# then mark the repack workflow as injected (if we don't wait, the
# task archiver will cleanup too early)
#
markRepackInjectedDAO.execute(transaction = False)
#
# close stream/lumis for run/streams that are active (fileset exists and open)
#
RunLumiCloseoutAPI.closeLumiSections(self.dbInterfaceStorageManager)
#
# feed new data into exisiting filesets
#
try:
myThread.transaction.begin()
feedStreamersDAO.execute(conn = myThread.transaction.conn, transaction = True)
except:
logging.exception("Can't feed data, bailing out...")
raise
else:
myThread.transaction.commit()
#
# run ended and run/stream fileset open
# => check for complete lumi_closed record, all lumis finally closed and all data feed
# => if all conditions satisfied, close the run/stream fileset
#
RunLumiCloseoutAPI.closeRunStreamFilesets()
#
# check and delete active split lumis
#
RunLumiCloseoutAPI.checkActiveSplitLumis()
#
# insert workflows into CouchDB for monitoring
#
self.feedCouchMonitoring()
#
# Update Couch when Repack and Express have closed input filesets (analog to old T0 closeout)
#
self.closeOutRealTimeWorkflows()
#
# send repacked notifications to StorageManager
#
if self.transferSystemBaseDir != None:
self.notifyStorageManager()
#
# upload PCL conditions to DropBox
#
ConditionUploadAPI.uploadConditions(self.dropboxuser, self.dropboxpass)
return
def feedCouchMonitoring(self):
"""
_feedCouchMonitoring_
check for workflows that haven't been uploaded to Couch for monitoring yet
"""
getStreamerWorkflowsForMonitoringDAO = self.daoFactory(classname = "Tier0Feeder.GetStreamerWorkflowsForMonitoring")
getPromptRecoWorkflowsForMonitoringDAO = self.daoFactory(classname = "Tier0Feeder.GetPromptRecoWorkflowsForMonitoring")
markTrackedWorkflowMonitoringDAO = self.daoFactory(classname = "Tier0Feeder.MarkTrackedWorkflowMonitoring")
workflows = getStreamerWorkflowsForMonitoringDAO.execute()
workflows += getPromptRecoWorkflowsForMonitoringDAO.execute()
if len(workflows) == 0:
logging.debug("No workflows to publish to couch monitoring, doing nothing")
if workflows:
logging.debug(" Going to publish %d workflows" % len(workflows))
for (workflowId, run, workflowName) in workflows:
logging.info(" Publishing workflow %s to monitoring" % workflowName)
doc = {}
doc["_id"] = workflowName
doc["workflow"] = workflowName
doc["type"] = "tier0_request"
doc["run"] = run
response = self.localSummaryCouchDB.insertGenericRequest(doc)
if response == "OK" or "EXISTS":
logging.info(" Successfully uploaded request %s" % workflowName)
# Here we have to trust the insert, if it doesn't happen will be easy to spot on the logs
markTrackedWorkflowMonitoringDAO.execute(workflowId)
return
def closeOutRealTimeWorkflows(self):
"""
_closeOutRealTimeWorkflows_
Updates couch with the closeout status of Repack and Express
PromptReco should be closed out automatically
"""
getNotClosedOutWorkflowsDAO = self.daoFactory(classname = "Tier0Feeder.GetNotClosedOutWorkflows")
workflows = getNotClosedOutWorkflowsDAO.execute()
if len(workflows) == 0:
logging.debug("No workflows to publish to couch monitoring, doing nothing")
if workflows:
for workflow in workflows:
(workflowId, filesetId, filesetOpen, workflowName) = workflow
# find returns -1 if the string is not found
if workflowName.find('PromptReco') >= 0:
logging.debug("Closing out instantaneously PromptReco Workflow %s" % workflowName)
self.updateClosedState(workflowName, workflowId)
else :
# Check if fileset (which you already know) is closed or not
# FIXME: No better way to do it? what comes from the DAO is a string, casting bool or int doesn't help much.
# Works like that :
if filesetOpen == '0':
self.updateClosedState(workflowName, workflowId)
return
def updateClosedState(self, workflowName, workflowId):
"""
_updateClosedState_
Mark a workflow as Closed
"""
markCloseoutWorkflowMonitoringDAO = self.daoFactory(classname = "Tier0Feeder.MarkCloseoutWorkflowMonitoring")
response = self.localSummaryCouchDB.updateRequestStatus(workflowName, 'Closed')
if response == "OK" or "EXISTS":
logging.debug("Successfully closed workflow %s" % workflowName)
markCloseoutWorkflowMonitoringDAO.execute(workflowId)
return
def notifyStorageManager(self):
"""
_notifyStorageManager_
Find all finished streamers for closed all run/stream
Send the notification message to StorageManager
Update the streamer status to finished (deleted = 1)
"""
getFinishedStreamersDAO = self.daoFactory(classname = "SMNotification.GetFinishedStreamers")
markStreamersFinishedDAO = self.daoFactory(classname = "SMNotification.MarkStreamersFinished")
allFinishedStreamers = getFinishedStreamersDAO.execute(transaction = False)
num = len(allFinishedStreamers)/50
for finishedStreamers in [allFinishedStreamers[i::num] for i in range(num)]:
streamers = []
filenameParams = ""
for (id, lfn) in finishedStreamers:
streamers.append(id)
filenameParams += "-FILENAME %s " % os.path.basename(lfn)
logging.debug("Notifying transfer system about processed streamers")
p = subprocess.Popen("/bin/bash",stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
output, error = p.communicate("""
export T0_BASE_DIR=%s
export T0ROOT=${T0_BASE_DIR}/T0
export CONFIG=${T0_BASE_DIR}/Config/TransferSystem_CERN.cfg
export PERL5LIB=${T0ROOT}/perl_lib
${T0ROOT}/operations/sendRepackedStatus.pl --config $CONFIG %s
""" % (self.transferSystemBaseDir, filenameParams))
if len(error) > 0:
logging.error("ERROR: Could not notify transfer system about processed streamers")
logging.error("ERROR: %s" % error)
markStreamersFinishedDAO.execute(streamers, transaction = False)
return
def terminate(self, params):
"""
_terminate_
Kill the code after one final pass when called by the master thread.
"""
logging.debug("terminating immediately")
