class WMStatsTest(unittest.TestCase):
"""
"""
def setUp(self):
"""
_setUp_
"""
self.schema = []
self.couchApps = ["WMStats"]
self.testInit = TestInitCouchApp('WorkQueueServiceTest')
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = self.schema,
useDefault = False)
self.testInit.setupCouch('wmstats_t', *self.couchApps)
self.wmstatsWriter = WMStatsWriter(self.testInit.couchUrl, 'wmstats_t');
return
def tearDown(self):
"""
_tearDown_
Drop all the WMBS tables.
"""
self.testInit.tearDownCouch()
def testWMStatsWriter(self):
# test getWork
schema = generate_reqmgr_schema()
self.assertEquals(self.wmstatsWriter.insertRequest(schema[0]), 'OK', 'insert fail');
self.assertEquals(self.wmstatsWriter.updateRequestStatus(schema[0]['RequestName'], "failed"), 'OK', 'update fail')
self.assertEquals(self.wmstatsWriter.updateRequestStatus("not_exist_schema", "assigned"),
'ERROR: request not found - not_exist_schema')
self.assertEquals(self.wmstatsWriter.updateTeam(schema[0]['RequestName'], 'teamA'), 'OK', 'update fail')
self.assertEquals(self.wmstatsWriter.updateTeam("not_exist_schema", 'teamA'),
'ERROR: request not found - not_exist_schema')
totalStats = {'total_jobs': 100, 'input_events': 1000, 'input_lumis': 1234, 'input_num_files': 5}
self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'INSERTED', 'update fail')
self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'UPDATED', 'update fail')
self.assertEquals(self.wmstatsWriter.insertTotalStats("not_exist_schema", totalStats),
'ERROR: request not found - not_exist_schema')
spec1 = newWorkload(schema[0]['RequestName'])
production = spec1.newTask("Production")
production.setTaskType("Merge")
production.setSiteWhitelist(['TEST_SITE'])
self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec1), 'OK', 'update fail')
spec2 = newWorkload("not_exist_schema")
production = spec2.newTask("Production")
production.setTaskType("Merge")
self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec2),
'ERROR: request not found - not_exist_schema')
def changeRequestStatus(requestName, newState, priority=None, wmstatUrl=None):
"""
_changeRequestStatus_
Basic API to change a request to a new state, also
includes optional priority change for the request
- *requestName* : name of the request to be modified
- *newState* : name of the new status for the request
- *priority* : optional integer priority
Apparently when changing request state (on assignment page),
it's possible to change priority at one go. Hence the argument is here.
"""
# MySQL/Oracle
factory = DBConnect.getConnection()
reqId = getRequestID(factory, requestName)
changeRequestIDStatus(reqId, newState, priority)
# CouchDB
# have to first get information where the request Couch document is,
# extracting the information from reqmgr_request.workflow table field
reqData = factory(classname="Request.Get").execute(reqId)
# this would be something like this:
# http://localhost:5984/reqmgr_workload_cache/maxa_RequestString-OVERRIDE-ME_130306_205649_8066/spec
wfUrl = reqData['workflow']
# cut off /maxa_RequestString-OVERRIDE-ME_130306_205649_8066/spec
couchUrl = wfUrl.replace('/' + requestName + "/spec", '')
couchDbName = couchUrl[couchUrl.rfind('/') + 1:]
# cut off database name from the URL
url = couchUrl.replace('/' + couchDbName, '')
couchDb = Database(couchDbName, url)
fields = {"RequestStatus": newState}
couchDb.updateDocument(requestName,
"ReqMgr",
"updaterequest",
fields=fields,
useBody=True)
#TODO: should we make this mendatory?
if wmstatUrl:
wmstatSvc = WMStatsWriter(wmstatUrl)
wmstatSvc.updateRequestStatus(requestName, newState)
def changeRequestStatus(requestName, newState, priority = None, wmstatUrl = None):
"""
_changeRequestStatus_
Basic API to change a request to a new state, also
includes optional priority change for the request
- *requestName* : name of the request to be modified
- *newState* : name of the new status for the request
- *priority* : optional integer priority
"""
#TODO: should we make this mendatory?
if wmstatUrl:
wmstatSvc = WMStatsWriter(wmstatUrl)
wmstatSvc.updateRequestStatus(requestName, newState)
factory = DBConnect.getConnection()
reqId = getRequestID(factory, requestName)
changeRequestIDStatus(reqId, newState, priority)
return
def changeRequestStatus(requestName, newState, priority=None, wmstatUrl=None):
"""
_changeRequestStatus_
Basic API to change a request to a new state, also
includes optional priority change for the request
- *requestName* : name of the request to be modified
- *newState* : name of the new status for the request
- *priority* : optional integer priority
Apparently when changing request state (on assignment page),
it's possible to change priority at one go. Hence the argument is here.
"""
# MySQL/Oracle
factory = DBConnect.getConnection()
reqId = getRequestID(factory, requestName)
changeRequestIDStatus(reqId, newState, priority)
# CouchDB
# have to first get information where the request Couch document is,
# extracting the information from reqmgr_request.workflow table field
reqData = factory(classname="Request.Get").execute(reqId)
# this would be something like this:
# http://localhost:5984/reqmgr_workload_cache/maxa_RequestString-OVERRIDE-ME_130306_205649_8066/spec
wfUrl = reqData["workflow"]
# cut off /maxa_RequestString-OVERRIDE-ME_130306_205649_8066/spec
couchUrl = wfUrl.replace("/" + requestName + "/spec", "")
couchDbName = couchUrl[couchUrl.rfind("/") + 1 :]
# cut off database name from the URL
url = couchUrl.replace("/" + couchDbName, "")
couchDb = Database(couchDbName, url)
fields = {"RequestStatus": newState}
couchDb.updateDocument(requestName, "ReqMgr", "updaterequest", fields=fields, useBody=True)
# TODO: should we make this mendatory?
if wmstatUrl:
wmstatSvc = WMStatsWriter(wmstatUrl)
wmstatSvc.updateRequestStatus(requestName, newState)
def changeRequestStatus(requestName, newState, priority = None, wmstatUrl = None):
"""
_changeRequestStatus_
Basic API to change a request to a new state, also
includes optional priority change for the request
- *requestName* : name of the request to be modified
- *newState* : name of the new status for the request
- *priority* : optional integer priority
Apparently when changing request state (on assignment page),
it's possible to change priority at one go. Hence the argument is here.
"""
#TODO: should we make this mendatory?
if wmstatUrl:
wmstatSvc = WMStatsWriter(wmstatUrl)
wmstatSvc.updateRequestStatus(requestName, newState)
factory = DBConnect.getConnection()
reqId = getRequestID(factory, requestName)
changeRequestIDStatus(reqId, newState, priority)
return
def changeRequestStatus(requestName, newState, priority=None, wmstatUrl=None):
"""
_changeRequestStatus_
Basic API to change a request to a new state, also
includes optional priority change for the request
- *requestName* : name of the request to be modified
- *newState* : name of the new status for the request
- *priority* : optional integer priority
Apparently when changing request state (on assignment page),
it's possible to change priority at one go. Hence the argument is here.
"""
#TODO: should we make this mendatory?
if wmstatUrl:
wmstatSvc = WMStatsWriter(wmstatUrl)
wmstatSvc.updateRequestStatus(requestName, newState)
factory = DBConnect.getConnection()
reqId = getRequestID(factory, requestName)
changeRequestIDStatus(reqId, newState, priority)
return
def updateRequestStatus(couchURL, requestList, status):
ww = WMStatsWriter(couchURL)
for request in requestList:
ww.updateRequestStatus(request, status)
print("%s is udated to %s" % (request, status))
class Tier0PluginTest(unittest.TestCase):
def setUp(self):
"""
_setUp_
Setup the test environment
"""
self.testInit = TestInit(__file__)
self.testInit.setDatabaseConnection()
self.testInit.setSchema(["WMCore.WMBS"])
self.wmstatsCouchDB = 'wmstats_plugin_t'
self.testInit.setupCouch(self.wmstatsCouchDB, 'WMStats')
self.testDir = self.testInit.generateWorkDir()
self.wmstatsWriter = WMStatsWriter(os.environ['COUCHURL'], self.wmstatsCouchDB)
self.stateMap = {}
self.orderedStates = []
self.plugin = None
return
def tearDown(self):
"""
_tearDown_
Clear databases and delete files
"""
self.testInit.tearDownCouch()
self.testInit.clearDatabase()
self.testInit.delWorkDir()
return
def setupRepackWorkflow(self):
"""
_setupRepackWorkflow_
Populate WMBS with a repack-like workflow,
every subscription must be unfinished at first
"""
workflowName = 'Repack_Run481516_StreamZ'
mergeTasks = ['RepackMergewrite_QuadElectron_RAW', 'RepackMergewrite_TriPhoton_RAW',
'RepackMergewrite_SingleNeutrino_RAW']
self.stateMap = {'Merge' : [],
'Processing Done' : []}
self.orderedStates = ['Merge', 'Processing Done']
# Populate WMStats
self.wmstatsWriter.insertGenericRequest({'_id' : workflowName})
self.wmstatsWriter.updateRequestStatus(workflowName, 'Closed')
# Create a wmspec in disk
workload = newWorkload(workflowName)
repackTask = workload.newTask('Repack')
for task in mergeTasks:
repackTask.addTask(task)
repackTask.addTask('RepackCleanupUnmergedwrite_QuadElectron_RAW')
specPath = os.path.join(self.testDir, 'Repack.pkl')
workload.save(specPath)
# Populate WMBS
topFileset = Fileset(name = 'TestStreamerFileset')
topFileset.create()
options = {'spec' : specPath, 'owner' : 'ItsAMeMario',
'name' : workflowName, 'wfType' : 'tier0'}
topLevelWorkflow = Workflow(task = '/%s/Repack' % workflowName,
**options)
topLevelWorkflow.create()
topLevelSub = Subscription(topFileset, topLevelWorkflow)
topLevelSub.create()
self.stateMap['Merge'].append(topFileset)
for task in mergeTasks:
mergeWorkflow = Workflow(task = '/%s/Repack/%s' % (workflowName, task), **options)
mergeWorkflow.create()
unmergedFileset = Fileset(name = 'TestUnmergedFileset%s' % task)
unmergedFileset.create()
mergeSub = Subscription(unmergedFileset, mergeWorkflow)
mergeSub.create()
self.stateMap['Processing Done'].append(unmergedFileset)
cleanupWorkflow = Workflow(task = '/Repack_Run481516_StreamZ/Repack/RepackCleanupUnmergedwrite_QuadElectron_RAW',
**options)
cleanupWorkflow.create()
unmergedFileset = Fileset(name = 'TestUnmergedFilesetToCleanup')
unmergedFileset.create()
cleanupSub = Subscription(unmergedFileset, cleanupWorkflow)
cleanupSub.create()
return
def setupExpressWorkflow(self):
"""
_setupExpressWorkflow_
Populate WMBS with a express-like workflow,
every subscription must be unfinished at first
"""
workflowName = 'Express_Run481516_StreamZFast'
secondLevelTasks = ['ExpressMergewrite_StreamZFast_DQM', 'ExpressMergewrite_ExpressPhysics_FEVT',
'ExpressAlcaSkimwrite_StreamZFast_ALCARECO', 'ExpressCleanupUnmergedwrite_StreamZFast_DQM',
'ExpressCleanupUnmergedwrite_ExpressPhysics_FEVT', 'ExpressCleanupUnmergedwrite_StreamZFast_ALCARECO']
alcaHarvestTask = 'ExpressAlcaSkimwrite_StreamZFast_ALCARECOAlcaHarvestALCARECOStreamPromptCalibProd'
dqmHarvestTask = 'ExpressMergewrite_StreamZFast_DQMEndOfRunDQMHarvestMerged'
self.stateMap = {'Merge' : [],
'Harvesting' : [],
'Processing Done' : []}
self.orderedStates = ['Merge', 'Harvesting', 'Processing Done']
# Populate WMStats
self.wmstatsWriter.insertGenericRequest({'_id' : workflowName})
self.wmstatsWriter.updateRequestStatus(workflowName, 'Closed')
# Create a wmspec in disk
workload = newWorkload(workflowName)
expressTask = workload.newTask('Express')
for task in secondLevelTasks:
secondLevelTask = expressTask.addTask(task)
if task == 'ExpressAlcaSkimwrite_StreamZFast_ALCARECO':
secondLevelTask.addTask(alcaHarvestTask)
elif task == 'ExpressMergewrite_StreamZFast_DQM':
secondLevelTask.addTask(dqmHarvestTask)
specPath = os.path.join(self.testDir, 'Express.pkl')
workload.save(specPath)
# Populate WMBS
sharedFileset = Fileset(name = 'TestFileset')
sharedFileset.create()
sharedFileset.markOpen(False)
options = {'spec' : specPath, 'owner' : 'ItsAMeMario',
'name' : workflowName, 'wfType' : 'tier0'}
topLevelWorkflow = Workflow(task = '/%s/Express' % workflowName,
**options)
topLevelWorkflow.create()
topLevelSub = Subscription(sharedFileset, topLevelWorkflow)
topLevelSub.create()
self.stateMap['Merge'].append(topLevelSub)
for task in filter(lambda x : not x.count('CleanupUnmerged'), secondLevelTasks):
secondLevelWorkflow = Workflow(task = '/%s/Express/%s' % (workflowName, task), **options)
secondLevelWorkflow.create()
mergeSub = Subscription(sharedFileset, secondLevelWorkflow)
mergeSub.create()
self.stateMap['Harvesting'].append(mergeSub)
for (parent, child) in [('ExpressAlcaSkimwrite_StreamZFast_ALCARECO', alcaHarvestTask),
('ExpressMergewrite_StreamZFast_DQM', dqmHarvestTask)]:
harvestingWorkflow = Workflow(task = '/%s/Express/%s/%s' % (workflowName, parent, child),
**options)
harvestingWorkflow.create()
harvestingSub = Subscription(sharedFileset, harvestingWorkflow)
harvestingSub.create()
self.stateMap['Processing Done'].append(harvestingSub)
return
def setupPromptRecoWorkflow(self):
"""
_setupPromptRecoWorkflow_
Populate WMBS with a real PromptReco workflow,
every subscription must be unfinished at first
"""
# Populate disk and WMBS
testArguments = getTestArguments()
workflowName = 'PromptReco_Run195360_Cosmics'
workload = promptrecoWorkload(workflowName, testArguments)
wmbsHelper = WMBSHelper(workload, 'Reco', 'SomeBlock', cachepath = self.testDir)
wmbsHelper.createTopLevelFileset()
wmbsHelper._createSubscriptionsInWMBS(wmbsHelper.topLevelTask, wmbsHelper.topLevelFileset)
self.stateMap = {'AlcaSkim' : [],
'Merge' : [],
'Harvesting' : [],
'Processing Done' : []}
self.orderedStates = ['AlcaSkim', 'Merge', 'Harvesting', 'Processing Done']
# Populate WMStats
self.wmstatsWriter.insertGenericRequest({'_id' : workflowName})
self.wmstatsWriter.updateRequestStatus(workflowName, 'Closed')
topLevelTask = '/%s/Reco' % workflowName
alcaSkimTask = '%s/AlcaSkim' % topLevelTask
mergeTasks = ['%s/AlcaSkim/AlcaSkimMergeALCARECOStreamHcalCalHOCosmics',
'%s/AlcaSkim/AlcaSkimMergeALCARECOStreamTkAlCosmics0T',
'%s/AlcaSkim/AlcaSkimMergeALCARECOStreamMuAlGlobalCosmics',
'%s/RecoMergewrite_AOD',
'%s/RecoMergewrite_DQM',
'%s/RecoMergewrite_RECO']
harvestingTask = '%s/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged' % topLevelTask
self.stateMap['AlcaSkim'].append(wmbsHelper.topLevelSubscription)
alcaSkimWorkflow = Workflow(name = workflowName, task = alcaSkimTask)
alcaSkimWorkflow.load()
alcarecoFileset = Fileset(name = '/PromptReco_Run195360_Cosmics/Reco/unmerged-write_ALCARECO')
alcarecoFileset.load()
alcaSkimSub = Subscription(alcarecoFileset, alcaSkimWorkflow)
alcaSkimSub.load()
self.stateMap['Merge'].append(alcaSkimSub)
for task in mergeTasks:
mergeTask = task % topLevelTask
mergeWorkflow = Workflow(name = workflowName, task = mergeTask)
mergeWorkflow.load()
if 'AlcaSkim' in mergeTask:
stream = mergeTask.split('/')[-1][13:]
unmergedFileset = Fileset(name = '%s/unmerged-%s' % (alcaSkimTask, stream))
unmergedFileset.load()
else:
dataTier = mergeTask.split('/')[-1].split('_')[-1]
unmergedFileset = Fileset(name = '%s/unmerged-write_%s' % (topLevelTask, dataTier))
unmergedFileset.load()
mergeSub = Subscription(unmergedFileset, mergeWorkflow)
mergeSub.load()
self.stateMap['Harvesting'].append(mergeSub)
harvestingWorkflow = Workflow(name = workflowName, task = harvestingTask)
harvestingWorkflow.load()
harvestingFileset = Fileset(name = '/PromptReco_Run195360_Cosmics/Reco/RecoMergewrite_DQM/merged-Merged')
harvestingFileset.load()
harvestingSub = Subscription(harvestingFileset, harvestingWorkflow)
harvestingSub.load()
self.stateMap['Processing Done'].append(harvestingSub)
return
def verifyStateTransitions(self, transitionMethod = 'markFinished', transitionTrigger = True):
"""
_verifyStateTransitions_
Utility method which goes through the list of states in self.orderedStates and
finishes the tasks that demand a state transition in each step. This according
to the defined transition method and trigger.
It verifies that the request document in WMStats is moving according to the transitions
"""
for idx in range(0, len(self.orderedStates) * 2):
nextState = self.orderedStates[idx / 2]
if (idx / 2) == 0:
currentState = 'Closed'
else:
currentState = self.orderedStates[idx / 2 - 1]
if idx % 2 == 0:
for transitionObject in self.stateMap[nextState][:-1]:
method = getattr(transitionObject, transitionMethod)
method(transitionTrigger)
self.plugin([], self.wmstatsWriter, self.wmstatsWriter)
currentStateWorkflows = self.wmstatsWriter.workflowsByStatus([currentState], stale = False)
nextStateWorkflows = self.wmstatsWriter.workflowsByStatus([nextState], stale = False)
self.assertEqual(len(currentStateWorkflows), 1, 'Workflow moved incorrectly from %s' % currentState)
self.assertEqual(len(nextStateWorkflows), 0, 'Workflow moved incorrectly to %s' % nextState)
else:
transitionObject = self.stateMap[nextState][-1]
method = getattr(transitionObject, transitionMethod)
method(transitionTrigger)
self.plugin([], self.wmstatsWriter, self.wmstatsWriter)
currentStateWorkflows = self.wmstatsWriter.workflowsByStatus([currentState], stale = False)
nextStateWorkflows = self.wmstatsWriter.workflowsByStatus([nextState], stale = False)
self.assertEqual(len(currentStateWorkflows), 0, 'Workflow did not move correctly from %s' % currentState)
self.assertEqual(len(nextStateWorkflows), 1, 'Workflow did not move correctly to %s' % nextState)
return
def testA_RepackStates(self):
"""
_testA_RepackStates_
Setup an environment with a Repack workflow
and traverse through the different states.
Check that the transitions are sane.
"""
# Set the environment
self.setupRepackWorkflow()
self.plugin = Tier0Plugin()
# Verify the transitions
self.verifyStateTransitions('markOpen', False)
return
def testB_ExpressStates(self):
"""
_testB_ExpressStates_
Setup an environment with a Express workflow
and traverse through the different states.
Check that the transitions are sane.
"""
# Set the environment
self.setupExpressWorkflow()
self.plugin = Tier0Plugin()
# Verify the transitions
self.verifyStateTransitions()
return
def testC_PromptRecoStates(self):
"""
_testC_PromptRecoStates_
Setup an environment with a PromptReco workflow
and traverse through the different states.
Check that the transitions are sane.
"""
# Set the environment
self.setupPromptRecoWorkflow()
self.plugin = Tier0Plugin()
# Verify the transitions
self.verifyStateTransitions()
return
parser = OptionParser()
parser.set_usage("wmstats-request-status-chagne [agent_url:port]")
parser.add_option("-r", "--request", dest="request", help="resquest name")
parser.add_option("-s",
"--status",
dest="newstatus",
help="set to new status")
(options, args) = parser.parse_args()
if not options.request:
print "request name needs to be set"
sys.exit(1)
if not options.newstatus:
print "new status needs to be set"
sys.exit(1)
answer = raw_input("%s change to %s in wmstats db (yes, no)?" %
(options.request, options.newstatus))
if not answer.lower() == "yes":
print "Canceled"
sys.exit(1)
report = wmstats.updateRequestStatus(options.request, options.newstatus)
print report
class CleanCouchPoller(BaseWorkerThread):
"""
Cleans up local couch db according the the given condition.
1. Cleans local couch db when request is completed and reported to cental db.
This will clean up local couchdb, local summary db, local queue
2. Cleans old couchdoc which is created older than the time threshold
"""
def __init__(self, config):
"""
Initialize config
"""
BaseWorkerThread.__init__(self)
# set the workqueue service for REST call
self.config = config
def setup(self, parameters):
"""
Called at startup
"""
# set the connection for local couchDB call
self.useReqMgrForCompletionCheck = getattr(self.config.TaskArchiver, 'useReqMgrForCompletionCheck', True)
self.wmstatsCouchDB = WMStatsWriter(self.config.TaskArchiver.localWMStatsURL)
self.centralCouchDBReader = WMStatsReader(self.config.TaskArchiver.centralWMStatsURL)
if self.useReqMgrForCompletionCheck:
self.deletableStates = ["announced"]
self.centralCouchDBWriter = WMStatsWriter(self.config.TaskArchiver.centralWMStatsURL)
else:
# Tier0 case
self.deletableStates = ["completed"]
self.centralCouchDBWriter = self.wmstatsCouchDB
jobDBurl = sanitizeURL(self.config.JobStateMachine.couchurl)['url']
jobDBName = self.config.JobStateMachine.couchDBName
self.jobCouchdb = CouchServer(jobDBurl)
self.jobsdatabase = self.jobCouchdb.connectDatabase("%s/jobs" % jobDBName)
self.fwjrdatabase = self.jobCouchdb.connectDatabase("%s/fwjrs" % jobDBName)
def algorithm(self, parameters):
"""
get information from wmbs, workqueue and local couch
"""
try:
logging.info("Cleaning up the old request docs")
report = self.wmstatsCouchDB.deleteOldDocs(self.config.TaskArchiver.DataKeepDays)
logging.info("%s docs deleted" % report)
logging.info("getting complete and announced requests")
deletableWorkflows = self.centralCouchDBReader.workflowsByStatus(self.deletableStates)
logging.info("Ready to delete %s" % deletableWorkflows)
for workflowName in deletableWorkflows:
if self.cleanAllLocalCouchDB(workflowName):
self.centralCouchDBWriter.updateRequestStatus(workflowName, "normal-archived")
logging.info("status updated to normal-archived %s" % workflowName)
abortedWorkflows = self.centralCouchDBReader.workflowsByStatus(["aborted-completed"])
logging.info("Ready to delete aborted %s" % abortedWorkflows)
for workflowName in abortedWorkflows:
if self.cleanAllLocalCouchDB(workflowName):
self.centralCouchDBWriter.updateRequestStatus(workflowName, "aborted-archived")
logging.info("status updated to aborted-archived %s" % workflowName)
#TODO: following code is temproraly - remove after production archived data is cleaned
removableWorkflows = self.centralCouchDBReader.workflowsByStatus(["archived"])
logging.info("Ready to delete %s from wmagent_summary" % removableWorkflows)
for workflowName in removableWorkflows:
logging.info("Deleting %s from WMAgent Summary Couch" % workflowName)
report = self.deleteWorkflowFromJobCouch(workflowName, "WMStats")
logging.info("%s docs deleted from wmagent_summary" % report)
# only updatet he status when delete is successful
# TODO: need to handle the case when there are multiple agent running the same request.
if report["status"] == "ok":
self.centralCouchDBWriter.updateRequestStatus(workflowName, "normal-archived")
logging.info("status updated to normal-archived from archived (this is temp solution for production) %s" % workflowName)
except Exception, ex:
logging.error(str(ex))
logging.error("Error occurred, will try again next cycle")
class CleanCouchPoller(BaseWorkerThread):
"""
Cleans up local couch db according the the given condition.
1. Cleans local couch db when request is completed and reported to cental db.
This will clean up local couchdb, local summary db, local queue
2. Cleans old couchdoc which is created older than the time threshold
"""
def __init__(self, config):
"""
Initialize config
"""
BaseWorkerThread.__init__(self)
# set the workqueue service for REST call
self.config = config
def setup(self, parameters):
"""
Called at startup
"""
# set the connection for local couchDB call
self.useReqMgrForCompletionCheck = getattr(self.config.TaskArchiver, 'useReqMgrForCompletionCheck', True)
self.wmstatsCouchDB = WMStatsWriter(self.config.TaskArchiver.localWMStatsURL)
self.centralCouchDBWriter = WMStatsWriter(self.config.TaskArchiver.centralWMStatsURL)
self.centralCouchDBReader = WMStatsReader(self.config.TaskArchiver.centralWMStatsURL)
jobDBurl = sanitizeURL(self.config.JobStateMachine.couchurl)['url']
jobDBName = self.config.JobStateMachine.couchDBName
self.jobCouchdb = CouchServer(jobDBurl)
self.jobsdatabase = self.jobCouchdb.connectDatabase("%s/jobs" % jobDBName)
self.fwjrdatabase = self.jobCouchdb.connectDatabase("%s/fwjrs" % jobDBName)
def algorithm(self, parameters):
"""
get information from wmbs, workqueue and local couch
"""
try:
logging.info("Cleaning up the old request docs")
report = self.wmstatsCouchDB.deleteOldDocs(self.config.TaskArchiver.DataKeepDays)
logging.info("%s docs deleted" % report)
logging.info("getting complete and announced requests")
#TODO: define what is deletable status. Also add the code to delet summary document,
# request summary and job summary
if self.useReqMgrForCompletionCheck:
deletableWorkflows = self.centralCouchDBReader.workflowsByStatus(["announced"])
else:
deletableWorkflows = self.centralCouchDBReader.workflowsByStatus(["completed"])
logging.info("Ready to delete %s" % deletableWorkflows)
for workflowName in deletableWorkflows:
logging.info("Deleting %s from JobCouch" % workflowName)
report = self.deleteWorkflowFromJobCouch(workflowName, "JobDump")
logging.info("%s docs deleted from JobDump" % report)
report = self.deleteWorkflowFromJobCouch(workflowName, "FWJRDump")
logging.info("%s docs deleted from FWJRDump" % report)
self.centralCouchDBWriter.updateRequestStatus(workflowName, "archived")
logging.info("status updated to archived %s" % workflowName)
except Exception, ex:
logging.error(str(ex))
logging.error("Error occurred, will try again next cycle")
class WMStatsTest(unittest.TestCase):
"""
"""
def setUp(self):
"""
_setUp_
"""
self.schema = []
self.couchApps = ["WMStats"]
self.testInit = TestInitCouchApp('WorkQueueServiceTest')
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=self.schema, useDefault=False)
self.testInit.setupCouch('wmstats_t', *self.couchApps)
self.wmstatsWriter = WMStatsWriter(self.testInit.couchUrl, 'wmstats_t')
return
def tearDown(self):
"""
_tearDown_
Drop all the WMBS tables.
"""
self.testInit.tearDownCouch()
def testWMStatsWriter(self):
# test getWork
schema = generate_reqmgr_schema()
self.assertEquals(self.wmstatsWriter.insertRequest(schema[0]), 'OK',
'insert fail')
self.assertEquals(
self.wmstatsWriter.updateRequestStatus(schema[0]['RequestName'],
"failed"), 'OK',
'update fail')
self.assertEquals(
self.wmstatsWriter.updateRequestStatus("not_exist_schema",
"assigned"),
'ERROR: request not found - not_exist_schema')
self.assertEquals(
self.wmstatsWriter.updateTeam(schema[0]['RequestName'], 'teamA'),
'OK', 'update fail')
self.assertEquals(
self.wmstatsWriter.updateTeam("not_exist_schema", 'teamA'),
'ERROR: request not found - not_exist_schema')
totalStats = {
'total_jobs': 100,
'input_events': 1000,
'input_lumis': 1234,
'input_num_file': 5
}
self.assertEquals(
self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'],
totalStats), 'OK',
'update fail')
self.assertEquals(
self.wmstatsWriter.insertTotalStats("not_exist_schema",
totalStats),
'ERROR: request not found - not_exist_schema')
spec1 = newWorkload(schema[0]['RequestName'])
production = spec1.newTask("Production")
production.setTaskType("Merge")
production.setSiteWhitelist(['TEST_SITE'])
self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec1), 'OK',
'update fail')
spec2 = newWorkload("not_exist_schema")
production = spec2.newTask("Production")
production.setTaskType("Merge")
self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec2),
'ERROR: request not found - not_exist_schema')
class Tier0FeederPoller(BaseWorkerThread):
def __init__(self, config):
"""
_init_
"""
BaseWorkerThread.__init__(self)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package = "T0.WMBS",
logger = logging,
dbinterface = myThread.dbi)
self.tier0ConfigFile = config.Tier0Feeder.tier0ConfigFile
self.specDirectory = config.Tier0Feeder.specDirectory
self.dropboxuser = config.Tier0Feeder.dropboxuser
self.dropboxpass = config.Tier0Feeder.dropboxpass
self.transferSystemBaseDir = getattr(config.Tier0Feeder, "transferSystemBaseDir", None)
if self.transferSystemBaseDir != None:
if not os.path.exists(self.transferSystemBaseDir):
self.transferSystemBaseDir = None
self.dqmUploadProxy = config.WMBSService.proxy
self.localSummaryCouchDB = WMStatsWriter(config.AnalyticsDataCollector.localWMStatsURL)
hltConfConnectUrl = config.HLTConfDatabase.connectUrl
dbFactoryHltConf = DBFactory(logging, dburl = hltConfConnectUrl, options = {})
dbInterfaceHltConf = dbFactoryHltConf.connect()
daoFactoryHltConf = DAOFactory(package = "T0.WMBS",
logger = logging,
dbinterface = dbInterfaceHltConf)
self.getHLTConfigDAO = daoFactoryHltConf(classname = "RunConfig.GetHLTConfig")
storageManagerConnectUrl = config.StorageManagerDatabase.connectUrl
dbFactoryStorageManager = DBFactory(logging, dburl = storageManagerConnectUrl, options = {})
self.dbInterfaceStorageManager = dbFactoryStorageManager.connect()
self.getExpressReadyRunsDAO = None
if hasattr(config, "PopConLogDatabase"):
popConLogConnectUrl = getattr(config.PopConLogDatabase, "connectUrl", None)
if popConLogConnectUrl != None:
dbFactoryPopConLog = DBFactory(logging, dburl = popConLogConnectUrl, options = {})
dbInterfacePopConLog = dbFactoryPopConLog.connect()
daoFactoryPopConLog = DAOFactory(package = "T0.WMBS",
logger = logging,
dbinterface = dbInterfacePopConLog)
self.getExpressReadyRunsDAO = daoFactoryPopConLog(classname = "Tier0Feeder.GetExpressReadyRuns")
return
def algorithm(self, parameters = None):
"""
_algorithm_
"""
logging.debug("Running Tier0Feeder algorithm...")
myThread = threading.currentThread()
findNewRunsDAO = self.daoFactory(classname = "Tier0Feeder.FindNewRuns")
findNewRunStreamsDAO = self.daoFactory(classname = "Tier0Feeder.FindNewRunStreams")
findNewExpressRunsDAO = self.daoFactory(classname = "Tier0Feeder.FindNewExpressRuns")
releaseExpressDAO = self.daoFactory(classname = "Tier0Feeder.ReleaseExpress")
feedStreamersDAO = self.daoFactory(classname = "Tier0Feeder.FeedStreamers")
markRepackInjectedDAO = self.daoFactory(classname = "Tier0Feeder.MarkRepackInjected")
tier0Config = None
try:
tier0Config = loadConfigurationFile(self.tier0ConfigFile)
except:
# usually happens when there are syntax errors in the configuration
logging.exception("Cannot load Tier0 configuration file, not configuring new runs and run/streams")
# only configure new runs and run/streams if we have a valid Tier0 configuration
if tier0Config != None:
#
# find new runs, setup global run settings and stream/dataset/trigger mapping
#
runHltkeys = findNewRunsDAO.execute(transaction = False)
for run, hltkey in sorted(runHltkeys.items()):
hltConfig = None
# local runs have no hltkey and are configured differently
if hltkey != None:
# retrieve HLT configuration and make sure it's usable
try:
hltConfig = self.getHLTConfigDAO.execute(hltkey, transaction = False)
if hltConfig['process'] == None or len(hltConfig['mapping']) == 0:
raise RuntimeError, "HLTConfDB query returned no process or mapping"
except:
logging.exception("Can't retrieve hltkey %s for run %d" % (hltkey, run))
continue
try:
RunConfigAPI.configureRun(tier0Config, run, hltConfig)
except:
logging.exception("Can't configure for run %d" % (run))
#
# find unconfigured run/stream with data
# populate RunConfig, setup workflows/filesets/subscriptions
#
runStreams = findNewRunStreamsDAO.execute(transaction = False)
for run in sorted(runStreams.keys()):
for stream in sorted(runStreams[run]):
try:
RunConfigAPI.configureRunStream(tier0Config,
run, stream,
self.specDirectory,
self.dqmUploadProxy)
except:
logging.exception("Can't configure for run %d and stream %s" % (run, stream))
#
# end runs which are active and have ended according to the EoR StorageManager records
#
RunLumiCloseoutAPI.endRuns(self.dbInterfaceStorageManager)
#
# release runs for Express
#
runs = findNewExpressRunsDAO.execute(transaction = False)
if len(runs) > 0:
binds = []
for run in runs:
binds.append( { 'RUN' : run } )
if self.getExpressReadyRunsDAO != None:
runs = self.getExpressReadyRunsDAO.execute(binds = binds, transaction = False)
if len(runs) > 0:
binds = []
for run in runs:
binds.append( { 'RUN' : run } )
releaseExpressDAO.execute(binds = binds, transaction = False)
#
# release runs for PromptReco
#
RunConfigAPI.releasePromptReco(tier0Config,
self.specDirectory,
self.dqmUploadProxy)
#
# check if all datasets for a stream had their PromptReco released
# then mark the repack workflow as injected (if we don't wait, the
# task archiver will cleanup too early)
#
markRepackInjectedDAO.execute(transaction = False)
#
# close stream/lumis for run/streams that are active (fileset exists and open)
#
RunLumiCloseoutAPI.closeLumiSections(self.dbInterfaceStorageManager)
#
# feed new data into exisiting filesets
#
try:
myThread.transaction.begin()
feedStreamersDAO.execute(conn = myThread.transaction.conn, transaction = True)
except:
logging.exception("Can't feed data, bailing out...")
raise
else:
myThread.transaction.commit()
#
# run ended and run/stream fileset open
# => check for complete lumi_closed record, all lumis finally closed and all data feed
# => if all conditions satisfied, close the run/stream fileset
#
RunLumiCloseoutAPI.closeRunStreamFilesets()
#
# check and delete active split lumis
#
RunLumiCloseoutAPI.checkActiveSplitLumis()
#
# insert workflows into CouchDB for monitoring
#
self.feedCouchMonitoring()
#
# Update Couch when Repack and Express have closed input filesets (analog to old T0 closeout)
#
self.closeOutRealTimeWorkflows()
#
# send repacked notifications to StorageManager
#
if self.transferSystemBaseDir != None:
self.notifyStorageManager()
#
# upload PCL conditions to DropBox
#
ConditionUploadAPI.uploadConditions(self.dropboxuser, self.dropboxpass)
return
def feedCouchMonitoring(self):
"""
_feedCouchMonitoring_
check for workflows that haven't been uploaded to Couch for monitoring yet
"""
getStreamerWorkflowsForMonitoringDAO = self.daoFactory(classname = "Tier0Feeder.GetStreamerWorkflowsForMonitoring")
getPromptRecoWorkflowsForMonitoringDAO = self.daoFactory(classname = "Tier0Feeder.GetPromptRecoWorkflowsForMonitoring")
markTrackedWorkflowMonitoringDAO = self.daoFactory(classname = "Tier0Feeder.MarkTrackedWorkflowMonitoring")
workflows = getStreamerWorkflowsForMonitoringDAO.execute()
workflows += getPromptRecoWorkflowsForMonitoringDAO.execute()
if len(workflows) == 0:
logging.debug("No workflows to publish to couch monitoring, doing nothing")
if workflows:
logging.debug(" Going to publish %d workflows" % len(workflows))
for (workflowId, run, workflowName) in workflows:
logging.info(" Publishing workflow %s to monitoring" % workflowName)
doc = {}
doc["_id"] = workflowName
doc["workflow"] = workflowName
doc["type"] = "tier0_request"
doc["run"] = run
response = self.localSummaryCouchDB.insertGenericRequest(doc)
if response == "OK" or "EXISTS":
logging.info(" Successfully uploaded request %s" % workflowName)
# Here we have to trust the insert, if it doesn't happen will be easy to spot on the logs
markTrackedWorkflowMonitoringDAO.execute(workflowId)
return
def closeOutRealTimeWorkflows(self):
"""
_closeOutRealTimeWorkflows_
Updates couch with the closeout status of Repack and Express
PromptReco should be closed out automatically
"""
getNotClosedOutWorkflowsDAO = self.daoFactory(classname = "Tier0Feeder.GetNotClosedOutWorkflows")
workflows = getNotClosedOutWorkflowsDAO.execute()
if len(workflows) == 0:
logging.debug("No workflows to publish to couch monitoring, doing nothing")
if workflows:
for workflow in workflows:
(workflowId, filesetId, filesetOpen, workflowName) = workflow
# find returns -1 if the string is not found
if workflowName.find('PromptReco') >= 0:
logging.debug("Closing out instantaneously PromptReco Workflow %s" % workflowName)
self.updateClosedState(workflowName, workflowId)
else :
# Check if fileset (which you already know) is closed or not
# FIXME: No better way to do it? what comes from the DAO is a string, casting bool or int doesn't help much.
# Works like that :
if filesetOpen == '0':
self.updateClosedState(workflowName, workflowId)
return
def updateClosedState(self, workflowName, workflowId):
"""
_updateClosedState_
Mark a workflow as Closed
"""
markCloseoutWorkflowMonitoringDAO = self.daoFactory(classname = "Tier0Feeder.MarkCloseoutWorkflowMonitoring")
response = self.localSummaryCouchDB.updateRequestStatus(workflowName, 'Closed')
if response == "OK" or "EXISTS":
logging.debug("Successfully closed workflow %s" % workflowName)
markCloseoutWorkflowMonitoringDAO.execute(workflowId)
return
def notifyStorageManager(self):
"""
_notifyStorageManager_
Find all finished streamers for closed all run/stream
Send the notification message to StorageManager
Update the streamer status to finished (deleted = 1)
"""
getFinishedStreamersDAO = self.daoFactory(classname = "SMNotification.GetFinishedStreamers")
markStreamersFinishedDAO = self.daoFactory(classname = "SMNotification.MarkStreamersFinished")
allFinishedStreamers = getFinishedStreamersDAO.execute(transaction = False)
num = len(allFinishedStreamers)/50
for finishedStreamers in [allFinishedStreamers[i::num] for i in range(num)]:
streamers = []
filenameParams = ""
for (id, lfn) in finishedStreamers:
streamers.append(id)
filenameParams += "-FILENAME %s " % os.path.basename(lfn)
logging.debug("Notifying transfer system about processed streamers")
p = subprocess.Popen("/bin/bash",stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
output, error = p.communicate("""
export T0_BASE_DIR=%s
export T0ROOT=${T0_BASE_DIR}/T0
export CONFIG=${T0_BASE_DIR}/Config/TransferSystem_CERN.cfg
export PERL5LIB=${T0ROOT}/perl_lib
${T0ROOT}/operations/sendRepackedStatus.pl --config $CONFIG %s
""" % (self.transferSystemBaseDir, filenameParams))
if len(error) > 0:
logging.error("ERROR: Could not notify transfer system about processed streamers")
logging.error("ERROR: %s" % error)
markStreamersFinishedDAO.execute(streamers, transaction = False)
return
def terminate(self, params):
"""
_terminate_
Kill the code after one final pass when called by the master thread.
"""
logging.debug("terminating immediately")
class CleanCouchPoller(BaseWorkerThread):
"""
Cleans up local couch db according the the given condition.
1. Cleans local couch db when request is completed and reported to cental db.
This will clean up local couchdb, local summary db, local queue
2. Cleans old couchdoc which is created older than the time threshold
"""
def __init__(self, config):
"""
Initialize config
"""
BaseWorkerThread.__init__(self)
# set the workqueue service for REST call
self.config = config
def setup(self, parameters):
"""
Called at startup
"""
# set the connection for local couchDB call
self.useReqMgrForCompletionCheck = getattr(
self.config.TaskArchiver, 'useReqMgrForCompletionCheck', True)
self.wmstatsCouchDB = WMStatsWriter(
self.config.TaskArchiver.localWMStatsURL)
self.centralCouchDBReader = WMStatsReader(
self.config.TaskArchiver.centralWMStatsURL)
if self.useReqMgrForCompletionCheck:
self.deletableStates = ["announced"]
self.centralCouchDBWriter = WMStatsWriter(
self.config.TaskArchiver.centralWMStatsURL)
else:
# Tier0 case
self.deletableStates = ["completed"]
self.centralCouchDBWriter = self.wmstatsCouchDB
jobDBurl = sanitizeURL(self.config.JobStateMachine.couchurl)['url']
jobDBName = self.config.JobStateMachine.couchDBName
self.jobCouchdb = CouchServer(jobDBurl)
self.jobsdatabase = self.jobCouchdb.connectDatabase("%s/jobs" %
jobDBName)
self.fwjrdatabase = self.jobCouchdb.connectDatabase("%s/fwjrs" %
jobDBName)
def algorithm(self, parameters):
"""
get information from wmbs, workqueue and local couch
"""
try:
logging.info("Cleaning up the old request docs")
report = self.wmstatsCouchDB.deleteOldDocs(
self.config.TaskArchiver.DataKeepDays)
logging.info("%s docs deleted" % report)
logging.info("getting complete and announced requests")
deletableWorkflows = self.centralCouchDBReader.workflowsByStatus(
self.deletableStates)
logging.info("Ready to delete %s" % deletableWorkflows)
for workflowName in deletableWorkflows:
if self.cleanAllLocalCouchDB(workflowName):
self.centralCouchDBWriter.updateRequestStatus(
workflowName, "normal-archived")
logging.info("status updated to normal-archived %s" %
workflowName)
abortedWorkflows = self.centralCouchDBReader.workflowsByStatus(
["aborted-completed"])
logging.info("Ready to delete aborted %s" % abortedWorkflows)
for workflowName in abortedWorkflows:
if self.cleanAllLocalCouchDB(workflowName):
self.centralCouchDBWriter.updateRequestStatus(
workflowName, "aborted-archived")
logging.info("status updated to aborted-archived %s" %
workflowName)
#TODO: following code is temproraly - remove after production archived data is cleaned
removableWorkflows = self.centralCouchDBReader.workflowsByStatus(
["archived"])
logging.info("Ready to delete %s from wmagent_summary" %
removableWorkflows)
for workflowName in removableWorkflows:
logging.info("Deleting %s from WMAgent Summary Couch" %
workflowName)
report = self.deleteWorkflowFromJobCouch(
workflowName, "WMStats")
logging.info("%s docs deleted from wmagent_summary" % report)
# only updatet he status when delete is successful
# TODO: need to handle the case when there are multiple agent running the same request.
if report["status"] == "ok":
self.centralCouchDBWriter.updateRequestStatus(
workflowName, "normal-archived")
logging.info(
"status updated to normal-archived from archived (this is temp solution for production) %s"
% workflowName)
except Exception, ex:
logging.error(str(ex))
logging.error("Error occurred, will try again next cycle")
print("AnalyticsDataCollector.centralWMStatsURL is not specified")
sys.exit(1)
parser = OptionParser()
parser.set_usage("wmstats-request-status-chagne [agent_url:port]")
parser.add_option("-r", "--request", dest = "request",
help = "resquest name")
parser.add_option("-s", "--status", dest = "newstatus",
help = "set to new status")
(options, args) = parser.parse_args()
if not options.request:
print("request name needs to be set")
sys.exit(1)
if not options.newstatus:
print("new status needs to be set")
sys.exit(1)
answer = raw_input("%s change to %s in wmstats db (yes, no)?" % (options.request, options.newstatus))
if not answer.lower() == "yes":
print("Canceled")
sys.exit(1)
report = wmstats.updateRequestStatus(options.request, options.newstatus)
print(report)
class Tier0PluginTest(unittest.TestCase):
def setUp(self):
"""
_setUp_
Setup the test environment
"""
self.testInit = TestInit(__file__)
self.testInit.setDatabaseConnection()
self.testInit.setSchema(["WMCore.WMBS"])
self.wmstatsCouchDB = 'wmstats_plugin_t'
self.testInit.setupCouch(self.wmstatsCouchDB, 'WMStats')
self.testDir = self.testInit.generateWorkDir()
self.wmstatsWriter = WMStatsWriter(os.environ['COUCHURL'],
self.wmstatsCouchDB)
self.stateMap = {}
self.orderedStates = []
self.plugin = None
return
def tearDown(self):
"""
_tearDown_
Clear databases and delete files
"""
self.testInit.tearDownCouch()
self.testInit.clearDatabase()
self.testInit.delWorkDir()
return
def setupRepackWorkflow(self):
"""
_setupRepackWorkflow_
Populate WMBS with a repack-like workflow,
every subscription must be unfinished at first
"""
workflowName = 'Repack_Run481516_StreamZ'
mergeTasks = [
'RepackMergewrite_QuadElectron_RAW',
'RepackMergewrite_TriPhoton_RAW',
'RepackMergewrite_SingleNeutrino_RAW'
]
self.stateMap = {'Merge': [], 'Processing Done': []}
self.orderedStates = ['Merge', 'Processing Done']
# Populate WMStats
self.wmstatsWriter.insertGenericRequest({'_id': workflowName})
self.wmstatsWriter.updateRequestStatus(workflowName, 'Closed')
# Create a wmspec in disk
workload = newWorkload(workflowName)
repackTask = workload.newTask('Repack')
for task in mergeTasks:
repackTask.addTask(task)
repackTask.addTask('RepackCleanupUnmergedwrite_QuadElectron_RAW')
specPath = os.path.join(self.testDir, 'Repack.pkl')
workload.save(specPath)
# Populate WMBS
topFileset = Fileset(name='TestStreamerFileset')
topFileset.create()
options = {
'spec': specPath,
'owner': 'ItsAMeMario',
'name': workflowName,
'wfType': 'tier0'
}
topLevelWorkflow = Workflow(task='/%s/Repack' % workflowName,
**options)
topLevelWorkflow.create()
topLevelSub = Subscription(topFileset, topLevelWorkflow)
topLevelSub.create()
self.stateMap['Merge'].append(topFileset)
for task in mergeTasks:
mergeWorkflow = Workflow(task='/%s/Repack/%s' %
(workflowName, task),
**options)
mergeWorkflow.create()
unmergedFileset = Fileset(name='TestUnmergedFileset%s' % task)
unmergedFileset.create()
mergeSub = Subscription(unmergedFileset, mergeWorkflow)
mergeSub.create()
self.stateMap['Processing Done'].append(unmergedFileset)
cleanupWorkflow = Workflow(
task=
'/Repack_Run481516_StreamZ/Repack/RepackCleanupUnmergedwrite_QuadElectron_RAW',
**options)
cleanupWorkflow.create()
unmergedFileset = Fileset(name='TestUnmergedFilesetToCleanup')
unmergedFileset.create()
cleanupSub = Subscription(unmergedFileset, cleanupWorkflow)
cleanupSub.create()
return
def setupExpressWorkflow(self):
"""
_setupExpressWorkflow_
Populate WMBS with a express-like workflow,
every subscription must be unfinished at first
"""
workflowName = 'Express_Run481516_StreamZFast'
secondLevelTasks = [
'ExpressMergewrite_StreamZFast_DQM',
'ExpressMergewrite_ExpressPhysics_FEVT',
'ExpressAlcaSkimwrite_StreamZFast_ALCARECO',
'ExpressCleanupUnmergedwrite_StreamZFast_DQM',
'ExpressCleanupUnmergedwrite_ExpressPhysics_FEVT',
'ExpressCleanupUnmergedwrite_StreamZFast_ALCARECO'
]
alcaHarvestTask = 'ExpressAlcaSkimwrite_StreamZFast_ALCARECOAlcaHarvestALCARECOStreamPromptCalibProd'
dqmHarvestTask = 'ExpressMergewrite_StreamZFast_DQMDQMHarvestMerged'
self.stateMap = {'Merge': [], 'Harvesting': [], 'Processing Done': []}
self.orderedStates = ['Merge', 'Harvesting', 'Processing Done']
# Populate WMStats
self.wmstatsWriter.insertGenericRequest({'_id': workflowName})
self.wmstatsWriter.updateRequestStatus(workflowName, 'Closed')
# Create a wmspec in disk
workload = newWorkload(workflowName)
expressTask = workload.newTask('Express')
for task in secondLevelTasks:
secondLevelTask = expressTask.addTask(task)
if task == 'ExpressAlcaSkimwrite_StreamZFast_ALCARECO':
secondLevelTask.addTask(alcaHarvestTask)
elif task == 'ExpressMergewrite_StreamZFast_DQM':
secondLevelTask.addTask(dqmHarvestTask)
specPath = os.path.join(self.testDir, 'Express.pkl')
workload.save(specPath)
# Populate WMBS
sharedFileset = Fileset(name='TestFileset')
sharedFileset.create()
sharedFileset.markOpen(False)
options = {
'spec': specPath,
'owner': 'ItsAMeMario',
'name': workflowName,
'wfType': 'tier0'
}
topLevelWorkflow = Workflow(task='/%s/Express' % workflowName,
**options)
topLevelWorkflow.create()
topLevelSub = Subscription(sharedFileset, topLevelWorkflow)
topLevelSub.create()
self.stateMap['Merge'].append(topLevelSub)
for task in filter(lambda x: not x.count('CleanupUnmerged'),
secondLevelTasks):
secondLevelWorkflow = Workflow(task='/%s/Express/%s' %
(workflowName, task),
**options)
secondLevelWorkflow.create()
mergeSub = Subscription(sharedFileset, secondLevelWorkflow)
mergeSub.create()
self.stateMap['Harvesting'].append(mergeSub)
for (parent, child) in [
('ExpressAlcaSkimwrite_StreamZFast_ALCARECO', alcaHarvestTask),
('ExpressMergewrite_StreamZFast_DQM', dqmHarvestTask)
]:
harvestingWorkflow = Workflow(task='/%s/Express/%s/%s' %
(workflowName, parent, child),
**options)
harvestingWorkflow.create()
harvestingSub = Subscription(sharedFileset, harvestingWorkflow)
harvestingSub.create()
self.stateMap['Processing Done'].append(harvestingSub)
return
def setupPromptRecoWorkflow(self):
"""
_setupPromptRecoWorkflow_
Populate WMBS with a real PromptReco workflow,
every subscription must be unfinished at first
"""
# Populate disk and WMBS
testArguments = getTestArguments()
workflowName = 'PromptReco_Run195360_Cosmics'
workload = promptrecoWorkload(workflowName, testArguments)
wmbsHelper = WMBSHelper(workload,
'Reco',
'SomeBlock',
cachepath=self.testDir)
wmbsHelper.createTopLevelFileset()
wmbsHelper.createSubscription(wmbsHelper.topLevelTask,
wmbsHelper.topLevelFileset)
self.stateMap = {
'AlcaSkim': [],
'Merge': [],
'Harvesting': [],
'Processing Done': []
}
self.orderedStates = [
'AlcaSkim', 'Merge', 'Harvesting', 'Processing Done'
]
# Populate WMStats
self.wmstatsWriter.insertGenericRequest({'_id': workflowName})
self.wmstatsWriter.updateRequestStatus(workflowName, 'Closed')
topLevelTask = '/%s/Reco' % workflowName
alcaSkimTask = '%s/AlcaSkim' % topLevelTask
mergeTasks = [
'%s/AlcaSkim/AlcaSkimMergeALCARECOStreamHcalCalHOCosmics',
'%s/AlcaSkim/AlcaSkimMergeALCARECOStreamTkAlCosmics0T',
'%s/AlcaSkim/AlcaSkimMergeALCARECOStreamMuAlGlobalCosmics',
'%s/RecoMergewrite_AOD', '%s/RecoMergewrite_DQM',
'%s/RecoMergewrite_RECO'
]
harvestingTask = '%s/RecoMergewrite_DQM/RecoMergewrite_DQMDQMHarvestMerged' % topLevelTask
self.stateMap['AlcaSkim'].append(wmbsHelper.topLevelSubscription)
alcaSkimWorkflow = Workflow(name=workflowName, task=alcaSkimTask)
alcaSkimWorkflow.load()
alcarecoFileset = Fileset(
name='/PromptReco_Run195360_Cosmics/Reco/unmerged-write_ALCARECO')
alcarecoFileset.load()
alcaSkimSub = Subscription(alcarecoFileset, alcaSkimWorkflow)
alcaSkimSub.load()
self.stateMap['Merge'].append(alcaSkimSub)
for task in mergeTasks:
mergeTask = task % topLevelTask
mergeWorkflow = Workflow(name=workflowName, task=mergeTask)
mergeWorkflow.load()
if 'AlcaSkim' in mergeTask:
stream = mergeTask.split('/')[-1][13:]
unmergedFileset = Fileset(name='%s/unmerged-%s' %
(alcaSkimTask, stream))
unmergedFileset.load()
else:
dataTier = mergeTask.split('/')[-1].split('_')[-1]
unmergedFileset = Fileset(name='%s/unmerged-write_%s' %
(topLevelTask, dataTier))
unmergedFileset.load()
mergeSub = Subscription(unmergedFileset, mergeWorkflow)
mergeSub.load()
self.stateMap['Harvesting'].append(mergeSub)
harvestingWorkflow = Workflow(name=workflowName, task=harvestingTask)
harvestingWorkflow.load()
harvestingFileset = Fileset(
name=
'/PromptReco_Run195360_Cosmics/Reco/RecoMergewrite_DQM/merged-Merged'
)
harvestingFileset.load()
harvestingSub = Subscription(harvestingFileset, harvestingWorkflow)
harvestingSub.load()
self.stateMap['Processing Done'].append(harvestingSub)
return
def verifyStateTransitions(self,
transitionMethod='markFinished',
transitionTrigger=True):
"""
_verifyStateTransitions_
Utility method which goes through the list of states in self.orderedStates and
finishes the tasks that demand a state transition in each step. This according
to the defined transition method and trigger.
It verifies that the request document in WMStats is moving according to the transitions
"""
for idx in range(0, len(self.orderedStates) * 2):
nextState = self.orderedStates[idx / 2]
if (idx / 2) == 0:
currentState = 'Closed'
else:
currentState = self.orderedStates[idx / 2 - 1]
if idx % 2 == 0:
for transitionObject in self.stateMap[nextState][:-1]:
method = getattr(transitionObject, transitionMethod)
method(transitionTrigger)
self.plugin([], self.wmstatsWriter, self.wmstatsWriter)
currentStateWorkflows = self.wmstatsWriter.workflowsByStatus(
[currentState], stale=False)
nextStateWorkflows = self.wmstatsWriter.workflowsByStatus(
[nextState], stale=False)
self.assertEqual(
len(currentStateWorkflows), 1,
'Workflow moved incorrectly from %s' % currentState)
self.assertEqual(
len(nextStateWorkflows), 0,
'Workflow moved incorrectly to %s' % nextState)
else:
transitionObject = self.stateMap[nextState][-1]
method = getattr(transitionObject, transitionMethod)
method(transitionTrigger)
self.plugin([], self.wmstatsWriter, self.wmstatsWriter)
currentStateWorkflows = self.wmstatsWriter.workflowsByStatus(
[currentState], stale=False)
nextStateWorkflows = self.wmstatsWriter.workflowsByStatus(
[nextState], stale=False)
self.assertEqual(
len(currentStateWorkflows), 0,
'Workflow did not move correctly from %s' % currentState)
self.assertEqual(
len(nextStateWorkflows), 1,
'Workflow did not move correctly to %s' % nextState)
return
def testA_RepackStates(self):
"""
_testA_RepackStates_
Setup an environment with a Repack workflow
and traverse through the different states.
Check that the transitions are sane.
"""
# Set the environment
self.setupRepackWorkflow()
self.plugin = Tier0Plugin()
# Verify the transitions
self.verifyStateTransitions('markOpen', False)
return
def testB_ExpressStates(self):
"""
_testB_ExpressStates_
Setup an environment with a Express workflow
and traverse through the different states.
Check that the transitions are sane.
"""
# Set the environment
self.setupExpressWorkflow()
self.plugin = Tier0Plugin()
# Verify the transitions
self.verifyStateTransitions()
return
def testC_PromptRecoStates(self):
"""
_testC_PromptRecoStates_
Setup an environment with a PromptReco workflow
and traverse through the different states.
Check that the transitions are sane.
"""
# Set the environment
self.setupPromptRecoWorkflow()
self.plugin = Tier0Plugin()
# Verify the transitions
self.verifyStateTransitions()
return
def updateRequestStatus(couchURL, requestList, status):
ww = WMStatsWriter(couchURL)
for request in requestList:
ww.updateRequestStatus(request, status)
print("%s is udated to %s" % (request, status))
