class RequestDBTest(unittest.TestCase):
"""
"""
def setUp(self):
"""
_setUp_
"""
self.schema = []
self.couchApps = ["ReqMgr"]
self.testInit = TestInitCouchApp("RequestDBServiceTest")
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=self.schema, useDefault=False)
dbName = "requsetdb_t"
self.testInit.setupCouch(dbName, *self.couchApps)
self.requestWriter = RequestDBWriter(self.testInit.couchUrl, dbName)
self.requestReader = RequestDBReader(self.testInit.couchUrl, dbName)
self.requestWriter.defaultStale = {}
self.requestReader.defaultStale = {}
return
def tearDown(self):
"""
_tearDown_
Drop all the WMBS tables.
"""
self.testInit.tearDownCouch()
def testRequestDBWriter(self):
# test getWork
schema = generate_reqmgr_schema()
result = self.requestWriter.insertGenericRequest(schema[0])
self.assertEquals(len(result), 1, "insert fail")
self.assertEquals(
self.requestWriter.updateRequestStatus(schema[0]["RequestName"], "failed"), "OK", "update fail"
)
self.assertEquals(
self.requestWriter.updateRequestStatus("not_exist_schema", "assigned"), "Error: document not found"
)
result = self.requestWriter.updateRequestProperty(schema[0]["RequestName"], {"Teams": ["teamA"]})
self.assertEquals(
self.requestWriter.updateRequestProperty(schema[0]["RequestName"], {"Teams": ["teamA"]}),
"OK",
"update fail",
)
self.assertEquals(
self.requestWriter.updateRequestProperty("not_exist_schema", {"Teams": "teamA"}),
"Error: document not found",
)
result = self.requestWriter.getRequestByNames([schema[0]["RequestName"]])
self.assertEquals(len(result), 1, "should be 1")
result = self.requestWriter.getRequestByStatus(["failed"], False, 1)
self.assertEquals(len(result), 1, "should be 1")
class T0RequestDBTest(unittest.TestCase):
"""
"""
def setUp(self):
"""
_setUp_
"""
self.schema = []
self.couchApps = ["T0Request"]
self.testInit = TestInitCouchApp('RequestDBServiceTest')
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = self.schema,
useDefault = False)
dbName = 't0_requsetdb_t'
self.testInit.setupCouch(dbName, *self.couchApps)
reqDBURL = "%s/%s" % (self.testInit.couchUrl, dbName)
self.requestWriter = RequestDBWriter(reqDBURL, self.couchApps[0])
self.requestReader = RequestDBReader(reqDBURL, self.couchApps[0])
self.requestWriter.defaultStale = {}
self.requestReader.defaultStale = {}
return
def tearDown(self):
"""
_tearDown_
Drop all the WMBS tables.
"""
self.testInit.tearDownCouch()
def testRequestDBWriter(self):
# test getWork
schema = generate_reqmgr_schema()
result = self.requestWriter.insertGenericRequest(schema[0])
self.assertEqual(len(result), 1, 'insert fail');
result = self.requestWriter.updateRequestStatus(schema[0]['RequestName'], "assigned")
self.assertEqual(result, 'not allowed state assigned', 'update fail')
self.assertEqual(self.requestWriter.updateRequestStatus("not_exist_schema", "new"),
'Error: document not found')
allowedStates = ["Closed", "Merge", "AlcaSkim", "Harvesting",
"Processing Done", "completed"]
for state in allowedStates:
self.assertEqual(self.requestWriter.updateRequestStatus(schema[0]['RequestName'], state),
'OK')
self.assertEqual(self.requestWriter.updateRequestStatus(schema[0]['RequestName'], "Processing Done"),
'not allowed transition completed to Processing Done')
self.assertEqual(self.requestWriter.updateRequestStatus(schema[0]['RequestName'], "normal-archived"),
'OK')
result = self.requestWriter.getRequestByStatus(["normal-archived"], False, 1)
self.assertEqual(len(result), 1, "should be 1 but %s" % result)
class T0RequestDBTest(unittest.TestCase):
"""
"""
def setUp(self):
"""
_setUp_
"""
self.schema = []
self.couchApps = ["T0Request"]
self.testInit = TestInitCouchApp('RequestDBServiceTest')
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = self.schema,
useDefault = False)
dbName = 't0_requsetdb_t'
self.testInit.setupCouch(dbName, *self.couchApps)
reqDBURL = "%s/%s" % (self.testInit.couchUrl, dbName)
self.requestWriter = RequestDBWriter(reqDBURL, self.couchApps[0])
self.requestReader = RequestDBReader(reqDBURL, self.couchApps[0])
self.requestWriter.defaultStale = {}
self.requestReader.defaultStale = {}
return
def tearDown(self):
"""
_tearDown_
Drop all the WMBS tables.
"""
self.testInit.tearDownCouch()
def testRequestDBWriter(self):
# test getWork
schema = generate_reqmgr_schema()
result = self.requestWriter.insertGenericRequest(schema[0])
self.assertEqual(len(result), 1, 'insert fail');
result = self.requestWriter.updateRequestStatus(schema[0]['RequestName'], "assigned")
self.assertEqual(result, 'not allowed state assigned', 'update fail')
self.assertEqual(self.requestWriter.updateRequestStatus("not_exist_schema", "new"),
'Error: document not found')
allowedStates = ["Closed", "Merge", "AlcaSkim", "Harvesting",
"Processing Done", "completed"]
for state in allowedStates:
self.assertEqual(self.requestWriter.updateRequestStatus(schema[0]['RequestName'], state),
'OK')
self.assertEqual(self.requestWriter.updateRequestStatus(schema[0]['RequestName'], "Processing Done"),
'not allowed transition completed to Processing Done')
self.assertEqual(self.requestWriter.updateRequestStatus(schema[0]['RequestName'], "normal-archived"),
'OK')
result = self.requestWriter.getRequestByStatus(["normal-archived"], False, 1)
self.assertEqual(len(result), 1, "should be 1 but %s" % result)
class RequestDBTest(unittest.TestCase):
"""
"""
def setUp(self):
"""
_setUp_
"""
self.schema = []
self.couchApps = ["ReqMgr"]
self.testInit = TestInitCouchApp('RequestDBServiceTest')
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = self.schema,
useDefault = False)
dbName = 'requsetdb_t'
self.testInit.setupCouch(dbName, *self.couchApps)
self.requestWriter = RequestDBWriter(self.testInit.couchUrl, dbName)
self.requestReader = RequestDBReader(self.testInit.couchUrl, dbName)
self.requestWriter.defaultStale = {}
self.requestReader.defaultStale = {}
return
def tearDown(self):
"""
_tearDown_
Drop all the WMBS tables.
"""
self.testInit.tearDownCouch()
def testRequestDBWriter(self):
# test getWork
schema = generate_reqmgr_schema()
result = self.requestWriter.insertGenericRequest(schema[0])
self.assertEquals(len(result), 1, 'insert fail');
self.assertEquals(self.requestWriter.updateRequestStatus(schema[0]['RequestName'], "failed"), 'OK', 'update fail')
self.assertEquals(self.requestWriter.updateRequestStatus("not_exist_schema", "assigned"),
'Error: document not found')
result = self.requestWriter.updateRequestProperty(schema[0]['RequestName'],
{'Teams': ['teamA']})
self.assertEquals(self.requestWriter.updateRequestProperty(schema[0]['RequestName'],
{'Teams': ['teamA']}), 'OK', 'update fail')
self.assertEquals(self.requestWriter.updateRequestProperty("not_exist_schema", {'Teams': 'teamA'}),
'Error: document not found')
result = self.requestWriter.getRequestByNames([schema[0]['RequestName']])
self.assertEquals(len(result), 1, "should be 1")
result = self.requestWriter.getRequestByStatus(["failed"], False, 1)
self.assertEquals(len(result), 1, "should be 1")
class WMStatsTest(unittest.TestCase):
"""
"""
def setUp(self):
"""
_setUp_
"""
self.schema = []
self.couchApps = ["WMStats"]
self.testInit = TestInitCouchApp('WorkQueueServiceTest')
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=self.schema, useDefault=False)
dbName = 'wmstats_t'
self.testInit.setupCouch(dbName, "WMStats")
reqDBName = "reqmgrdb_t"
self.testInit.setupCouch(reqDBName, "ReqMgr")
wmstatsURL = "%s/%s" % (self.testInit.couchUrl, dbName)
reqDBURL = "%s/%s" % (self.testInit.couchUrl, reqDBName)
self.reqDBWriter = RequestDBWriter(reqDBURL)
self.wmstatsReader = WMStatsReader(wmstatsURL, reqdbURL=reqDBURL)
self.wmstatsReader.defaultStale = {}
self.wmstatsReader.reqDB.defaultStale = {}
return
def tearDown(self):
"""
_tearDown_
Drop all the WMBS tables.
"""
self.testInit.tearDownCouch()
def testWMStatsWriter(self):
# test getWork
schema = generate_reqmgr_schema()
result = self.reqDBWriter.insertGenericRequest(schema[0])
self.assertEquals(result[0]['ok'], True, 'insert fail')
result = self.reqDBWriter.updateRequestStatus(schema[0]['RequestName'],
"failed")
self.assertEquals(result, 'OK', 'update fail')
result = self.reqDBWriter.updateRequestStatus("not_exist_schema",
"assigned")
self.assertEquals(result, 'Error: document not found')
result = self.reqDBWriter.updateRequestProperty(
schema[0]['RequestName'], {"Teams": ['teamA']})
self.assertEquals(result, 'OK', 'update fail')
result = self.reqDBWriter.updateRequestProperty(
"not_exist_schema", {"Teams": ['teamA']})
self.assertEquals(result, 'Error: document not found')
totalStats = {
'TotalEstimatedJobs': 100,
'TotalInputEvents': 1000,
'TotalInputLumis': 1234,
'TotalInputFiles': 5
}
result = self.reqDBWriter.updateRequestProperty(
schema[0]['RequestName'], totalStats)
self.assertEquals(result, 'OK', 'update fail')
result = self.reqDBWriter.updateRequestProperty(
schema[0]['RequestName'], totalStats)
self.assertEquals(result, 'OK', 'update fail')
result = self.reqDBWriter.updateRequestProperty(
"not_exist_schema", totalStats)
self.assertEquals(result, 'Error: document not found')
spec1 = newWorkload(schema[0]['RequestName'])
production = spec1.newTask("Production")
production.setTaskType("Merge")
production.setSiteWhitelist(['TEST_SITE'])
properties = {
"RequestPriority": spec1.priority(),
'SiteWhitelist': spec1.getTopLevelTask()[0].siteWhitelist(),
'OutputDatasets': spec1.listOutputDatasets()
}
result = self.reqDBWriter.updateRequestProperty(
spec1.name(), properties)
self.assertEquals(result, 'OK', 'update fail')
spec2 = newWorkload("not_exist_schema")
production = spec2.newTask("Production")
production.setTaskType("Merge")
properties = {
"RequestPriority": spec2.priority(),
'SiteWhitelist': spec2.getTopLevelTask()[0].siteWhitelist(),
'OutputDatasets': spec2.listOutputDatasets()
}
result = self.reqDBWriter.updateRequestProperty(
spec2.name(), properties)
self.assertEquals(result, 'Error: document not found')
requests = self.wmstatsReader.getRequestByStatus(["failed"],
jobInfoFlag=False,
legacyFormat=True)
self.assertEquals(requests.keys(), [schema[0]['RequestName']])
requestCollection = RequestInfoCollection(requests)
result = requestCollection.getJSONData()
self.assertEquals(result.keys(), [schema[0]['RequestName']])
requests = self.wmstatsReader.getActiveData()
self.assertEquals(requests.keys(), [schema[0]['RequestName']])
requests = self.wmstatsReader.getRequestByStatus(["failed"])
self.assertEquals(requests.keys(), [schema[0]['RequestName']])
requests = self.wmstatsReader.getRequestSummaryWithJobInfo(
schema[0]['RequestName'])
self.assertEquals(requests.keys(), [schema[0]['RequestName']])
class WMStatsTest(unittest.TestCase):
"""
"""
def setUp(self):
"""
_setUp_
"""
self.schema = []
self.couchApps = ["WMStats"]
self.testInit = TestInitCouchApp('WorkQueueServiceTest')
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = self.schema,
useDefault = False)
dbName = 'wmstats_t'
self.testInit.setupCouch(dbName, "WMStats")
reqDBName = "reqmgrdb_t"
self.testInit.setupCouch(reqDBName, "ReqMgr")
wmstatsURL = "%s/%s" % (self.testInit.couchUrl, dbName)
reqDBURL = "%s/%s" % (self.testInit.couchUrl, reqDBName)
self.reqDBWriter = RequestDBWriter(reqDBURL)
self.wmstatsReader = WMStatsReader(wmstatsURL, reqdbURL=reqDBURL)
self.wmstatsReader.defaultStale = {}
self.wmstatsReader.reqDB.defaultStale = {}
return
def tearDown(self):
"""
_tearDown_
Drop all the WMBS tables.
"""
self.testInit.tearDownCouch()
def testWMStatsWriter(self):
# test getWork
schema = generate_reqmgr_schema()
result = self.reqDBWriter.insertGenericRequest(schema[0])
self.assertEquals(result[0]['ok'], True, 'insert fail')
result = self.reqDBWriter.updateRequestStatus(schema[0]['RequestName'], "failed")
self.assertEquals(result, 'OK', 'update fail')
result = self.reqDBWriter.updateRequestStatus("not_exist_schema", "assigned")
self.assertEquals(result,'Error: document not found')
result = self.reqDBWriter.updateRequestProperty(schema[0]['RequestName'], {"Teams": ['teamA']})
self.assertEquals(result, 'OK', 'update fail')
result = self.reqDBWriter.updateRequestProperty("not_exist_schema", {"Teams": ['teamA']})
self.assertEquals(result, 'Error: document not found')
totalStats = {'TotalEstimatedJobs': 100, 'TotalInputEvents': 1000, 'TotalInputLumis': 1234, 'TotalInputFiles': 5}
result = self.reqDBWriter.updateRequestProperty(schema[0]['RequestName'], totalStats)
self.assertEquals(result, 'OK', 'update fail')
result = self.reqDBWriter.updateRequestProperty(schema[0]['RequestName'], totalStats)
self.assertEquals(result, 'OK', 'update fail')
result = self.reqDBWriter.updateRequestProperty("not_exist_schema", totalStats)
self.assertEquals(result, 'Error: document not found')
spec1 = newWorkload(schema[0]['RequestName'])
production = spec1.newTask("Production")
production.setTaskType("Merge")
production.setSiteWhitelist(['TEST_SITE'])
properties = {"RequestPriority": spec1.priority(),
'SiteWhitelist': spec1.getTopLevelTask()[0].siteWhitelist(),
'OutputDatasets': spec1.listOutputDatasets()}
result = self.reqDBWriter.updateRequestProperty(spec1.name(), properties)
self.assertEquals(result, 'OK', 'update fail')
spec2 = newWorkload("not_exist_schema")
production = spec2.newTask("Production")
production.setTaskType("Merge")
properties = {"RequestPriority": spec2.priority(),
'SiteWhitelist': spec2.getTopLevelTask()[0].siteWhitelist(),
'OutputDatasets': spec2.listOutputDatasets()}
result = self.reqDBWriter.updateRequestProperty(spec2.name(), properties)
self.assertEquals(result, 'Error: document not found')
requests = self.wmstatsReader.getRequestByStatus(["failed"], jobInfoFlag = False, legacyFormat = True)
self.assertEquals(requests.keys(), [schema[0]['RequestName']])
requestCollection = RequestInfoCollection(requests)
result = requestCollection.getJSONData()
self.assertEquals(result.keys(), [schema[0]['RequestName']])
requests = self.wmstatsReader.getActiveData()
self.assertEquals(requests.keys(), [schema[0]['RequestName']])
requests = self.wmstatsReader.getRequestByStatus(["failed"])
self.assertEquals(requests.keys(), [schema[0]['RequestName']])
requests = self.wmstatsReader.getRequestSummaryWithJobInfo(schema[0]['RequestName'])
self.assertEquals(requests.keys(), [schema[0]['RequestName']])
class TaskArchiverTest(unittest.TestCase):
"""
TestCase for TestTaskArchiver module
"""
_setup_done = False
_teardown = False
_maxMessage = 10
OWNERDN = os.environ['OWNERDN'] if 'OWNERDN' in os.environ else "Generic/OWNERDN"
def setUp(self):
"""
setup for test.
"""
myThread = threading.currentThread()
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection(destroyAllDatabase = True)
self.testInit.setSchema(customModules = ["WMCore.WMBS", "WMComponent.DBS3Buffer"],
useDefault = False)
self.databaseName = "taskarchiver_t_0"
self.testInit.setupCouch("%s/workloadsummary" % self.databaseName, "WorkloadSummary")
self.testInit.setupCouch("%s/jobs" % self.databaseName, "JobDump")
self.testInit.setupCouch("%s/fwjrs" % self.databaseName, "FWJRDump")
self.testInit.setupCouch("wmagent_summary_t", "WMStats")
self.testInit.setupCouch("wmagent_summary_central_t", "WMStats")
self.testInit.setupCouch("stat_summary_t", "SummaryStats")
reqmgrdb = "reqmgrdb_t"
self.testInit.setupCouch(reqmgrdb, "ReqMgr")
reqDBURL = "%s/%s" % (self.testInit.couchUrl, reqmgrdb)
self.requestWriter = RequestDBWriter(reqDBURL)
self.requestWriter.defaultStale = {}
self.daofactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.dbsDaoFactory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.getJobs = self.daofactory(classname = "Jobs.GetAllJobs")
self.inject = self.daofactory(classname = "Workflow.MarkInjectedWorkflows")
self.testDir = self.testInit.generateWorkDir()
os.makedirs(os.path.join(self.testDir, 'specDir'))
self.nJobs = 10
self.campaignName = 'aCampaign'
self.uploadPublishInfo = False
self.uploadPublishDir = None
return
def tearDown(self):
"""
Database deletion
"""
myThread = threading.currentThread()
self.testInit.clearDatabase(modules = ["WMCore.WMBS"])
self.testInit.delWorkDir()
self.testInit.tearDownCouch()
return
def getConfig(self):
"""
_createConfig_
General config file
"""
config = self.testInit.getConfiguration()
#self.testInit.generateWorkDir(config)
config.section_("General")
config.General.workDir = "."
config.section_("JobStateMachine")
config.JobStateMachine.couchurl = os.getenv("COUCHURL", "cmssrv52.fnal.gov:5984")
config.JobStateMachine.couchDBName = self.databaseName
config.JobStateMachine.jobSummaryDBName = 'wmagent_summary_t'
config.JobStateMachine.summaryStatsDBName = 'stat_summary_t'
config.component_("JobCreator")
config.JobCreator.jobCacheDir = os.path.join(self.testDir, 'testDir')
config.component_("TaskArchiver")
config.TaskArchiver.componentDir = self.testDir
config.TaskArchiver.WorkQueueParams = {}
config.TaskArchiver.pollInterval = 60
config.TaskArchiver.logLevel = 'INFO'
config.TaskArchiver.timeOut = 0
config.TaskArchiver.histogramKeys = ['AvgEventTime', 'writeTotalMB', 'jobTime']
config.TaskArchiver.histogramBins = 5
config.TaskArchiver.histogramLimit = 5
config.TaskArchiver.perfPrimaryDatasets = ['SingleMu', 'MuHad', 'MinimumBias']
config.TaskArchiver.perfDashBoardMinLumi = 50
config.TaskArchiver.perfDashBoardMaxLumi = 9000
config.TaskArchiver.dqmUrl = 'https://cmsweb.cern.ch/dqm/dev/'
config.TaskArchiver.dashBoardUrl = 'http://dashboard43.cern.ch/dashboard/request.py/putluminositydata'
config.TaskArchiver.workloadSummaryCouchDBName = "%s/workloadsummary" % self.databaseName
config.TaskArchiver.workloadSummaryCouchURL = config.JobStateMachine.couchurl
config.TaskArchiver.requireCouch = True
config.TaskArchiver.uploadPublishInfo = self.uploadPublishInfo
config.TaskArchiver.uploadPublishDir = self.uploadPublishDir
config.TaskArchiver.userFileCacheURL = os.getenv('UFCURL', 'http://cms-xen38.fnal.gov:7725/userfilecache/')
config.TaskArchiver.ReqMgr2ServiceURL = "https://cmsweb-dev.cern.ch/reqmgr2"
config.TaskArchiver.ReqMgrServiceURL = "https://cmsweb-dev.cern.ch/reqmgr/rest"
config.TaskArchiver.localWMStatsURL = "%s/%s" % (config.JobStateMachine.couchurl, config.JobStateMachine.jobSummaryDBName)
config.component_("AnalyticsDataCollector")
config.AnalyticsDataCollector.centralRequestDBURL = '%s/reqmgrdb_t' % config.JobStateMachine.couchurl
config.AnalyticsDataCollector.RequestCouchApp = "ReqMgr"
config.section_("ACDC")
config.ACDC.couchurl = config.JobStateMachine.couchurl
config.ACDC.database = config.JobStateMachine.couchDBName
# Make the jobCacheDir
os.mkdir(config.JobCreator.jobCacheDir)
# addition for Alerts messaging framework, work (alerts) and control
# channel addresses to which the component will be sending alerts
# these are destination addresses where AlertProcessor:Receiver listens
config.section_("Alert")
config.Alert.address = "tcp://127.0.0.1:5557"
config.Alert.controlAddr = "tcp://127.0.0.1:5559"
config.section_("BossAir")
config.BossAir.UISetupScript = '/afs/cern.ch/cms/LCG/LCG-2/UI/cms_ui_env.sh'
config.BossAir.gliteConf = '/afs/cern.ch/cms/LCG/LCG-2/UI/conf/glite_wms_CERN.conf'
config.BossAir.credentialDir = '/home/crab/ALL_SETUP/credentials/'
config.BossAir.gLiteProcesses = 2
config.BossAir.gLitePrefixEnv = "/lib64/"
config.BossAir.pluginNames = ["gLitePlugin"]
config.BossAir.proxyDir = "/tmp/credentials"
config.BossAir.manualProxyPath = os.environ['X509_USER_PROXY'] if 'X509_USER_PROXY' in os.environ else None
config.section_("Agent")
config.Agent.serverDN = "/we/bypass/myproxy/logon"
return config
def createWorkload(self, workloadName = 'Test', emulator = True):
"""
_createTestWorkload_
Creates a test workload for us to run on, hold the basic necessities.
"""
workload = testWorkload("Tier1ReReco")
taskMaker = TaskMaker(workload, os.path.join(self.testDir, 'workloadTest'))
taskMaker.skipSubscription = True
taskMaker.processWorkload()
workload.setCampaign(self.campaignName)
workload.save(workloadName)
return workload
def createTestJobGroup(self, config, name = "TestWorkthrough",
filesetName = "TestFileset",
specLocation = "spec.xml", error = False,
task = "/TestWorkload/ReReco",
type = "Processing"):
"""
Creates a group of several jobs
"""
myThread = threading.currentThread()
testWorkflow = Workflow(spec = specLocation, owner = self.OWNERDN,
name = name, task = task, owner_vogroup="", owner_vorole="")
testWorkflow.create()
self.inject.execute(names = [name], injected = True)
testWMBSFileset = Fileset(name = filesetName)
testWMBSFileset.create()
testFileA = File(lfn = "/this/is/a/lfnA" , size = 1024, events = 10)
testFileA.addRun(Run(10, *[12312]))
testFileA.setLocation('malpaquet')
testFileB = File(lfn = "/this/is/a/lfnB", size = 1024, events = 10)
testFileB.addRun(Run(10, *[12314]))
testFileB.setLocation('malpaquet')
testFileA.create()
testFileB.create()
testWMBSFileset.addFile(testFileA)
testWMBSFileset.addFile(testFileB)
testWMBSFileset.commit()
testWMBSFileset.markOpen(0)
outputWMBSFileset = Fileset(name = '%sOutput' % filesetName)
outputWMBSFileset.create()
testFileC = File(lfn = "/this/is/a/lfnC" , size = 1024, events = 10)
testFileC.addRun(Run(10, *[12312]))
testFileC.setLocation('malpaquet')
testFileC.create()
outputWMBSFileset.addFile(testFileC)
outputWMBSFileset.commit()
outputWMBSFileset.markOpen(0)
testWorkflow.addOutput('output', outputWMBSFileset)
testSubscription = Subscription(fileset = testWMBSFileset,
workflow = testWorkflow,
type = type)
testSubscription.create()
testJobGroup = JobGroup(subscription = testSubscription)
testJobGroup.create()
for i in range(0,self.nJobs):
testJob = Job(name = makeUUID())
testJob.addFile(testFileA)
testJob.addFile(testFileB)
testJob['retry_count'] = 1
testJob['retry_max'] = 10
testJob['mask'].addRunAndLumis(run = 10, lumis = [12312, 12313])
testJobGroup.add(testJob)
testJobGroup.commit()
changer = ChangeState(config)
report1 = Report()
report2 = Report()
if error:
path1 = os.path.join(WMCore.WMBase.getTestBase(),
"WMComponent_t/JobAccountant_t/fwjrs", "badBackfillJobReport.pkl")
path2 = os.path.join(WMCore.WMBase.getTestBase(),
'WMComponent_t/TaskArchiver_t/fwjrs',
'logCollectReport2.pkl')
else:
path1 = os.path.join(WMCore.WMBase.getTestBase(),
'WMComponent_t/TaskArchiver_t/fwjrs',
'mergeReport1.pkl')
path2 = os.path.join(WMCore.WMBase.getTestBase(),
'WMComponent_t/TaskArchiver_t/fwjrs',
'logCollectReport2.pkl')
report1.load(filename = path1)
report2.load(filename = path2)
changer.propagate(testJobGroup.jobs, 'created', 'new')
changer.propagate(testJobGroup.jobs, 'executing', 'created')
changer.propagate(testJobGroup.jobs, 'complete', 'executing')
for i in range(self.nJobs):
if i < self.nJobs/2:
testJobGroup.jobs[i]['fwjr'] = report1
else:
testJobGroup.jobs[i]['fwjr'] = report2
changer.propagate(testJobGroup.jobs, 'jobfailed', 'complete')
changer.propagate(testJobGroup.jobs, 'jobcooloff', 'jobfailed')
changer.propagate(testJobGroup.jobs, 'created', 'jobcooloff')
changer.propagate(testJobGroup.jobs, 'executing', 'created')
changer.propagate(testJobGroup.jobs, 'complete', 'executing')
changer.propagate(testJobGroup.jobs, 'jobfailed', 'complete')
changer.propagate(testJobGroup.jobs, 'retrydone', 'jobfailed')
changer.propagate(testJobGroup.jobs, 'exhausted', 'retrydone')
changer.propagate(testJobGroup.jobs, 'cleanout', 'exhausted')
testSubscription.completeFiles([testFileA, testFileB])
return testJobGroup
def createGiantJobSet(self, name, config, nSubs = 10, nJobs = 10,
nFiles = 1, spec = "spec.xml"):
"""
Creates a massive set of jobs
"""
jobList = []
for i in range(0, nSubs):
# Make a bunch of subscriptions
localName = '%s-%i' % (name, i)
testWorkflow = Workflow(spec = spec, owner = self.OWNERDN,
name = localName, task="Test", owner_vogroup="", owner_vorole="")
testWorkflow.create()
testWMBSFileset = Fileset(name = localName)
testWMBSFileset.create()
testSubscription = Subscription(fileset = testWMBSFileset,
workflow = testWorkflow)
testSubscription.create()
testJobGroup = JobGroup(subscription = testSubscription)
testJobGroup.create()
filesToComplete = []
for j in range(0, nJobs):
# Create jobs for each subscription
testFileA = File(lfn = "%s-%i-lfnA" % (localName, j) , size = 1024, events = 10)
testFileA.addRun(Run(10, *[11,12,13,14,15,16,17,18,19,20,
21,22,23,24,25,26,27,28,29,30,
31,32,33,34,35,36,37,38,39,40]))
testFileA.setLocation('malpaquet')
testFileA.create()
testWMBSFileset.addFile(testFileA)
testWMBSFileset.commit()
filesToComplete.append(testFileA)
testJob = Job(name = '%s-%i' % (localName, j))
testJob.addFile(testFileA)
testJob['retry_count'] = 1
testJob['retry_max'] = 10
testJobGroup.add(testJob)
jobList.append(testJob)
for k in range(0, nFiles):
# Create output files
testFile = File(lfn = "%s-%i-output" % (localName, k) , size = 1024, events = 10)
testFile.addRun(Run(10, *[12312]))
testFile.setLocation('malpaquet')
testFile.create()
testJobGroup.output.addFile(testFile)
testJobGroup.output.commit()
testJobGroup.commit()
changer = ChangeState(config)
changer.propagate(testJobGroup.jobs, 'created', 'new')
changer.propagate(testJobGroup.jobs, 'executing', 'created')
changer.propagate(testJobGroup.jobs, 'complete', 'executing')
changer.propagate(testJobGroup.jobs, 'success', 'complete')
changer.propagate(testJobGroup.jobs, 'cleanout', 'success')
testWMBSFileset.markOpen(0)
testSubscription.completeFiles(filesToComplete)
return jobList
def getPerformanceFromDQM(self, dqmUrl, dataset, run):
# Make function to fetch this from DQM. Returning Null or False if it fails
getUrl = "%sjsonfairy/archive/%s%s/DQM/TimerService/event_byluminosity" % (dqmUrl, run, dataset)
# Assert if the URL is assembled as expected
if run == 207214:
self.assertEqual('https://cmsweb.cern.ch/dqm/dev/jsonfairy/archive/207214/MinimumBias/Commissioning10-v4/DQM/DQM/TimerService/event_byluminosity',
getUrl)
# let's suppose it works..
testResponseFile = open(os.path.join(getTestBase(),
'WMComponent_t/TaskArchiver_t/DQMGUIResponse.json'), 'r')
response = testResponseFile.read()
testResponseFile.close()
responseJSON = json.loads(response)
return responseJSON
def filterInterestingPerfPoints(self, responseJSON, minLumi, maxLumi):
worthPoints = {}
points = responseJSON["hist"]["bins"]["content"]
for i in range(responseJSON["hist"]["xaxis"]["first"]["id"], responseJSON["hist"]["xaxis"]["last"]["id"]):
# is the point worth it? if yes add to interesting points dictionary.
# 1 - non 0
# 2 - between minimum and maximum expected luminosity
# FIXME : 3 - population in dashboard for the bin interval < 100
# Those should come from the config :
if points[i] == 0:
continue
binSize = responseJSON["hist"]["xaxis"]["last"]["value"]/responseJSON["hist"]["xaxis"]["last"]["id"]
# Fetching the important values
instLuminosity = i*binSize
timePerEvent = points[i]
if instLuminosity > minLumi and instLuminosity < maxLumi :
worthPoints[instLuminosity] = timePerEvent
return worthPoints
def publishPerformanceDashBoard(self, dashBoardUrl, PD, release, worthPoints):
dashboardPayload = []
for instLuminosity in worthPoints :
timePerEvent = int(worthPoints[instLuminosity])
dashboardPayload.append({"primaryDataset" : PD,
"release" : release,
"integratedLuminosity" : instLuminosity,
"timePerEvent" : timePerEvent})
data = "{\"data\":%s}" % str(dashboardPayload).replace("\'","\"")
# let's suppose it works..
testDashBoardPayloadFile = open(os.path.join(getTestBase(),
'WMComponent_t/TaskArchiver_t/DashBoardPayload.json'), 'r')
testDashBoardPayload = testDashBoardPayloadFile.read()
testDashBoardPayloadFile.close()
self.assertEqual(data, testDashBoardPayload)
return True
def populateWorkflowWithCompleteStatus(self, name ="TestWorkload"):
schema = generate_reqmgr_schema(1)
schema[0]["RequestName"] = name
self.requestWriter.insertGenericRequest(schema[0])
result = self.requestWriter.updateRequestStatus(name, "completed")
return result
def testA_BasicFunctionTest(self):
"""
_BasicFunctionTest_
Tests the components, by seeing if they can process a simple set of closeouts
"""
myThread = threading.currentThread()
config = self.getConfig()
workloadPath = os.path.join(self.testDir, 'specDir', 'spec.pkl')
workload = self.createWorkload(workloadName = workloadPath)
testJobGroup = self.createTestJobGroup(config = config,
name = workload.name(),
specLocation = workloadPath,
error = False)
# Create second workload
testJobGroup2 = self.createTestJobGroup(config = config,
name = workload.name(),
filesetName = "TestFileset_2",
specLocation = workloadPath,
task = "/TestWorkload/ReReco/LogCollect",
type = "LogCollect")
cachePath = os.path.join(config.JobCreator.jobCacheDir,
"TestWorkload", "ReReco")
os.makedirs(cachePath)
self.assertTrue(os.path.exists(cachePath))
cachePath2 = os.path.join(config.JobCreator.jobCacheDir,
"TestWorkload", "LogCollect")
os.makedirs(cachePath2)
self.assertTrue(os.path.exists(cachePath2))
result = myThread.dbi.processData("SELECT * FROM wmbs_subscription")[0].fetchall()
self.assertEqual(len(result), 2)
workflowName = "TestWorkload"
dbname = config.TaskArchiver.workloadSummaryCouchDBName
couchdb = CouchServer(config.JobStateMachine.couchurl)
workdatabase = couchdb.connectDatabase(dbname)
jobdb = couchdb.connectDatabase("%s/jobs" % self.databaseName)
fwjrdb = couchdb.connectDatabase("%s/fwjrs" % self.databaseName)
jobs = jobdb.loadView("JobDump", "jobsByWorkflowName",
options = {"startkey": [workflowName],
"endkey": [workflowName, {}]})['rows']
fwjrdb.loadView("FWJRDump", "fwjrsByWorkflowName",
options = {"startkey": [workflowName],
"endkey": [workflowName, {}]})['rows']
self.assertEqual(len(jobs), 2*self.nJobs)
from WMCore.WMBS.CreateWMBSBase import CreateWMBSBase
create = CreateWMBSBase()
tables = []
for x in create.requiredTables:
tables.append(x[2:])
self.populateWorkflowWithCompleteStatus()
testTaskArchiver = TaskArchiverPoller(config = config)
testTaskArchiver.algorithm()
cleanCouch = CleanCouchPoller(config = config)
cleanCouch.setup()
cleanCouch.algorithm()
result = myThread.dbi.processData("SELECT * FROM wmbs_job")[0].fetchall()
self.assertEqual(len(result), 0)
result = myThread.dbi.processData("SELECT * FROM wmbs_subscription")[0].fetchall()
self.assertEqual(len(result), 0)
result = myThread.dbi.processData("SELECT * FROM wmbs_jobgroup")[0].fetchall()
self.assertEqual(len(result), 0)
result = myThread.dbi.processData("SELECT * FROM wmbs_fileset")[0].fetchall()
self.assertEqual(len(result), 0)
result = myThread.dbi.processData("SELECT * FROM wmbs_file_details")[0].fetchall()
self.assertEqual(len(result), 0)
# Make sure we deleted the directory
self.assertFalse(os.path.exists(cachePath))
self.assertFalse(os.path.exists(os.path.join(self.testDir, 'workloadTest/TestWorkload')))
testWMBSFileset = Fileset(id = 1)
self.assertEqual(testWMBSFileset.exists(), False)
workloadSummary = workdatabase.document(id = "TestWorkload")
# Check ACDC
self.assertEqual(workloadSummary['ACDCServer'], sanitizeURL(config.ACDC.couchurl)['url'])
# Check the output
self.assertEqual(workloadSummary['output'].keys(), ['/Electron/MorePenguins-v0/RECO'])
self.assertEqual(sorted(workloadSummary['output']['/Electron/MorePenguins-v0/RECO']['tasks']),
['/TestWorkload/ReReco', '/TestWorkload/ReReco/LogCollect'])
# Check performance
# Check histograms
self.assertAlmostEquals(workloadSummary['performance']['/TestWorkload/ReReco']['cmsRun1']['AvgEventTime']['histogram'][0]['average'],
0.89405199999999996, places = 2)
self.assertEqual(workloadSummary['performance']['/TestWorkload/ReReco']['cmsRun1']['AvgEventTime']['histogram'][0]['nEvents'],
10)
# Check standard performance
self.assertAlmostEquals(workloadSummary['performance']['/TestWorkload/ReReco']['cmsRun1']['TotalJobCPU']['average'], 17.786300000000001,
places = 2)
self.assertAlmostEquals(workloadSummary['performance']['/TestWorkload/ReReco']['cmsRun1']['TotalJobCPU']['stdDev'], 0.0,
places = 2)
# Check worstOffenders
self.assertEqual(workloadSummary['performance']['/TestWorkload/ReReco']['cmsRun1']['AvgEventTime']['worstOffenders'],
[{'logCollect': None, 'log': None, 'value': '0.894052', 'jobID': 1},
{'logCollect': None, 'log': None, 'value': '0.894052', 'jobID': 1},
{'logCollect': None, 'log': None, 'value': '0.894052', 'jobID': 2}])
# Check retryData
self.assertEqual(workloadSummary['retryData']['/TestWorkload/ReReco'], {'1': 10})
logCollectPFN = 'srm://srm-cms.cern.ch:8443/srm/managerv2?SFN=/castor/cern.ch/cms/store/logs/prod/2012/11/WMAgent/Run206446-MinimumBias-Run2012D-v1-Tier1PromptReco-4af7e658-23a4-11e2-96c7-842b2b4671d8/Run206446-MinimumBias-Run2012D-v1-Tier1PromptReco-4af7e658-23a4-11e2-96c7-842b2b4671d8-AlcaSkimLogCollect-1-logs.tar'
self.assertEqual(workloadSummary['logArchives'], {'/TestWorkload/ReReco/LogCollect' : [logCollectPFN for _ in range(10)]})
# LogCollect task is made out of identical FWJRs
# assert that it is identical
for x in workloadSummary['performance']['/TestWorkload/ReReco/LogCollect']['cmsRun1'].keys():
if x in config.TaskArchiver.histogramKeys:
continue
for y in ['average', 'stdDev']:
self.assertAlmostEquals(workloadSummary['performance']['/TestWorkload/ReReco/LogCollect']['cmsRun1'][x][y],
workloadSummary['performance']['/TestWorkload/ReReco']['cmsRun1'][x][y],
places = 2)
return
def testB_testErrors(self):
"""
_testErrors_
Test with a failed FWJR
"""
myThread = threading.currentThread()
config = self.getConfig()
workloadPath = os.path.join(self.testDir, 'specDir', 'spec.pkl')
workload = self.createWorkload(workloadName = workloadPath)
testJobGroup = self.createTestJobGroup(config = config,
name = workload.name(),
specLocation = workloadPath,
error = True)
# Create second workload
testJobGroup2 = self.createTestJobGroup(config = config,
name = workload.name(),
filesetName = "TestFileset_2",
specLocation = workloadPath,
task = "/TestWorkload/ReReco/LogCollect",
type = "LogCollect")
cachePath = os.path.join(config.JobCreator.jobCacheDir,
"TestWorkload", "ReReco")
os.makedirs(cachePath)
self.assertTrue(os.path.exists(cachePath))
couchdb = CouchServer(config.JobStateMachine.couchurl)
jobdb = couchdb.connectDatabase("%s/jobs" % self.databaseName)
fwjrdb = couchdb.connectDatabase("%s/fwjrs" % self.databaseName)
jobdb.loadView("JobDump", "jobsByWorkflowName",
options = {"startkey": [workload.name()],
"endkey": [workload.name(), {}]})['rows']
fwjrdb.loadView("FWJRDump", "fwjrsByWorkflowName",
options = {"startkey": [workload.name()],
"endkey": [workload.name(), {}]})['rows']
self.populateWorkflowWithCompleteStatus()
testTaskArchiver = TaskArchiverPoller(config = config)
testTaskArchiver.algorithm()
cleanCouch = CleanCouchPoller(config = config)
cleanCouch.setup()
cleanCouch.algorithm()
dbname = getattr(config.JobStateMachine, "couchDBName")
workdatabase = couchdb.connectDatabase("%s/workloadsummary" % dbname)
workloadSummary = workdatabase.document(id = workload.name())
self.assertEqual(workloadSummary['errors']['/TestWorkload/ReReco']['failureTime'], 500)
self.assertTrue('99999' in workloadSummary['errors']['/TestWorkload/ReReco']['cmsRun1'])
failedRunInfo = workloadSummary['errors']['/TestWorkload/ReReco']['cmsRun1']['99999']['runs']
self.assertEqual(failedRunInfo, {'10' : [[12312, 12312]]},
"Wrong lumi information in the summary for failed jobs")
# Check the failures by site histograms
self.assertEqual(workloadSummary['histograms']['workflowLevel']['failuresBySite']['data']['T1_IT_CNAF']['Failed Jobs'], 10)
self.assertEqual(workloadSummary['histograms']['stepLevel']['/TestWorkload/ReReco']['cmsRun1']['errorsBySite']['data']['T1_IT_CNAF']['99999'], 10)
self.assertEqual(workloadSummary['histograms']['stepLevel']['/TestWorkload/ReReco']['cmsRun1']['errorsBySite']['data']['T1_IT_CNAF']['8020'], 10)
self.assertEqual(workloadSummary['histograms']['workflowLevel']['failuresBySite']['average']['Failed Jobs'], 10)
self.assertEqual(workloadSummary['histograms']['stepLevel']['/TestWorkload/ReReco']['cmsRun1']['errorsBySite']['average']['99999'], 10)
self.assertEqual(workloadSummary['histograms']['stepLevel']['/TestWorkload/ReReco']['cmsRun1']['errorsBySite']['average']['8020'], 10)
self.assertEqual(workloadSummary['histograms']['workflowLevel']['failuresBySite']['stdDev']['Failed Jobs'], 0)
self.assertEqual(workloadSummary['histograms']['stepLevel']['/TestWorkload/ReReco']['cmsRun1']['errorsBySite']['stdDev']['99999'], 0)
self.assertEqual(workloadSummary['histograms']['stepLevel']['/TestWorkload/ReReco']['cmsRun1']['errorsBySite']['stdDev']['8020'], 0)
return
def testC_Profile(self):
"""
_Profile_
DON'T RUN THIS!
"""
return
import cProfile, pstats
myThread = threading.currentThread()
name = makeUUID()
config = self.getConfig()
jobList = self.createGiantJobSet(name = name, config = config,
nSubs = 10, nJobs = 1000, nFiles = 10)
cleanCouch = CleanCouchPoller(config = config)
cleanCouch.setup()
cProfile.runctx("cleanCouch.algorithm()", globals(), locals(), filename = "testStats.stat")
p = pstats.Stats('testStats.stat')
p.sort_stats('cumulative')
p.print_stats()
return
def testD_Timing(self):
"""
_Timing_
This is to see how fast things go.
"""
return
myThread = threading.currentThread()
name = makeUUID()
config = self.getConfig()
jobList = self.createGiantJobSet(name = name, config = config, nSubs = 10,
nJobs = 1000, nFiles = 10)
testTaskArchiver = TaskArchiverPoller(config = config)
startTime = time.time()
testTaskArchiver.algorithm()
stopTime = time.time()
result = myThread.dbi.processData("SELECT * FROM wmbs_job")[0].fetchall()
self.assertEqual(len(result), 0)
result = myThread.dbi.processData("SELECT * FROM wmbs_subscription")[0].fetchall()
self.assertEqual(len(result), 0)
result = myThread.dbi.processData("SELECT * FROM wmbs_jobgroup")[0].fetchall()
self.assertEqual(len(result), 0)
result = myThread.dbi.processData("SELECT * FROM wmbs_file_details")[0].fetchall()
self.assertEqual(len(result), 0)
testWMBSFileset = Fileset(id = 1)
self.assertEqual(testWMBSFileset.exists(), False)
logging.info("TaskArchiver took %f seconds" % (stopTime - startTime))
def testDQMRecoPerformanceToDashBoard(self):
myThread = threading.currentThread()
listRunsWorkflow = self.dbsDaoFactory(classname="ListRunsWorkflow")
# Didn't like to have done that, but the test doesn't provide all info I need in the system, so faking it:
myThread.dbi.processData("""insert into dbsbuffer_workflow(id, name) values (1, 'TestWorkload')"""
, transaction = False)
myThread.dbi.processData("""insert into dbsbuffer_file (id, lfn, dataset_algo, workflow) values (1, '/store/t/e/s/t.test', 1, 1)"""
, transaction = False)
myThread.dbi.processData("""insert into dbsbuffer_file (id, lfn, dataset_algo, workflow) values (2, '/store/t/e/s/t.test2', 1, 1)"""
, transaction = False)
myThread.dbi.processData("""insert into dbsbuffer_file_runlumi_map (run, lumi, filename) values (207214, 100, 1)"""
, transaction = False)
myThread.dbi.processData("""insert into dbsbuffer_file_runlumi_map (run, lumi, filename) values (207215, 200, 2)"""
, transaction = False)
config = self.getConfig()
dqmUrl = getattr(config.TaskArchiver, "dqmUrl")
perfDashBoardMinLumi = getattr(config.TaskArchiver, "perfDashBoardMinLumi")
perfDashBoardMaxLumi = getattr(config.TaskArchiver, "perfDashBoardMaxLumi")
dashBoardUrl = getattr(config.TaskArchiver, "dashBoardUrl")
workloadPath = os.path.join(self.testDir, 'specDir', 'spec.pkl')
workload = self.createWorkload(workloadName = workloadPath)
testJobGroup = self.createTestJobGroup(config = config,
name = workload.name(),
specLocation = workloadPath,
error = True)
testJobGroup2 = self.createTestJobGroup(config = config,
name = workload.name(),
filesetName = "TestFileset_2",
specLocation = workloadPath,
task = "/TestWorkload/ReReco/LogCollect",
type = "LogCollect")
# Adding request type as ReReco, real ReqMgr requests have it
workload.data.request.section_("schema")
workload.data.request.schema.RequestType = "ReReco"
workload.data.request.schema.CMSSWVersion = 'test_compops_CMSSW_5_3_6_patch1'
workload.getTask('ReReco').addInputDataset(primary='a',processed='b',tier='c')
interestingPDs = getattr(config.TaskArchiver, "perfPrimaryDatasets")
interestingDatasets = []
# Are the datasets from this request interesting? Do they have DQM output? One might ask afterwards if they have harvest
for dataset in workload.listOutputDatasets():
(nothing, PD, procDataSet, dataTier) = dataset.split('/')
if PD in interestingPDs and dataTier == "DQM":
interestingDatasets.append(dataset)
# We should have found 1 interesting dataset
self.assertAlmostEquals(len(interestingDatasets), 1)
if len(interestingDatasets) == 0 :
return
# Request will be only interesting for performance if it's a ReReco or PromptReco
(isReReco, isPromptReco) = (False, False)
if getattr(workload.data.request.schema, "RequestType", None) == 'ReReco':
isReReco=True
# Yes, few people like magic strings, but have a look at :
# https://github.com/dmwm/T0/blob/master/src/python/T0/RunConfig/RunConfigAPI.py#L718
# Might be safe enough
# FIXME: in TaskArchiver, add a test to make sure that the dataset makes sense (procDataset ~= /a/ERA-PromptReco-vVERSON/DQM)
if re.search('PromptReco', workload.name()):
isPromptReco = True
if not (isReReco or isPromptReco):
return
self.assertTrue(isReReco)
self.assertFalse(isPromptReco)
# We are not interested if it's not a PromptReco or a ReReco
if (isReReco or isPromptReco) == False:
return
if isReReco :
release = getattr(workload.data.request.schema, "CMSSWVersion")
if not release :
logging.info("no release for %s, bailing out" % workload.name())
else :
release = getattr(workload.tasks.Reco.steps.cmsRun1.application.setup, "cmsswVersion")
if not release :
logging.info("no release for %s, bailing out" % workload.name())
self.assertEqual(release, "test_compops_CMSSW_5_3_6_patch1")
# If all is true, get the run numbers processed by this worklfow
runList = listRunsWorkflow.execute(workflow = workload.name())
self.assertEqual([207214, 207215], runList)
# GO to DQM GUI, get what you want
# https://cmsweb.cern.ch/dqm/offline/jsonfairy/archive/211313/PAMuon/HIRun2013-PromptReco-v1/DQM/DQM/TimerService/event
for dataset in interestingDatasets :
(nothing, PD, procDataSet, dataTier) = dataset.split('/')
worthPoints = {}
for run in runList :
responseJSON = self.getPerformanceFromDQM(dqmUrl, dataset, run)
worthPoints.update(self.filterInterestingPerfPoints(responseJSON, perfDashBoardMinLumi, perfDashBoardMaxLumi))
# Publish dataset performance to DashBoard.
if self.publishPerformanceDashBoard(dashBoardUrl, PD, release, worthPoints) == False:
logging.info("something went wrong when publishing dataset %s to DashBoard" % dataset)
return
# Requires a running UserFileCache to succeed. https://cmsweb.cern.ch worked for me
# The environment variable OWNERDN needs to be set. Used to retrieve an already delegated proxy and contact the ufc
@attr('integration')
def testPublishJSONCreate(self):
"""
Re-run testA_BasicFunctionTest with data in DBSBuffer
Make sure files are generated
"""
# Set up uploading and write them elsewhere since the test deletes them.
self.uploadPublishInfo = True
self.uploadPublishDir = self.testDir
# Insert some DBSFiles
testFileChildA = DBSBufferFile(lfn = "/this/is/a/child/lfnA", size = 1024, events = 20)
testFileChildA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileChildB = DBSBufferFile(lfn = "/this/is/a/child/lfnB", size = 1024, events = 20)
testFileChildB.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileChildC = DBSBufferFile(lfn = "/this/is/a/child/lfnC", size = 1024, events = 20)
testFileChildC.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileChildA.setDatasetPath("/Cosmics/USER-DATASET1-v1/USER")
testFileChildB.setDatasetPath("/Cosmics/USER-DATASET1-v1/USER")
testFileChildC.setDatasetPath("/Cosmics/USER-DATASET2-v1/USER")
testFileChildA.create()
testFileChildB.create()
testFileChildC.create()
testFile = DBSBufferFile(lfn = "/this/is/a/lfn", size = 1024, events = 10)
testFile.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFile.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO")
testFile.create()
testFileChildA.addParents([testFile["lfn"]])
testFileChildB.addParents([testFile["lfn"]])
testFileChildC.addParents([testFile["lfn"]])
myThread = threading.currentThread()
self.dbsDaoFactory = DAOFactory(package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi)
self.insertWorkflow = self.dbsDaoFactory(classname="InsertWorkflow")
workflowID = self.insertWorkflow.execute(requestName='TestWorkload', taskPath='TestWorkload/Production',
blockMaxCloseTime=100, blockMaxFiles=100,
blockMaxEvents=100, blockMaxSize=100)
myThread.dbi.processData("update dbsbuffer_file set workflow=1 where id < 4")
# Run the test again
self.testA_BasicFunctionTest()
# Reset default values
self.uploadPublishInfo = False
self.uploadPublishDir = None
# Make sure the files are there
self.assertTrue(os.path.exists( os.path.join(self.testDir, 'TestWorkload_publish.json')))
self.assertTrue(os.path.getsize(os.path.join(self.testDir, 'TestWorkload_publish.json')) > 100)
self.assertTrue(os.path.exists( os.path.join(self.testDir, 'TestWorkload_publish.tgz' )))
return
class Tier0PluginTest(unittest.TestCase):
def setUp(self):
"""
_setUp_
Setup the test environment
"""
self.testInit = TestInit(__file__)
self.testInit.setDatabaseConnection()
self.testInit.setSchema(["WMCore.WMBS"])
self.requestCouchDB = 'wmstats_plugin_t'
self.testInit.setupCouch(self.requestCouchDB, 'T0Request')
self.testDir = self.testInit.generateWorkDir()
reqDBURL = "%s/%s" % (os.environ['COUCHURL'], self.requestCouchDB)
self.requestDBWriter = RequestDBWriter(reqDBURL, couchapp="T0Request")
self.requestDBWriter._setNoStale()
self.stateMap = {}
self.orderedStates = []
self.plugin = None
return
def tearDown(self):
"""
_tearDown_
Clear databases and delete files
"""
self.testInit.tearDownCouch()
self.testInit.clearDatabase()
self.testInit.delWorkDir()
return
def setupRepackWorkflow(self):
"""
_setupRepackWorkflow_
Populate WMBS with a repack-like workflow,
every subscription must be unfinished at first
"""
workflowName = 'Repack_Run481516_StreamZ'
mergeTasks = ['RepackMergewrite_QuadElectron_RAW', 'RepackMergewrite_TriPhoton_RAW',
'RepackMergewrite_SingleNeutrino_RAW']
self.stateMap = {'Merge': [],
'Processing Done': []}
self.orderedStates = ['Merge', 'Processing Done']
# Populate WMStats
self.requestDBWriter.insertGenericRequest({'RequestName': workflowName})
self.requestDBWriter.updateRequestStatus(workflowName, 'Closed')
# Create a wmspec in disk
workload = newWorkload(workflowName)
repackTask = workload.newTask('Repack')
for task in mergeTasks:
repackTask.addTask(task)
repackTask.addTask('RepackCleanupUnmergedwrite_QuadElectron_RAW')
specPath = os.path.join(self.testDir, 'Repack.pkl')
workload.save(specPath)
# Populate WMBS
topFileset = Fileset(name='TestStreamerFileset')
topFileset.create()
options = {'spec': specPath, 'owner': 'ItsAMeMario',
'name': workflowName, 'wfType': 'tier0'}
topLevelWorkflow = Workflow(task='/%s/Repack' % workflowName,
**options)
topLevelWorkflow.create()
topLevelSub = Subscription(topFileset, topLevelWorkflow)
topLevelSub.create()
self.stateMap['Merge'].append(topFileset)
for task in mergeTasks:
mergeWorkflow = Workflow(task='/%s/Repack/%s' % (workflowName, task), **options)
mergeWorkflow.create()
unmergedFileset = Fileset(name='TestUnmergedFileset%s' % task)
unmergedFileset.create()
mergeSub = Subscription(unmergedFileset, mergeWorkflow)
mergeSub.create()
self.stateMap['Processing Done'].append(unmergedFileset)
cleanupWorkflow = Workflow(task='/Repack_Run481516_StreamZ/Repack/RepackCleanupUnmergedwrite_QuadElectron_RAW',
**options)
cleanupWorkflow.create()
unmergedFileset = Fileset(name='TestUnmergedFilesetToCleanup')
unmergedFileset.create()
cleanupSub = Subscription(unmergedFileset, cleanupWorkflow)
cleanupSub.create()
return
def setupExpressWorkflow(self):
"""
_setupExpressWorkflow_
Populate WMBS with a express-like workflow,
every subscription must be unfinished at first
"""
workflowName = 'Express_Run481516_StreamZFast'
secondLevelTasks = ['ExpressMergewrite_StreamZFast_DQM', 'ExpressMergewrite_ExpressPhysics_FEVT',
'ExpressAlcaSkimwrite_StreamZFast_ALCARECO', 'ExpressCleanupUnmergedwrite_StreamZFast_DQM',
'ExpressCleanupUnmergedwrite_ExpressPhysics_FEVT',
'ExpressCleanupUnmergedwrite_StreamZFast_ALCARECO']
alcaHarvestTask = 'ExpressAlcaSkimwrite_StreamZFast_ALCARECOAlcaHarvestALCARECOStreamPromptCalibProd'
dqmHarvestTask = 'ExpressMergewrite_StreamZFast_DQMEndOfRunDQMHarvestMerged'
self.stateMap = {'Merge': [],
'Harvesting': [],
'Processing Done': []}
self.orderedStates = ['Merge', 'Harvesting', 'Processing Done']
# Populate WMStats
self.requestDBWriter.insertGenericRequest({'RequestName': workflowName})
self.requestDBWriter.updateRequestStatus(workflowName, 'Closed')
# Create a wmspec in disk
workload = newWorkload(workflowName)
expressTask = workload.newTask('Express')
for task in secondLevelTasks:
secondLevelTask = expressTask.addTask(task)
if task == 'ExpressAlcaSkimwrite_StreamZFast_ALCARECO':
secondLevelTask.addTask(alcaHarvestTask)
elif task == 'ExpressMergewrite_StreamZFast_DQM':
secondLevelTask.addTask(dqmHarvestTask)
specPath = os.path.join(self.testDir, 'Express.pkl')
workload.save(specPath)
# Populate WMBS
sharedFileset = Fileset(name='TestFileset')
sharedFileset.create()
sharedFileset.markOpen(False)
options = {'spec': specPath, 'owner': 'ItsAMeMario',
'name': workflowName, 'wfType': 'tier0'}
topLevelWorkflow = Workflow(task='/%s/Express' % workflowName,
**options)
topLevelWorkflow.create()
topLevelSub = Subscription(sharedFileset, topLevelWorkflow)
topLevelSub.create()
self.stateMap['Merge'].append(topLevelSub)
for task in [x for x in secondLevelTasks if not x.count('CleanupUnmerged')]:
secondLevelWorkflow = Workflow(task='/%s/Express/%s' % (workflowName, task), **options)
secondLevelWorkflow.create()
mergeSub = Subscription(sharedFileset, secondLevelWorkflow)
mergeSub.create()
self.stateMap['Harvesting'].append(mergeSub)
for (parent, child) in [('ExpressAlcaSkimwrite_StreamZFast_ALCARECO', alcaHarvestTask),
('ExpressMergewrite_StreamZFast_DQM', dqmHarvestTask)]:
harvestingWorkflow = Workflow(task='/%s/Express/%s/%s' % (workflowName, parent, child),
**options)
harvestingWorkflow.create()
harvestingSub = Subscription(sharedFileset, harvestingWorkflow)
harvestingSub.create()
self.stateMap['Processing Done'].append(harvestingSub)
return
def setupPromptRecoWorkflow(self):
"""
_setupPromptRecoWorkflow_
Populate WMBS with a real PromptReco workflow,
every subscription must be unfinished at first
"""
# Populate disk and WMBS
testArguments = PromptRecoWorkloadFactory.getTestArguments()
workflowName = 'PromptReco_Run195360_Cosmics'
factory = PromptRecoWorkloadFactory()
testArguments["EnableHarvesting"] = True
testArguments["CouchURL"] = os.environ["COUCHURL"]
workload = factory.factoryWorkloadConstruction(workflowName, testArguments)
wmbsHelper = WMBSHelper(workload, 'Reco', 'SomeBlock', cachepath=self.testDir)
wmbsHelper.createTopLevelFileset()
wmbsHelper._createSubscriptionsInWMBS(wmbsHelper.topLevelTask, wmbsHelper.topLevelFileset)
self.stateMap = {'AlcaSkim': [],
'Merge': [],
'Harvesting': [],
'Processing Done': []}
self.orderedStates = ['AlcaSkim', 'Merge', 'Harvesting', 'Processing Done']
# Populate WMStats
self.requestDBWriter.insertGenericRequest({'RequestName': workflowName})
self.requestDBWriter.updateRequestStatus(workflowName, 'Closed')
topLevelTask = '/%s/Reco' % workflowName
alcaSkimTask = '%s/AlcaSkim' % topLevelTask
mergeTasks = ['%s/AlcaSkim/AlcaSkimMergeALCARECOStreamHcalCalHOCosmics',
'%s/AlcaSkim/AlcaSkimMergeALCARECOStreamTkAlCosmics0T',
'%s/AlcaSkim/AlcaSkimMergeALCARECOStreamMuAlGlobalCosmics',
'%s/RecoMergewrite_AOD',
'%s/RecoMergewrite_DQM',
'%s/RecoMergewrite_RECO']
harvestingTask = '%s/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged' % topLevelTask
self.stateMap['AlcaSkim'].append(wmbsHelper.topLevelSubscription)
alcaSkimWorkflow = Workflow(name=workflowName, task=alcaSkimTask)
alcaSkimWorkflow.load()
alcarecoFileset = Fileset(name='/PromptReco_Run195360_Cosmics/Reco/unmerged-write_ALCARECOALCARECO')
alcarecoFileset.load()
alcaSkimSub = Subscription(alcarecoFileset, alcaSkimWorkflow)
alcaSkimSub.load()
self.stateMap['Merge'].append(alcaSkimSub)
for task in mergeTasks:
mergeTask = task % topLevelTask
mergeWorkflow = Workflow(name=workflowName, task=mergeTask)
mergeWorkflow.load()
if 'AlcaSkim' in mergeTask:
stream = mergeTask.split('/')[-1][13:]
unmergedFileset = Fileset(name='%s/unmerged-%sALCARECO' % (alcaSkimTask, stream))
unmergedFileset.load()
else:
dataTier = mergeTask.split('/')[-1].split('_')[-1]
unmergedFileset = Fileset(name='%s/unmerged-write_%s%s' % (topLevelTask, dataTier, dataTier))
unmergedFileset.load()
mergeSub = Subscription(unmergedFileset, mergeWorkflow)
mergeSub.load()
self.stateMap['Harvesting'].append(mergeSub)
harvestingWorkflow = Workflow(name=workflowName, task=harvestingTask)
harvestingWorkflow.load()
harvestingFileset = Fileset(name='/PromptReco_Run195360_Cosmics/Reco/RecoMergewrite_DQM/merged-MergedDQM')
harvestingFileset.load()
harvestingSub = Subscription(harvestingFileset, harvestingWorkflow)
harvestingSub.load()
self.stateMap['Processing Done'].append(harvestingSub)
return
def verifyStateTransitions(self, transitionMethod='markFinished', transitionTrigger=True):
"""
_verifyStateTransitions_
Utility method which goes through the list of states in self.orderedStates and
finishes the tasks that demand a state transition in each step. This according
to the defined transition method and trigger.
It verifies that the request document in WMStats is moving according to the transitions
"""
for idx in range(0, len(self.orderedStates) * 2):
nextState = self.orderedStates[idx / 2]
if (idx / 2) == 0:
currentState = 'Closed'
else:
currentState = self.orderedStates[idx / 2 - 1]
if idx % 2 == 0:
for transitionObject in self.stateMap[nextState][:-1]:
method = getattr(transitionObject, transitionMethod)
method(transitionTrigger)
self.plugin([], self.requestDBWriter, self.requestDBWriter)
currentStateWorkflows = self.requestDBWriter.getRequestByStatus([currentState])
nextStateWorkflows = self.requestDBWriter.getRequestByStatus([nextState])
self.assertEqual(len(currentStateWorkflows), 1, 'Workflow moved incorrectly from %s' % currentState)
self.assertEqual(len(nextStateWorkflows), 0, 'Workflow moved incorrectly to %s' % nextState)
else:
transitionObject = self.stateMap[nextState][-1]
method = getattr(transitionObject, transitionMethod)
method(transitionTrigger)
self.plugin([], self.requestDBWriter, self.requestDBWriter)
currentStateWorkflows = self.requestDBWriter.getRequestByStatus([currentState])
nextStateWorkflows = self.requestDBWriter.getRequestByStatus([nextState])
self.assertEqual(len(currentStateWorkflows), 0,
'Workflow did not move correctly from %s' % currentState)
self.assertEqual(len(nextStateWorkflows), 1, 'Workflow did not move correctly to %s' % nextState)
return
def testA_RepackStates(self):
"""
_testA_RepackStates_
Setup an environment with a Repack workflow
and traverse through the different states.
Check that the transitions are sane.
"""
# Set the environment
self.setupRepackWorkflow()
self.plugin = Tier0Plugin()
# Verify the transitions
self.verifyStateTransitions('markOpen', False)
return
def testB_ExpressStates(self):
"""
_testB_ExpressStates_
Setup an environment with a Express workflow
and traverse through the different states.
Check that the transitions are sane.
"""
# Set the environment
self.setupExpressWorkflow()
self.plugin = Tier0Plugin()
# Verify the transitions
self.verifyStateTransitions()
return
def testC_PromptRecoStates(self):
"""
_testC_PromptRecoStates_
Setup an environment with a PromptReco workflow
and traverse through the different states.
Check that the transitions are sane.
"""
# Set the environment
self.setupPromptRecoWorkflow()
self.plugin = Tier0Plugin()
# Verify the transitions
self.verifyStateTransitions()
return
class Tier0FeederPoller(BaseWorkerThread):
def __init__(self, config):
"""
_init_
"""
BaseWorkerThread.__init__(self)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package = "T0.WMBS",
logger = logging,
dbinterface = myThread.dbi)
self.tier0ConfigFile = config.Tier0Feeder.tier0ConfigFile
self.specDirectory = config.Tier0Feeder.specDirectory
self.dropboxuser = getattr(config.Tier0Feeder, "dropboxuser", None)
self.dropboxpass = getattr(config.Tier0Feeder, "dropboxpass", None)
self.transferSystemBaseDir = getattr(config.Tier0Feeder, "transferSystemBaseDir", None)
if self.transferSystemBaseDir != None:
if not os.path.exists(self.transferSystemBaseDir):
self.transferSystemBaseDir = None
self.dqmUploadProxy = getattr(config.Tier0Feeder, "dqmUploadProxy", None)
self.serviceProxy = getattr(config.Tier0Feeder, "serviceProxy", None)
self.localRequestCouchDB = RequestDBWriter(config.AnalyticsDataCollector.localT0RequestDBURL,
couchapp = config.AnalyticsDataCollector.RequestCouchApp)
hltConfConnectUrl = config.HLTConfDatabase.connectUrl
dbFactoryHltConf = DBFactory(logging, dburl = hltConfConnectUrl, options = {})
dbInterfaceHltConf = dbFactoryHltConf.connect()
daoFactoryHltConf = DAOFactory(package = "T0.WMBS",
logger = logging,
dbinterface = dbInterfaceHltConf)
self.getHLTConfigDAO = daoFactoryHltConf(classname = "RunConfig.GetHLTConfig")
storageManagerConnectUrl = config.StorageManagerDatabase.connectUrl
dbFactoryStorageManager = DBFactory(logging, dburl = storageManagerConnectUrl, options = {})
self.dbInterfaceStorageManager = dbFactoryStorageManager.connect()
self.getExpressReadyRunsDAO = None
if hasattr(config, "PopConLogDatabase"):
popConLogConnectUrl = getattr(config.PopConLogDatabase, "connectUrl", None)
if popConLogConnectUrl != None:
dbFactoryPopConLog = DBFactory(logging, dburl = popConLogConnectUrl, options = {})
dbInterfacePopConLog = dbFactoryPopConLog.connect()
daoFactoryPopConLog = DAOFactory(package = "T0.WMBS",
logger = logging,
dbinterface = dbInterfacePopConLog)
self.getExpressReadyRunsDAO = daoFactoryPopConLog(classname = "Tier0Feeder.GetExpressReadyRuns")
self.haveT0DataSvc = False
if hasattr(config, "T0DataSvcDatabase"):
t0datasvcConnectUrl = getattr(config.T0DataSvcDatabase, "connectUrl", None)
if t0datasvcConnectUrl != None:
self.haveT0DataSvc = True
dbFactoryT0DataSvc = DBFactory(logging, dburl = t0datasvcConnectUrl, options = {})
dbInterfaceT0DataSvc = dbFactoryT0DataSvc.connect()
self.daoFactoryT0DataSvc = DAOFactory(package = "T0.WMBS",
logger = logging,
dbinterface = dbInterfaceT0DataSvc)
return
def algorithm(self, parameters = None):
"""
_algorithm_
"""
logging.debug("Running Tier0Feeder algorithm...")
myThread = threading.currentThread()
findNewRunsDAO = self.daoFactory(classname = "Tier0Feeder.FindNewRuns")
findNewRunStreamsDAO = self.daoFactory(classname = "Tier0Feeder.FindNewRunStreams")
findNewExpressRunsDAO = self.daoFactory(classname = "Tier0Feeder.FindNewExpressRuns")
releaseExpressDAO = self.daoFactory(classname = "Tier0Feeder.ReleaseExpress")
feedStreamersDAO = self.daoFactory(classname = "Tier0Feeder.FeedStreamers")
markWorkflowsInjectedDAO = self.daoFactory(classname = "Tier0Feeder.MarkWorkflowsInjected")
tier0Config = None
try:
tier0Config = loadConfigurationFile(self.tier0ConfigFile)
except:
# usually happens when there are syntax errors in the configuration
logging.exception("Cannot load Tier0 configuration file, not configuring new runs and run/streams")
# only configure new runs and run/streams if we have a valid Tier0 configuration
if tier0Config != None:
#
# find new runs, setup global run settings and stream/dataset/trigger mapping
#
runHltkeys = findNewRunsDAO.execute(transaction = False)
for run, hltkey in sorted(runHltkeys.items()):
hltConfig = None
# local runs have no hltkey and are configured differently
if hltkey != None:
# retrieve HLT configuration and make sure it's usable
try:
hltConfig = self.getHLTConfigDAO.execute(hltkey, transaction = False)
if hltConfig['process'] == None or len(hltConfig['mapping']) == 0:
raise RuntimeError, "HLTConfDB query returned no process or mapping"
except:
logging.exception("Can't retrieve hltkey %s for run %d" % (hltkey, run))
continue
try:
RunConfigAPI.configureRun(tier0Config, run, hltConfig)
except:
logging.exception("Can't configure for run %d" % (run))
#
# find unconfigured run/stream with data
# populate RunConfig, setup workflows/filesets/subscriptions
#
runStreams = findNewRunStreamsDAO.execute(transaction = False)
for run in sorted(runStreams.keys()):
for stream in sorted(runStreams[run]):
try:
RunConfigAPI.configureRunStream(tier0Config,
run, stream,
self.specDirectory,
self.dqmUploadProxy)
except:
logging.exception("Can't configure for run %d and stream %s" % (run, stream))
#
# end runs which are active and have ended according to the EoR StorageManager records
#
RunLumiCloseoutAPI.endRuns(self.dbInterfaceStorageManager)
#
# release runs for Express
#
runs = findNewExpressRunsDAO.execute(transaction = False)
if len(runs) > 0:
binds = []
for run in runs:
binds.append( { 'RUN' : run } )
if self.getExpressReadyRunsDAO != None:
runs = self.getExpressReadyRunsDAO.execute(binds = binds, transaction = False)
if len(runs) > 0:
binds = []
for run in runs:
binds.append( { 'RUN' : run } )
releaseExpressDAO.execute(binds = binds, transaction = False)
#
# release runs for PromptReco
#
RunConfigAPI.releasePromptReco(tier0Config,
self.specDirectory,
self.dqmUploadProxy)
#
# insert express and reco configs into Tier0 Data Service
#
if self.haveT0DataSvc:
self.updateExpressConfigsT0DataSvc()
self.updateRecoConfigsT0DataSvc()
self.updateRecoReleaseConfigsT0DataSvc()
#
# mark express and repack workflows as injected if certain conditions are met
# (we don't do it immediately to prevent the TaskArchiver from cleaning up too early)
#
markWorkflowsInjectedDAO.execute(self.transferSystemBaseDir != None,
transaction = False)
#
# close stream/lumis for run/streams that are active (fileset exists and open)
#
RunLumiCloseoutAPI.closeLumiSections(self.dbInterfaceStorageManager)
#
# feed new data into exisiting filesets
#
try:
myThread.transaction.begin()
feedStreamersDAO.execute(conn = myThread.transaction.conn, transaction = True)
except:
logging.exception("Can't feed data, bailing out...")
raise
else:
myThread.transaction.commit()
#
# run ended and run/stream fileset open
# => check for complete lumi_closed record, all lumis finally closed and all data feed
# => if all conditions satisfied, close the run/stream fileset
#
RunLumiCloseoutAPI.closeRunStreamFilesets()
#
# check and delete active split lumis
#
RunLumiCloseoutAPI.checkActiveSplitLumis()
#
# insert workflows into CouchDB for monitoring
#
self.feedCouchMonitoring()
#
# Update Couch when Repack and Express have closed input filesets (analog to old T0 closeout)
#
self.closeOutRealTimeWorkflows()
#
# send repacked notifications to StorageManager
#
if self.transferSystemBaseDir != None:
self.notifyStorageManager()
#
# upload PCL conditions to DropBox
#
ConditionUploadAPI.uploadConditions(self.dropboxuser, self.dropboxpass, self.serviceProxy)
return
def feedCouchMonitoring(self):
"""
_feedCouchMonitoring_
check for workflows that haven't been uploaded to Couch for monitoring yet
"""
getStreamerWorkflowsForMonitoringDAO = self.daoFactory(classname = "Tier0Feeder.GetStreamerWorkflowsForMonitoring")
getPromptRecoWorkflowsForMonitoringDAO = self.daoFactory(classname = "Tier0Feeder.GetPromptRecoWorkflowsForMonitoring")
markTrackedWorkflowMonitoringDAO = self.daoFactory(classname = "Tier0Feeder.MarkTrackedWorkflowMonitoring")
workflows = getStreamerWorkflowsForMonitoringDAO.execute()
workflows += getPromptRecoWorkflowsForMonitoringDAO.execute()
if len(workflows) == 0:
logging.debug("No workflows to publish to couch monitoring, doing nothing")
if workflows:
logging.debug(" Going to publish %d workflows" % len(workflows))
for (workflowId, run, workflowName) in workflows:
logging.info(" Publishing workflow %s to monitoring" % workflowName)
#TODO: add more information about workflow if there need to be kept longer than
# workflow life cycle.
doc = {}
doc["RequestName"] = workflowName
doc["Run"] = run
response = self.localRequestCouchDB.insertGenericRequest(doc)
if response == "OK" or "EXISTS":
logging.info(" Successfully uploaded request %s" % workflowName)
# Here we have to trust the insert, if it doesn't happen will be easy to spot on the logs
markTrackedWorkflowMonitoringDAO.execute(workflowId)
return
def closeOutRealTimeWorkflows(self):
"""
_closeOutRealTimeWorkflows_
Updates couch with the closeout status of Repack and Express
PromptReco should be closed out automatically
"""
getNotClosedOutWorkflowsDAO = self.daoFactory(classname = "Tier0Feeder.GetNotClosedOutWorkflows")
workflows = getNotClosedOutWorkflowsDAO.execute()
if len(workflows) == 0:
logging.debug("No workflows to publish to couch monitoring, doing nothing")
if workflows:
for workflow in workflows:
(workflowId, filesetId, filesetOpen, workflowName) = workflow
# find returns -1 if the string is not found
if workflowName.find('PromptReco') >= 0:
logging.debug("Closing out instantaneously PromptReco Workflow %s" % workflowName)
self.updateClosedState(workflowName, workflowId)
else :
# Check if fileset (which you already know) is closed or not
# FIXME: No better way to do it? what comes from the DAO is a string, casting bool or int doesn't help much.
# Works like that :
if filesetOpen == '0':
self.updateClosedState(workflowName, workflowId)
return
def updateClosedState(self, workflowName, workflowId):
"""
_updateClosedState_
Mark a workflow as Closed
"""
markCloseoutWorkflowMonitoringDAO = self.daoFactory(classname = "Tier0Feeder.MarkCloseoutWorkflowMonitoring")
response = self.localRequestCouchDB.updateRequestStatus(workflowName, 'Closed')
if response == "OK" or "EXISTS":
logging.debug("Successfully closed workflow %s" % workflowName)
markCloseoutWorkflowMonitoringDAO.execute(workflowId)
return
def notifyStorageManager(self):
"""
_notifyStorageManager_
Find all finished streamers for closed all run/stream
Send the notification message to StorageManager
Update the streamer status to finished (deleted = 1)
"""
getFinishedStreamersDAO = self.daoFactory(classname = "SMNotification.GetFinishedStreamers")
markStreamersFinishedDAO = self.daoFactory(classname = "SMNotification.MarkStreamersFinished")
allFinishedStreamers = getFinishedStreamersDAO.execute(transaction = False)
num = len(allFinishedStreamers)/50
for finishedStreamers in [allFinishedStreamers[i::num] for i in range(num)]:
streamers = []
filenameParams = ""
for (id, lfn) in finishedStreamers:
streamers.append(id)
filenameParams += "-FILENAME %s " % os.path.basename(lfn)
logging.debug("Notifying transfer system about processed streamers")
p = subprocess.Popen("/bin/bash",stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
output, error = p.communicate("""
export T0_BASE_DIR=%s
export T0ROOT=${T0_BASE_DIR}/T0
export CONFIG=${T0_BASE_DIR}/Config/TransferSystem_CERN.cfg
export PERL5LIB=${T0ROOT}/perl_lib
${T0ROOT}/operations/sendRepackedStatus.pl --config $CONFIG %s
""" % (self.transferSystemBaseDir, filenameParams))
if len(error) > 0:
logging.error("ERROR: Could not notify transfer system about processed streamers")
logging.error("ERROR: %s" % error)
markStreamersFinishedDAO.execute(streamers, transaction = False)
return
def updateExpressConfigsT0DataSvc(self):
"""
_updateExpressConfigsT0DataSvc_
Check which express_config rows are missing
in the Tier0 Data Service and insert them,
also record that fact in t0ast
"""
getExpressConfigsDAO = self.daoFactory(classname = "T0DataSvc.GetExpressConfigs")
expressConfigs = getExpressConfigsDAO.execute(transaction = False)
if len(expressConfigs) > 0:
bindsInsert = []
bindsUpdate = []
for config in expressConfigs:
bindsInsert.append( { 'RUN' : config['run'],
'STREAM' : config['stream'],
'CMSSW' : config['cmssw'],
'SCRAM_ARCH' : config['scram_arch'],
'RECO_CMSSW' : config['reco_cmssw'],
'RECO_SCRAM_ARCH' : config['reco_scram_arch'],
'GLOBAL_TAG' : config['global_tag'][:50],
'SCENARIO' : config['scenario'] } )
bindsUpdate.append( { 'RUN' : config['run'],
'STREAM' : config['stream'] } )
insertExpressConfigsDAO = self.daoFactoryT0DataSvc(classname = "T0DataSvc.InsertExpressConfigs")
insertExpressConfigsDAO.execute(binds = bindsInsert, transaction = False)
updateExpressConfigsDAO = self.daoFactory(classname = "T0DataSvc.UpdateExpressConfigs")
updateExpressConfigsDAO.execute(binds = bindsUpdate, transaction = False)
return
def updateRecoConfigsT0DataSvc(self):
"""
_updateRecoConfigsT0DataSvc_
Check which reco_config rows are missing
in the Tier0 Data Service and insert them,
also record that fact in t0ast
"""
getRecoConfigsDAO = self.daoFactory(classname = "T0DataSvc.GetRecoConfigs")
recoConfigs = getRecoConfigsDAO.execute(transaction = False)
if len(recoConfigs) > 0:
bindsInsert = []
bindsUpdate = []
for config in recoConfigs:
bindsInsert.append( { 'RUN' : config['run'],
'PRIMDS' : config['primds'],
'CMSSW' : config['cmssw'],
'SCRAM_ARCH' : config['scram_arch'],
'GLOBAL_TAG' : config['global_tag'][:50],
'SCENARIO' : config['scenario'] } )
bindsUpdate.append( { 'RUN' : config['run'],
'PRIMDS' : config['primds'] } )
insertRecoConfigsDAO = self.daoFactoryT0DataSvc(classname = "T0DataSvc.InsertRecoConfigs")
insertRecoConfigsDAO.execute(binds = bindsInsert, transaction = False)
updateRecoConfigsDAO = self.daoFactory(classname = "T0DataSvc.UpdateRecoConfigs")
updateRecoConfigsDAO.execute(binds = bindsUpdate, transaction = False)
return
def updateRecoReleaseConfigsT0DataSvc(self):
"""
_updateRecoReleaseConfigsT0DataSvc_
Insert information about PromptReco release into the Tier0 Data Service.
Already aggregate by run, if one primary dataset is released that means
the whole run is considered released.
"""
getRecoReleaseConfigsDAO = self.daoFactory(classname = "T0DataSvc.GetRecoReleaseConfigs")
recoReleaseConfigs = getRecoReleaseConfigsDAO.execute(transaction = False)
if len(recoReleaseConfigs) > 0:
bindsInsert = []
bindsUpdate = []
for config in recoReleaseConfigs:
locked = int(config['released'] > 0)
bindsInsert.append( { 'RUN' : config['run'],
'LOCKED' : locked } )
bindsUpdate.append( { 'RUN' : config['run'],
'IN_DATASVC' : locked + 1 } )
insertRecoReleaseConfigsDAO = self.daoFactoryT0DataSvc(classname = "T0DataSvc.InsertRecoReleaseConfigs")
insertRecoReleaseConfigsDAO.execute(binds = bindsInsert, transaction = False)
updateRecoReleaseConfigsDAO = self.daoFactory(classname = "T0DataSvc.UpdateRecoReleaseConfigs")
updateRecoReleaseConfigsDAO.execute(binds = bindsUpdate, transaction = False)
return
def terminate(self, params):
"""
_terminate_
Kill the code after one final pass when called by the master thread.
"""
logging.debug("terminating immediately")
class Tier0FeederPoller(BaseWorkerThread):
def __init__(self, config):
"""
_init_
"""
BaseWorkerThread.__init__(self)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package = "T0.WMBS",
logger = logging,
dbinterface = myThread.dbi)
self.tier0ConfigFile = config.Tier0Feeder.tier0ConfigFile
self.specDirectory = config.Tier0Feeder.specDirectory
self.dropboxuser = getattr(config.Tier0Feeder, "dropboxuser", None)
self.dropboxpass = getattr(config.Tier0Feeder, "dropboxpass", None)
self.transferSystemBaseDir = getattr(config.Tier0Feeder, "transferSystemBaseDir", None)
if self.transferSystemBaseDir != None:
if not os.path.exists(self.transferSystemBaseDir):
self.transferSystemBaseDir = None
self.dqmUploadProxy = getattr(config.Tier0Feeder, "dqmUploadProxy", None)
self.serviceProxy = getattr(config.Tier0Feeder, "serviceProxy", None)
self.localRequestCouchDB = RequestDBWriter(config.AnalyticsDataCollector.localT0RequestDBURL,
couchapp = config.AnalyticsDataCollector.RequestCouchApp)
hltConfConnectUrl = config.HLTConfDatabase.connectUrl
dbFactoryHltConf = DBFactory(logging, dburl = hltConfConnectUrl, options = {})
dbInterfaceHltConf = dbFactoryHltConf.connect()
daoFactoryHltConf = DAOFactory(package = "T0.WMBS",
logger = logging,
dbinterface = dbInterfaceHltConf)
self.getHLTConfigDAO = daoFactoryHltConf(classname = "RunConfig.GetHLTConfig")
storageManagerConnectUrl = config.StorageManagerDatabase.connectUrl
dbFactoryStorageManager = DBFactory(logging, dburl = storageManagerConnectUrl, options = {})
self.dbInterfaceStorageManager = dbFactoryStorageManager.connect()
self.getExpressReadyRunsDAO = None
if hasattr(config, "PopConLogDatabase"):
popConLogConnectUrl = getattr(config.PopConLogDatabase, "connectUrl", None)
if popConLogConnectUrl != None:
dbFactoryPopConLog = DBFactory(logging, dburl = popConLogConnectUrl, options = {})
dbInterfacePopConLog = dbFactoryPopConLog.connect()
daoFactoryPopConLog = DAOFactory(package = "T0.WMBS",
logger = logging,
dbinterface = dbInterfacePopConLog)
self.getExpressReadyRunsDAO = daoFactoryPopConLog(classname = "Tier0Feeder.GetExpressReadyRuns")
self.haveT0DataSvc = False
if hasattr(config, "T0DataSvcDatabase"):
t0datasvcConnectUrl = getattr(config.T0DataSvcDatabase, "connectUrl", None)
if t0datasvcConnectUrl != None:
self.haveT0DataSvc = True
dbFactoryT0DataSvc = DBFactory(logging, dburl = t0datasvcConnectUrl, options = {})
dbInterfaceT0DataSvc = dbFactoryT0DataSvc.connect()
self.daoFactoryT0DataSvc = DAOFactory(package = "T0.WMBS",
logger = logging,
dbinterface = dbInterfaceT0DataSvc)
return
def algorithm(self, parameters = None):
"""
_algorithm_
"""
logging.debug("Running Tier0Feeder algorithm...")
myThread = threading.currentThread()
findNewRunsDAO = self.daoFactory(classname = "Tier0Feeder.FindNewRuns")
findNewRunStreamsDAO = self.daoFactory(classname = "Tier0Feeder.FindNewRunStreams")
findNewExpressRunsDAO = self.daoFactory(classname = "Tier0Feeder.FindNewExpressRuns")
releaseExpressDAO = self.daoFactory(classname = "Tier0Feeder.ReleaseExpress")
feedStreamersDAO = self.daoFactory(classname = "Tier0Feeder.FeedStreamers")
markWorkflowsInjectedDAO = self.daoFactory(classname = "Tier0Feeder.MarkWorkflowsInjected")
tier0Config = None
try:
tier0Config = loadConfigurationFile(self.tier0ConfigFile)
except:
# usually happens when there are syntax errors in the configuration
logging.exception("Cannot load Tier0 configuration file, not configuring new runs and run/streams")
# only configure new runs and run/streams if we have a valid Tier0 configuration
if tier0Config != None:
#
# find new runs, setup global run settings and stream/dataset/trigger mapping
#
runHltkeys = findNewRunsDAO.execute(transaction = False)
for run, hltkey in sorted(runHltkeys.items()):
hltConfig = None
# local runs have no hltkey and are configured differently
if hltkey != None:
# retrieve HLT configuration and make sure it's usable
try:
hltConfig = self.getHLTConfigDAO.execute(hltkey, transaction = False)
if hltConfig['process'] == None or len(hltConfig['mapping']) == 0:
raise RuntimeError("HLTConfDB query returned no process or mapping")
except:
logging.exception("Can't retrieve hltkey %s for run %d" % (hltkey, run))
continue
try:
RunConfigAPI.configureRun(tier0Config, run, hltConfig)
except:
logging.exception("Can't configure for run %d" % (run))
#
# find unconfigured run/stream with data
# populate RunConfig, setup workflows/filesets/subscriptions
#
runStreams = findNewRunStreamsDAO.execute(transaction = False)
for run in sorted(runStreams.keys()):
for stream in sorted(runStreams[run]):
try:
RunConfigAPI.configureRunStream(tier0Config,
run, stream,
self.specDirectory,
self.dqmUploadProxy)
except:
logging.exception("Can't configure for run %d and stream %s" % (run, stream))
#
# stop and close runs based on RunSummary and StorageManager records
#
RunLumiCloseoutAPI.stopRuns(self.dbInterfaceStorageManager)
RunLumiCloseoutAPI.closeRuns(self.dbInterfaceStorageManager)
#
# release runs for Express
#
runs = findNewExpressRunsDAO.execute(transaction = False)
if len(runs) > 0:
binds = []
for run in runs:
binds.append( { 'RUN' : run } )
if self.getExpressReadyRunsDAO != None:
runs = self.getExpressReadyRunsDAO.execute(binds = binds, transaction = False)
if len(runs) > 0:
binds = []
for run in runs:
binds.append( { 'RUN' : run } )
releaseExpressDAO.execute(binds = binds, transaction = False)
#
# release runs for PromptReco
#
RunConfigAPI.releasePromptReco(tier0Config,
self.specDirectory,
self.dqmUploadProxy)
#
# insert express and reco configs into Tier0 Data Service
#
if self.haveT0DataSvc:
self.updateRunStreamDoneT0DataSvc()
self.updateExpressConfigsT0DataSvc()
self.updateRecoConfigsT0DataSvc()
self.updateRecoReleaseConfigsT0DataSvc()
self.lockDatasetsT0DataSvc()
#
# mark express and repack workflows as injected if certain conditions are met
# (we don't do it immediately to prevent the TaskArchiver from cleaning up too early)
#
markWorkflowsInjectedDAO.execute(self.transferSystemBaseDir != None,
transaction = False)
#
# close stream/lumis for run/streams that are active (fileset exists and open)
#
RunLumiCloseoutAPI.closeLumiSections(self.dbInterfaceStorageManager)
#
# feed new data into exisiting filesets
#
try:
myThread.transaction.begin()
feedStreamersDAO.execute(conn = myThread.transaction.conn, transaction = True)
except:
logging.exception("Can't feed data, bailing out...")
raise
else:
myThread.transaction.commit()
#
# run ended and run/stream fileset open
# => check for complete lumi_closed record, all lumis finally closed and all data feed
# => if all conditions satisfied, close the run/stream fileset
#
RunLumiCloseoutAPI.closeRunStreamFilesets()
#
# check and delete active split lumis
#
RunLumiCloseoutAPI.checkActiveSplitLumis()
#
# insert workflows into CouchDB for monitoring
#
self.feedCouchMonitoring()
#
# Update Couch when Repack and Express have closed input filesets (analog to old T0 closeout)
#
self.closeOutRealTimeWorkflows()
#
# send repacked notifications to StorageManager
#
if self.transferSystemBaseDir != None:
self.notifyStorageManager()
#
# upload PCL conditions to DropBox
#
ConditionUploadAPI.uploadConditions(self.dropboxuser, self.dropboxpass, self.serviceProxy)
return
def feedCouchMonitoring(self):
"""
_feedCouchMonitoring_
check for workflows that haven't been uploaded to Couch for monitoring yet
"""
getStreamerWorkflowsForMonitoringDAO = self.daoFactory(classname = "Tier0Feeder.GetStreamerWorkflowsForMonitoring")
getPromptRecoWorkflowsForMonitoringDAO = self.daoFactory(classname = "Tier0Feeder.GetPromptRecoWorkflowsForMonitoring")
markTrackedWorkflowMonitoringDAO = self.daoFactory(classname = "Tier0Feeder.MarkTrackedWorkflowMonitoring")
workflows = getStreamerWorkflowsForMonitoringDAO.execute()
workflows += getPromptRecoWorkflowsForMonitoringDAO.execute()
if len(workflows) == 0:
logging.debug("No workflows to publish to couch monitoring, doing nothing")
if workflows:
logging.debug(" Going to publish %d workflows" % len(workflows))
for (workflowId, run, workflowName) in workflows:
logging.info(" Publishing workflow %s to monitoring" % workflowName)
#TODO: add more information about workflow if there need to be kept longer than
# workflow life cycle.
doc = {}
doc["RequestName"] = workflowName
doc["Run"] = run
response = self.localRequestCouchDB.insertGenericRequest(doc)
if response == "OK" or "EXISTS":
logging.info(" Successfully uploaded request %s" % workflowName)
# Here we have to trust the insert, if it doesn't happen will be easy to spot on the logs
markTrackedWorkflowMonitoringDAO.execute(workflowId)
return
def closeOutRealTimeWorkflows(self):
"""
_closeOutRealTimeWorkflows_
Updates couch with the closeout status of Repack and Express
PromptReco should be closed out automatically
"""
getNotClosedOutWorkflowsDAO = self.daoFactory(classname = "Tier0Feeder.GetNotClosedOutWorkflows")
workflows = getNotClosedOutWorkflowsDAO.execute()
if len(workflows) == 0:
logging.debug("No workflows to publish to couch monitoring, doing nothing")
if workflows:
for workflow in workflows:
(workflowId, filesetId, filesetOpen, workflowName) = workflow
# find returns -1 if the string is not found
if workflowName.find('PromptReco') >= 0:
logging.debug("Closing out instantaneously PromptReco Workflow %s" % workflowName)
self.updateClosedState(workflowName, workflowId)
else :
# Check if fileset (which you already know) is closed or not
# FIXME: No better way to do it? what comes from the DAO is a string, casting bool or int doesn't help much.
# Works like that :
if filesetOpen == '0':
self.updateClosedState(workflowName, workflowId)
return
def updateClosedState(self, workflowName, workflowId):
"""
_updateClosedState_
Mark a workflow as Closed
"""
markCloseoutWorkflowMonitoringDAO = self.daoFactory(classname = "Tier0Feeder.MarkCloseoutWorkflowMonitoring")
response = self.localRequestCouchDB.updateRequestStatus(workflowName, 'Closed')
if response == "OK" or "EXISTS":
logging.debug("Successfully closed workflow %s" % workflowName)
markCloseoutWorkflowMonitoringDAO.execute(workflowId)
return
def notifyStorageManager(self):
"""
_notifyStorageManager_
Find all finished streamers for closed all run/stream
Send the notification message to StorageManager
Update the streamer status to finished (deleted = 1)
"""
getFinishedStreamersDAO = self.daoFactory(classname = "SMNotification.GetFinishedStreamers")
markStreamersFinishedDAO = self.daoFactory(classname = "SMNotification.MarkStreamersFinished")
allFinishedStreamers = getFinishedStreamersDAO.execute(transaction = False)
chunkSize = 50
for finishedStreamers in [ allFinishedStreamers[i:i+chunkSize] for i in range(0, len(allFinishedStreamers), chunkSize) ]:
streamers = []
filenameParams = ""
for (id, lfn) in finishedStreamers:
streamers.append(id)
filenameParams += "-FILENAME %s " % os.path.basename(lfn)
logging.debug("Notifying transfer system about processed streamers")
p = subprocess.Popen("/bin/bash", stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output, error = p.communicate("""
export T0_BASE_DIR=%s
export T0ROOT=${T0_BASE_DIR}/T0
export CONFIG=${T0_BASE_DIR}/Config/TransferSystem_CERN.cfg
export PERL5LIB=${T0ROOT}/perl_lib
unset LANGUAGE
unset LC_ALL
unset LC_CTYPE
export LANG=C
${T0ROOT}/operations/sendRepackedStatus.pl --config $CONFIG %s
""" % (self.transferSystemBaseDir, filenameParams))
if len(error) > 0:
logging.error("ERROR: Could not notify transfer system about processed streamers")
logging.error("ERROR: %s" % error)
else:
markStreamersFinishedDAO.execute(streamers, transaction = False)
return
def updateRunStreamDoneT0DataSvc(self):
"""
_updateRunStreamDoneT0DataSvc_
Check if a run/stream workflow (express or repack) is finished and
cleaned up and push a completion record into the Tier0 Data Service.
"""
getRunStreamDoneDAO = self.daoFactory(classname = "T0DataSvc.GetRunStreamDone")
runStreamDone = getRunStreamDoneDAO.execute(transaction = False)
if len(runStreamDone) > 0:
binds = []
for runStream in runStreamDone:
binds.append( { 'RUN' : runStream['run'],
'STREAM' : runStream['stream'] } )
insertRunStreamDoneDAO = self.daoFactoryT0DataSvc(classname = "T0DataSvc.InsertRunStreamDone")
insertRunStreamDoneDAO.execute(binds = binds, transaction = False)
updateRunStreamDoneDAO = self.daoFactory(classname = "T0DataSvc.UpdateRunStreamDone")
updateRunStreamDoneDAO.execute(binds = binds, transaction = False)
return
def updateExpressConfigsT0DataSvc(self):
"""
_updateExpressConfigsT0DataSvc_
Check which express_config rows are missing
in the Tier0 Data Service and insert them,
also record that fact in t0ast
"""
getExpressConfigsDAO = self.daoFactory(classname = "T0DataSvc.GetExpressConfigs")
expressConfigs = getExpressConfigsDAO.execute(transaction = False)
if len(expressConfigs) > 0:
bindsInsert = []
bindsUpdate = []
for config in expressConfigs:
bindsInsert.append( { 'RUN' : config['run'],
'STREAM' : config['stream'],
'CMSSW' : config['cmssw'],
'SCRAM_ARCH' : config['scram_arch'],
'RECO_CMSSW' : config['reco_cmssw'],
'RECO_SCRAM_ARCH' : config['reco_scram_arch'],
'ALCA_SKIM' : config['alca_skim'],
'DQM_SEQ' : config['dqm_seq'],
'GLOBAL_TAG' : config['global_tag'][:50],
'SCENARIO' : config['scenario'] } )
bindsUpdate.append( { 'RUN' : config['run'],
'STREAM' : config['stream'] } )
insertExpressConfigsDAO = self.daoFactoryT0DataSvc(classname = "T0DataSvc.InsertExpressConfigs")
insertExpressConfigsDAO.execute(binds = bindsInsert, transaction = False)
updateExpressConfigsDAO = self.daoFactory(classname = "T0DataSvc.UpdateExpressConfigs")
updateExpressConfigsDAO.execute(binds = bindsUpdate, transaction = False)
return
def updateRecoConfigsT0DataSvc(self):
"""
_updateRecoConfigsT0DataSvc_
Check which reco_config rows are missing
in the Tier0 Data Service and insert them,
also record that fact in t0ast
"""
getRecoConfigsDAO = self.daoFactory(classname = "T0DataSvc.GetRecoConfigs")
recoConfigs = getRecoConfigsDAO.execute(transaction = False)
if len(recoConfigs) > 0:
bindsInsert = []
bindsUpdate = []
for config in recoConfigs:
bindsInsert.append( { 'RUN' : config['run'],
'PRIMDS' : config['primds'],
'CMSSW' : config['cmssw'],
'SCRAM_ARCH' : config['scram_arch'],
'ALCA_SKIM' : config['alca_skim'],
'PHYSICS_SKIM' : config['physics_skim'],
'DQM_SEQ' : config['dqm_seq'],
'GLOBAL_TAG' : config['global_tag'][:50],
'SCENARIO' : config['scenario'] } )
bindsUpdate.append( { 'RUN' : config['run'],
'PRIMDS' : config['primds'] } )
insertRecoConfigsDAO = self.daoFactoryT0DataSvc(classname = "T0DataSvc.InsertRecoConfigs")
insertRecoConfigsDAO.execute(binds = bindsInsert, transaction = False)
updateRecoConfigsDAO = self.daoFactory(classname = "T0DataSvc.UpdateRecoConfigs")
updateRecoConfigsDAO.execute(binds = bindsUpdate, transaction = False)
return
def updateRecoReleaseConfigsT0DataSvc(self):
"""
_updateRecoReleaseConfigsT0DataSvc_
Insert information about PromptReco release into the Tier0 Data Service.
Already aggregate by run, if one primary dataset is released that means
the whole run is considered released.
"""
getRecoReleaseConfigsDAO = self.daoFactory(classname = "T0DataSvc.GetRecoReleaseConfigs")
recoReleaseConfigs = getRecoReleaseConfigsDAO.execute(transaction = False)
if len(recoReleaseConfigs) > 0:
bindsInsert = []
bindsUpdate = []
for config in recoReleaseConfigs:
locked = int(config['released'] > 0)
bindsInsert.append( { 'RUN' : config['run'],
'LOCKED' : locked } )
bindsUpdate.append( { 'RUN' : config['run'],
'IN_DATASVC' : locked + 1 } )
insertRecoReleaseConfigsDAO = self.daoFactoryT0DataSvc(classname = "T0DataSvc.InsertRecoReleaseConfigs")
insertRecoReleaseConfigsDAO.execute(binds = bindsInsert, transaction = False)
updateRecoReleaseConfigsDAO = self.daoFactory(classname = "T0DataSvc.UpdateRecoReleaseConfigs")
updateRecoReleaseConfigsDAO.execute(binds = bindsUpdate, transaction = False)
return
def lockDatasetsT0DataSvc(self):
"""
_lockDatasetsT0DataSvc_
Publish dataset information into the Tier0 Data Service.
"""
getDatasetLockedDAO = self.daoFactory(classname = "T0DataSvc.GetDatasetLocked")
datasetConfigs = getDatasetLockedDAO.execute(transaction = False)
if len(datasetConfigs) > 0:
bindsInsert = []
bindsUpdate = []
for config in datasetConfigs:
bindsInsert.append( { 'PATH' : config['path'] } )
bindsUpdate.append( { 'ID' : config['id'] } )
insertDatasetLockedDAO = self.daoFactoryT0DataSvc(classname = "T0DataSvc.InsertDatasetLocked")
insertDatasetLockedDAO.execute(binds = bindsInsert, transaction = False)
updateDatasetLockedDAO = self.daoFactory(classname = "T0DataSvc.UpdateDatasetLocked")
updateDatasetLockedDAO.execute(binds = bindsUpdate, transaction = False)
return
def terminate(self, params):
"""
_terminate_
Kill the code after one final pass when called by the master thread.
"""
logging.debug("terminating immediately")
class TaskArchiverTest(EmulatedUnitTestCase):
"""
TestCase for TestTaskArchiver module
"""
_setup_done = False
_teardown = False
_maxMessage = 10
OWNERDN = os.environ[
'OWNERDN'] if 'OWNERDN' in os.environ else "Generic/OWNERDN"
def setUp(self):
"""
setup for test.
"""
super(TaskArchiverTest, self).setUp()
myThread = threading.currentThread()
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection(destroyAllDatabase=True)
self.testInit.setSchema(
customModules=["WMCore.WMBS", "WMComponent.DBS3Buffer"],
useDefault=False)
self.databaseName = "taskarchiver_t_0"
self.testInit.setupCouch("%s/workloadsummary" % self.databaseName,
"WorkloadSummary")
self.testInit.setupCouch("%s/jobs" % self.databaseName, "JobDump")
self.testInit.setupCouch("%s/fwjrs" % self.databaseName, "FWJRDump")
self.testInit.setupCouch("wmagent_summary_t", "WMStats")
self.testInit.setupCouch("wmagent_summary_central_t", "WMStats")
self.testInit.setupCouch("stat_summary_t", "SummaryStats")
reqmgrdb = "reqmgrdb_t"
self.testInit.setupCouch(reqmgrdb, "ReqMgr")
reqDBURL = "%s/%s" % (self.testInit.couchUrl, reqmgrdb)
self.requestWriter = RequestDBWriter(reqDBURL)
self.requestWriter.defaultStale = {}
self.daofactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.dbsDaoFactory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.getJobs = self.daofactory(classname="Jobs.GetAllJobs")
self.inject = self.daofactory(
classname="Workflow.MarkInjectedWorkflows")
self.testDir = self.testInit.generateWorkDir()
os.makedirs(os.path.join(self.testDir, 'specDir'))
self.nJobs = 10
self.campaignName = 'aCampaign'
return
def tearDown(self):
"""
Database deletion
"""
self.testInit.clearDatabase(modules=["WMCore.WMBS"])
self.testInit.delWorkDir()
self.testInit.tearDownCouch()
return
def getConfig(self):
"""
_createConfig_
General config file
"""
config = self.testInit.getConfiguration()
# self.testInit.generateWorkDir(config)
config.section_("General")
config.General.workDir = "."
config.General.ReqMgr2ServiceURL = "https://cmsweb-dev.cern.ch/reqmgr2"
config.section_("JobStateMachine")
config.JobStateMachine.couchurl = os.getenv("COUCHURL",
"cmssrv52.fnal.gov:5984")
config.JobStateMachine.couchDBName = self.databaseName
config.JobStateMachine.jobSummaryDBName = 'wmagent_summary_t'
config.JobStateMachine.summaryStatsDBName = 'stat_summary_t'
config.component_("JobCreator")
config.JobCreator.jobCacheDir = os.path.join(self.testDir, 'testDir')
config.component_("TaskArchiver")
config.TaskArchiver.componentDir = self.testDir
config.TaskArchiver.WorkQueueParams = {
'CacheDir': config.JobCreator.jobCacheDir
}
config.TaskArchiver.pollInterval = 60
config.TaskArchiver.logLevel = 'INFO'
config.TaskArchiver.timeOut = 0
config.TaskArchiver.histogramKeys = [
'AvgEventTime', 'writeTotalMB', 'jobTime'
]
config.TaskArchiver.histogramBins = 5
config.TaskArchiver.histogramLimit = 5
config.TaskArchiver.perfPrimaryDatasets = [
'SingleMu', 'MuHad', 'MinimumBias'
]
config.TaskArchiver.perfDashBoardMinLumi = 50
config.TaskArchiver.perfDashBoardMaxLumi = 9000
config.TaskArchiver.dqmUrl = 'https://cmsweb.cern.ch/dqm/dev/'
config.TaskArchiver.dashBoardUrl = 'http://dashboard43.cern.ch/dashboard/request.py/putluminositydata'
config.TaskArchiver.workloadSummaryCouchDBName = "%s/workloadsummary" % self.databaseName
config.TaskArchiver.localWMStatsURL = "%s/%s" % (
config.JobStateMachine.couchurl,
config.JobStateMachine.jobSummaryDBName)
config.TaskArchiver.workloadSummaryCouchURL = config.JobStateMachine.couchurl
config.TaskArchiver.requireCouch = True
config.component_("AnalyticsDataCollector")
config.AnalyticsDataCollector.centralRequestDBURL = '%s/reqmgrdb_t' % config.JobStateMachine.couchurl
config.AnalyticsDataCollector.RequestCouchApp = "ReqMgr"
config.section_("ACDC")
config.ACDC.couchurl = config.JobStateMachine.couchurl
config.ACDC.database = config.JobStateMachine.couchDBName
# Make the jobCacheDir
os.mkdir(config.JobCreator.jobCacheDir)
# addition for Alerts messaging framework, work (alerts) and control
# channel addresses to which the component will be sending alerts
# these are destination addresses where AlertProcessor:Receiver listens
config.section_("Alert")
config.Alert.address = "tcp://127.0.0.1:5557"
config.Alert.controlAddr = "tcp://127.0.0.1:5559"
config.section_("Agent")
config.Agent.serverDN = "/we/bypass/myproxy/logon"
return config
def createWorkload(self, workloadName):
"""
_createTestWorkload_
Creates a test workload for us to run on, hold the basic necessities.
"""
workload = testWorkload(workloadName)
taskMaker = TaskMaker(workload,
os.path.join(self.testDir, 'workloadTest'))
taskMaker.skipSubscription = True
taskMaker.processWorkload()
workload.setCampaign(self.campaignName)
workload.save(workloadName)
return workload
def createTestJobGroup(self,
config,
name="TestWorkthrough",
filesetName="TestFileset",
specLocation="spec.xml",
error=False,
task="/TestWorkload/ReReco",
jobType="Processing"):
"""
Creates a group of several jobs
"""
testWorkflow = Workflow(spec=specLocation,
owner=self.OWNERDN,
name=name,
task=task,
owner_vogroup="",
owner_vorole="")
testWorkflow.create()
self.inject.execute(names=[name], injected=True)
testWMBSFileset = Fileset(name=filesetName)
testWMBSFileset.create()
testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10)
testFileA.addRun(Run(10, *[12312]))
testFileA.setLocation('malpaquet')
testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10)
testFileB.addRun(Run(10, *[12314]))
testFileB.setLocation('malpaquet')
testFileA.create()
testFileB.create()
testWMBSFileset.addFile(testFileA)
testWMBSFileset.addFile(testFileB)
testWMBSFileset.commit()
testWMBSFileset.markOpen(0)
outputWMBSFileset = Fileset(name='%sOutput' % filesetName)
outputWMBSFileset.create()
testFileC = File(lfn="/this/is/a/lfnC", size=1024, events=10)
testFileC.addRun(Run(10, *[12312]))
testFileC.setLocation('malpaquet')
testFileC.create()
outputWMBSFileset.addFile(testFileC)
outputWMBSFileset.commit()
outputWMBSFileset.markOpen(0)
testWorkflow.addOutput('output', outputWMBSFileset)
testSubscription = Subscription(fileset=testWMBSFileset,
workflow=testWorkflow,
type=jobType)
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
for i in range(0, self.nJobs):
testJob = Job(name=makeUUID())
testJob.addFile(testFileA)
testJob.addFile(testFileB)
testJob['retry_count'] = 1
testJob['retry_max'] = 10
testJob['mask'].addRunAndLumis(run=10, lumis=[12312, 12313])
testJobGroup.add(testJob)
testJobGroup.commit()
changer = ChangeState(config)
report1 = Report()
report2 = Report()
if error:
path1 = os.path.join(WMCore.WMBase.getTestBase(),
"WMComponent_t/JobAccountant_t/fwjrs",
"badBackfillJobReport.pkl")
path2 = os.path.join(WMCore.WMBase.getTestBase(),
'WMComponent_t/TaskArchiver_t/fwjrs',
'logCollectReport2.pkl')
else:
path1 = os.path.join(WMCore.WMBase.getTestBase(),
'WMComponent_t/TaskArchiver_t/fwjrs',
'mergeReport1.pkl')
path2 = os.path.join(WMCore.WMBase.getTestBase(),
'WMComponent_t/TaskArchiver_t/fwjrs',
'logCollectReport2.pkl')
report1.load(filename=path1)
report2.load(filename=path2)
changer.propagate(testJobGroup.jobs, 'created', 'new')
changer.propagate(testJobGroup.jobs, 'executing', 'created')
changer.propagate(testJobGroup.jobs, 'complete', 'executing')
for i in range(self.nJobs):
if i < self.nJobs / 2:
testJobGroup.jobs[i]['fwjr'] = report1
else:
testJobGroup.jobs[i]['fwjr'] = report2
changer.propagate(testJobGroup.jobs, 'jobfailed', 'complete')
changer.propagate(testJobGroup.jobs, 'jobcooloff', 'jobfailed')
changer.propagate(testJobGroup.jobs, 'created', 'jobcooloff')
changer.propagate(testJobGroup.jobs, 'executing', 'created')
changer.propagate(testJobGroup.jobs, 'complete', 'executing')
changer.propagate(testJobGroup.jobs, 'jobfailed', 'complete')
changer.propagate(testJobGroup.jobs, 'retrydone', 'jobfailed')
changer.propagate(testJobGroup.jobs, 'exhausted', 'retrydone')
changer.propagate(testJobGroup.jobs, 'cleanout', 'exhausted')
testSubscription.completeFiles([testFileA, testFileB])
return testJobGroup
def createGiantJobSet(self,
name,
config,
nSubs=10,
nJobs=10,
nFiles=1,
spec="spec.xml"):
"""
Creates a massive set of jobs
"""
jobList = []
for i in range(0, nSubs):
# Make a bunch of subscriptions
localName = '%s-%i' % (name, i)
testWorkflow = Workflow(spec=spec,
owner=self.OWNERDN,
name=localName,
task="Test",
owner_vogroup="",
owner_vorole="")
testWorkflow.create()
testWMBSFileset = Fileset(name=localName)
testWMBSFileset.create()
testSubscription = Subscription(fileset=testWMBSFileset,
workflow=testWorkflow)
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
filesToComplete = []
for j in range(0, nJobs):
# Create jobs for each subscription
testFileA = File(lfn="%s-%i-lfnA" % (localName, j),
size=1024,
events=10)
testFileA.addRun(
Run(
10, *[
11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36,
37, 38, 39, 40
]))
testFileA.setLocation('malpaquet')
testFileA.create()
testWMBSFileset.addFile(testFileA)
testWMBSFileset.commit()
filesToComplete.append(testFileA)
testJob = Job(name='%s-%i' % (localName, j))
testJob.addFile(testFileA)
testJob['retry_count'] = 1
testJob['retry_max'] = 10
testJobGroup.add(testJob)
jobList.append(testJob)
for k in range(0, nFiles):
# Create output files
testFile = File(lfn="%s-%i-output" % (localName, k),
size=1024,
events=10)
testFile.addRun(Run(10, *[12312]))
testFile.setLocation('malpaquet')
testFile.create()
testJobGroup.output.addFile(testFile)
testJobGroup.output.commit()
testJobGroup.commit()
changer = ChangeState(config)
changer.propagate(testJobGroup.jobs, 'created', 'new')
changer.propagate(testJobGroup.jobs, 'executing', 'created')
changer.propagate(testJobGroup.jobs, 'complete', 'executing')
changer.propagate(testJobGroup.jobs, 'success', 'complete')
changer.propagate(testJobGroup.jobs, 'cleanout', 'success')
testWMBSFileset.markOpen(0)
testSubscription.completeFiles(filesToComplete)
return jobList
def getPerformanceFromDQM(self, dqmUrl, dataset, run):
# Make function to fetch this from DQM. Returning Null or False if it fails
getUrl = "%sjsonfairy/archive/%s%s/DQM/TimerService/event_byluminosity" % (
dqmUrl, run, dataset)
# Assert if the URL is assembled as expected
if run == 207214:
self.assertEqual(
'https://cmsweb.cern.ch/dqm/dev/jsonfairy/archive/207214/MinimumBias/Commissioning10-v4/DQM/DQM/TimerService/event_byluminosity',
getUrl)
# let's suppose it works..
testResponseFile = open(
os.path.join(getTestBase(),
'WMComponent_t/TaskArchiver_t/DQMGUIResponse.json'),
'r')
response = testResponseFile.read()
testResponseFile.close()
responseJSON = json.loads(response)
return responseJSON
def filterInterestingPerfPoints(self, responseJSON, minLumi, maxLumi):
worthPoints = {}
points = responseJSON["hist"]["bins"]["content"]
for i in range(responseJSON["hist"]["xaxis"]["first"]["id"],
responseJSON["hist"]["xaxis"]["last"]["id"]):
# is the point worth it? if yes add to interesting points dictionary.
# 1 - non 0
# 2 - between minimum and maximum expected luminosity
# FIXME : 3 - population in dashboard for the bin interval < 100
# Those should come from the config :
if points[i] == 0:
continue
binSize = responseJSON["hist"]["xaxis"]["last"][
"value"] / responseJSON["hist"]["xaxis"]["last"]["id"]
# Fetching the important values
instLuminosity = i * binSize
timePerEvent = points[i]
if instLuminosity > minLumi and instLuminosity < maxLumi:
worthPoints[instLuminosity] = timePerEvent
return worthPoints
def publishPerformanceDashBoard(self, dashBoardUrl, PD, release,
worthPoints):
dashboardPayload = []
for instLuminosity in worthPoints:
timePerEvent = int(worthPoints[instLuminosity])
dashboardPayload.append({
"primaryDataset": PD,
"release": release,
"integratedLuminosity": instLuminosity,
"timePerEvent": timePerEvent
})
data = "{\"data\":%s}" % str(dashboardPayload).replace("\'", "\"")
# let's suppose it works..
testDashBoardPayloadFile = open(
os.path.join(getTestBase(),
'WMComponent_t/TaskArchiver_t/DashBoardPayload.json'),
'r')
testDashBoardPayload = testDashBoardPayloadFile.read()
testDashBoardPayloadFile.close()
self.assertEqual(data, testDashBoardPayload)
return True
def populateWorkflowWithCompleteStatus(self, name="TestWorkload"):
schema = generate_reqmgr_schema(1)
schema[0]["RequestName"] = name
self.requestWriter.insertGenericRequest(schema[0])
result = self.requestWriter.updateRequestStatus(name, "completed")
return result
def testA_BasicFunctionTest(self):
"""
_BasicFunctionTest_
Tests the components, by seeing if they can process a simple set of closeouts
"""
myThread = threading.currentThread()
config = self.getConfig()
workloadPath = os.path.join(self.testDir, 'specDir', 'spec.pkl')
workload = self.createWorkload(workloadName=workloadPath)
testJobGroup = self.createTestJobGroup(config=config,
name=workload.name(),
specLocation=workloadPath,
error=False)
# Create second workload
testJobGroup2 = self.createTestJobGroup(
config=config,
name=workload.name(),
filesetName="TestFileset_2",
specLocation=workloadPath,
task="/TestWorkload/ReReco/LogCollect",
jobType="LogCollect")
cachePath = os.path.join(config.JobCreator.jobCacheDir, "TestWorkload",
"ReReco")
os.makedirs(cachePath)
self.assertTrue(os.path.exists(cachePath))
cachePath2 = os.path.join(config.JobCreator.jobCacheDir,
"TestWorkload", "LogCollect")
os.makedirs(cachePath2)
self.assertTrue(os.path.exists(cachePath2))
result = myThread.dbi.processData(
"SELECT * FROM wmbs_subscription")[0].fetchall()
self.assertEqual(len(result), 2)
workflowName = "TestWorkload"
dbname = config.TaskArchiver.workloadSummaryCouchDBName
couchdb = CouchServer(config.JobStateMachine.couchurl)
workdatabase = couchdb.connectDatabase(dbname)
jobdb = couchdb.connectDatabase("%s/jobs" % self.databaseName)
fwjrdb = couchdb.connectDatabase("%s/fwjrs" % self.databaseName)
jobs = jobdb.loadView("JobDump",
"jobsByWorkflowName",
options={
"startkey": [workflowName],
"endkey": [workflowName, {}]
})['rows']
fwjrdb.loadView("FWJRDump",
"fwjrsByWorkflowName",
options={
"startkey": [workflowName],
"endkey": [workflowName, {}]
})['rows']
self.assertEqual(len(jobs), 2 * self.nJobs)
from WMCore.WMBS.CreateWMBSBase import CreateWMBSBase
create = CreateWMBSBase()
tables = []
for x in create.requiredTables:
tables.append(x[2:])
self.populateWorkflowWithCompleteStatus()
testTaskArchiver = TaskArchiverPoller(config=config)
testTaskArchiver.algorithm()
cleanCouch = CleanCouchPoller(config=config)
cleanCouch.setup()
cleanCouch.algorithm()
result = myThread.dbi.processData(
"SELECT * FROM wmbs_job")[0].fetchall()
self.assertEqual(len(result), 0)
result = myThread.dbi.processData(
"SELECT * FROM wmbs_subscription")[0].fetchall()
self.assertEqual(len(result), 0)
result = myThread.dbi.processData(
"SELECT * FROM wmbs_jobgroup")[0].fetchall()
self.assertEqual(len(result), 0)
result = myThread.dbi.processData(
"SELECT * FROM wmbs_fileset")[0].fetchall()
self.assertEqual(len(result), 0)
result = myThread.dbi.processData(
"SELECT * FROM wmbs_file_details")[0].fetchall()
self.assertEqual(len(result), 0)
# Make sure we deleted the directory
self.assertFalse(os.path.exists(cachePath))
self.assertFalse(
os.path.exists(
os.path.join(self.testDir, 'workloadTest/TestWorkload')))
testWMBSFileset = Fileset(id=1)
self.assertEqual(testWMBSFileset.exists(), False)
workloadSummary = workdatabase.document(id="TestWorkload")
# Check ACDC
self.assertEqual(workloadSummary['ACDCServer'],
sanitizeURL(config.ACDC.couchurl)['url'])
# Check the output
self.assertEqual(workloadSummary['output'].keys(),
['/Electron/MorePenguins-v0/RECO'])
self.assertEqual(
sorted(workloadSummary['output']['/Electron/MorePenguins-v0/RECO']
['tasks']),
['/TestWorkload/ReReco', '/TestWorkload/ReReco/LogCollect'])
# Check performance
# Check histograms
self.assertAlmostEqual(
workloadSummary['performance']['/TestWorkload/ReReco']['cmsRun1']
['AvgEventTime']['histogram'][0]['average'],
0.89405199999999996,
places=2)
self.assertEqual(
workloadSummary['performance']['/TestWorkload/ReReco']['cmsRun1']
['AvgEventTime']['histogram'][0]['nEvents'], 10)
# Check standard performance
self.assertAlmostEqual(
workloadSummary['performance']['/TestWorkload/ReReco']['cmsRun1']
['TotalJobCPU']['average'],
17.786300000000001,
places=2)
self.assertAlmostEqual(
workloadSummary['performance']['/TestWorkload/ReReco']['cmsRun1']
['TotalJobCPU']['stdDev'],
0.0,
places=2)
# Check worstOffenders
self.assertEqual(
workloadSummary['performance']['/TestWorkload/ReReco']['cmsRun1']
['AvgEventTime']['worstOffenders'], [{
'logCollect': None,
'log': None,
'value': '0.894052',
'jobID': 1
}, {
'logCollect': None,
'log': None,
'value': '0.894052',
'jobID': 1
}, {
'logCollect': None,
'log': None,
'value': '0.894052',
'jobID': 2
}])
# Check retryData
self.assertEqual(workloadSummary['retryData']['/TestWorkload/ReReco'],
{'1': 10})
logCollectPFN = 'srm://srm-cms.cern.ch:8443/srm/managerv2?SFN=/castor/cern.ch/cms/store/logs/prod/2012/11/WMAgent/Run206446-MinimumBias-Run2012D-v1-Tier1PromptReco-4af7e658-23a4-11e2-96c7-842b2b4671d8/Run206446-MinimumBias-Run2012D-v1-Tier1PromptReco-4af7e658-23a4-11e2-96c7-842b2b4671d8-AlcaSkimLogCollect-1-logs.tar'
self.assertEqual(workloadSummary['logArchives'], {
'/TestWorkload/ReReco/LogCollect':
[logCollectPFN for _ in range(10)]
})
# LogCollect task is made out of identical FWJRs
# assert that it is identical
for x in workloadSummary['performance'][
'/TestWorkload/ReReco/LogCollect']['cmsRun1'].keys():
if x in config.TaskArchiver.histogramKeys:
continue
for y in ['average', 'stdDev']:
self.assertAlmostEqual(
workloadSummary['performance']
['/TestWorkload/ReReco/LogCollect']['cmsRun1'][x][y],
workloadSummary['performance']['/TestWorkload/ReReco']
['cmsRun1'][x][y],
places=2)
return
def testB_testErrors(self):
"""
_testErrors_
Test with a failed FWJR
"""
config = self.getConfig()
workloadPath = os.path.join(self.testDir, 'specDir', 'spec.pkl')
workload = self.createWorkload(workloadName=workloadPath)
testJobGroup = self.createTestJobGroup(config=config,
name=workload.name(),
specLocation=workloadPath,
error=True)
# Create second workload
testJobGroup2 = self.createTestJobGroup(
config=config,
name=workload.name(),
filesetName="TestFileset_2",
specLocation=workloadPath,
task="/TestWorkload/ReReco/LogCollect",
jobType="LogCollect")
cachePath = os.path.join(config.JobCreator.jobCacheDir, "TestWorkload",
"ReReco")
os.makedirs(cachePath)
self.assertTrue(os.path.exists(cachePath))
couchdb = CouchServer(config.JobStateMachine.couchurl)
jobdb = couchdb.connectDatabase("%s/jobs" % self.databaseName)
fwjrdb = couchdb.connectDatabase("%s/fwjrs" % self.databaseName)
jobdb.loadView("JobDump",
"jobsByWorkflowName",
options={
"startkey": [workload.name()],
"endkey": [workload.name(), {}]
})['rows']
fwjrdb.loadView("FWJRDump",
"fwjrsByWorkflowName",
options={
"startkey": [workload.name()],
"endkey": [workload.name(), {}]
})['rows']
self.populateWorkflowWithCompleteStatus()
testTaskArchiver = TaskArchiverPoller(config=config)
testTaskArchiver.algorithm()
cleanCouch = CleanCouchPoller(config=config)
cleanCouch.setup()
cleanCouch.algorithm()
dbname = getattr(config.JobStateMachine, "couchDBName")
workdatabase = couchdb.connectDatabase("%s/workloadsummary" % dbname)
workloadSummary = workdatabase.document(id=workload.name())
self.assertEqual(
workloadSummary['errors']['/TestWorkload/ReReco']['failureTime'],
500)
self.assertTrue('99999' in workloadSummary['errors']
['/TestWorkload/ReReco']['cmsRun1'])
failedRunInfo = workloadSummary['errors']['/TestWorkload/ReReco'][
'cmsRun1']['99999']['runs']
self.assertEqual(
failedRunInfo, {'10': [[12312, 12312]]},
"Wrong lumi information in the summary for failed jobs")
# Check the failures by site histograms
self.assertEqual(
workloadSummary['histograms']['workflowLevel']['failuresBySite']
['data']['T1_IT_CNAF']['Failed Jobs'], 10)
self.assertEqual(
workloadSummary['histograms']['stepLevel']['/TestWorkload/ReReco']
['cmsRun1']['errorsBySite']['data']['T1_IT_CNAF']['99999'], 10)
self.assertEqual(
workloadSummary['histograms']['stepLevel']['/TestWorkload/ReReco']
['cmsRun1']['errorsBySite']['data']['T1_IT_CNAF']['8020'], 10)
self.assertEqual(
workloadSummary['histograms']['workflowLevel']['failuresBySite']
['average']['Failed Jobs'], 10)
self.assertEqual(
workloadSummary['histograms']['stepLevel']['/TestWorkload/ReReco']
['cmsRun1']['errorsBySite']['average']['99999'], 10)
self.assertEqual(
workloadSummary['histograms']['stepLevel']['/TestWorkload/ReReco']
['cmsRun1']['errorsBySite']['average']['8020'], 10)
self.assertEqual(
workloadSummary['histograms']['workflowLevel']['failuresBySite']
['stdDev']['Failed Jobs'], 0)
self.assertEqual(
workloadSummary['histograms']['stepLevel']['/TestWorkload/ReReco']
['cmsRun1']['errorsBySite']['stdDev']['99999'], 0)
self.assertEqual(
workloadSummary['histograms']['stepLevel']['/TestWorkload/ReReco']
['cmsRun1']['errorsBySite']['stdDev']['8020'], 0)
return
@attr("integration")
def testC_Profile(self):
"""
_Profile_
DON'T RUN THIS!
"""
import cProfile
import pstats
name = makeUUID()
config = self.getConfig()
jobList = self.createGiantJobSet(name=name,
config=config,
nSubs=10,
nJobs=1000,
nFiles=10)
cleanCouch = CleanCouchPoller(config=config)
cleanCouch.setup()
cProfile.runctx("cleanCouch.algorithm()",
globals(),
locals(),
filename="testStats.stat")
p = pstats.Stats('testStats.stat')
p.sort_stats('cumulative')
p.print_stats()
return
@attr("integration")
def testD_Timing(self):
"""
_Timing_
This is to see how fast things go.
"""
myThread = threading.currentThread()
name = makeUUID()
config = self.getConfig()
jobList = self.createGiantJobSet(name=name,
config=config,
nSubs=10,
nJobs=1000,
nFiles=10)
testTaskArchiver = TaskArchiverPoller(config=config)
startTime = time.time()
testTaskArchiver.algorithm()
stopTime = time.time()
result = myThread.dbi.processData(
"SELECT * FROM wmbs_job")[0].fetchall()
self.assertEqual(len(result), 0)
result = myThread.dbi.processData(
"SELECT * FROM wmbs_subscription")[0].fetchall()
self.assertEqual(len(result), 0)
result = myThread.dbi.processData(
"SELECT * FROM wmbs_jobgroup")[0].fetchall()
self.assertEqual(len(result), 0)
result = myThread.dbi.processData(
"SELECT * FROM wmbs_file_details")[0].fetchall()
self.assertEqual(len(result), 0)
testWMBSFileset = Fileset(id=1)
self.assertEqual(testWMBSFileset.exists(), False)
logging.info("TaskArchiver took %f seconds", (stopTime - startTime))
def testDQMRecoPerformanceToDashBoard(self):
myThread = threading.currentThread()
listRunsWorkflow = self.dbsDaoFactory(classname="ListRunsWorkflow")
# Didn't like to have done that, but the test doesn't provide all info I need in the system, so faking it:
myThread.dbi.processData(
"""insert into dbsbuffer_workflow(id, name) values (1, 'TestWorkload')""",
transaction=False)
myThread.dbi.processData(
"""insert into dbsbuffer_file (id, lfn, dataset_algo, workflow) values (1, '/store/t/e/s/t.test', 1, 1)""",
transaction=False)
myThread.dbi.processData(
"""insert into dbsbuffer_file (id, lfn, dataset_algo, workflow) values (2, '/store/t/e/s/t.test2', 1, 1)""",
transaction=False)
myThread.dbi.processData(
"""insert into dbsbuffer_file_runlumi_map (run, lumi, filename) values (207214, 100, 1)""",
transaction=False)
myThread.dbi.processData(
"""insert into dbsbuffer_file_runlumi_map (run, lumi, filename) values (207215, 200, 2)""",
transaction=False)
config = self.getConfig()
dqmUrl = getattr(config.TaskArchiver, "dqmUrl")
perfDashBoardMinLumi = getattr(config.TaskArchiver,
"perfDashBoardMinLumi")
perfDashBoardMaxLumi = getattr(config.TaskArchiver,
"perfDashBoardMaxLumi")
dashBoardUrl = getattr(config.TaskArchiver, "dashBoardUrl")
workloadPath = os.path.join(self.testDir, 'specDir', 'spec.pkl')
workload = self.createWorkload(workloadName=workloadPath)
testJobGroup = self.createTestJobGroup(config=config,
name=workload.name(),
specLocation=workloadPath,
error=True)
testJobGroup2 = self.createTestJobGroup(
config=config,
name=workload.name(),
filesetName="TestFileset_2",
specLocation=workloadPath,
task="/TestWorkload/ReReco/LogCollect",
jobType="LogCollect")
# Adding request type as ReReco, real ReqMgr requests have it
workload.data.request.section_("schema")
workload.data.request.schema.RequestType = "ReReco"
workload.data.request.schema.CMSSWVersion = 'test_compops_CMSSW_5_3_6_patch1'
workload.getTask('ReReco').addInputDataset(name='/a/b/c',
primary='a',
processed='b',
tier='c')
interestingPDs = getattr(config.TaskArchiver, "perfPrimaryDatasets")
interestingDatasets = []
# Are the datasets from this request interesting? Do they have DQM output? One might ask afterwards if they have harvest
for dataset in workload.listOutputDatasets():
(nothing, PD, procDataSet, dataTier) = dataset.split('/')
if PD in interestingPDs and dataTier == "DQM":
interestingDatasets.append(dataset)
# We should have found 1 interesting dataset
self.assertAlmostEqual(len(interestingDatasets), 1)
if len(interestingDatasets) == 0:
return
# Request will be only interesting for performance if it's a ReReco or PromptReco
(isReReco, isPromptReco) = (False, False)
if getattr(workload.data.request.schema, "RequestType",
None) == 'ReReco':
isReReco = True
# Yes, few people like magic strings, but have a look at :
# https://github.com/dmwm/T0/blob/master/src/python/T0/RunConfig/RunConfigAPI.py#L718
# Might be safe enough
# FIXME: in TaskArchiver, add a test to make sure that the dataset makes sense (procDataset ~= /a/ERA-PromptReco-vVERSON/DQM)
if re.search('PromptReco', workload.name()):
isPromptReco = True
if not (isReReco or isPromptReco):
return
self.assertTrue(isReReco)
self.assertFalse(isPromptReco)
# We are not interested if it's not a PromptReco or a ReReco
if not (isReReco or isPromptReco):
return
if isReReco:
release = getattr(workload.data.request.schema, "CMSSWVersion")
if not release:
logging.info("no release for %s, bailing out", workload.name())
else:
release = getattr(
workload.tasks.Reco.steps.cmsRun1.application.setup,
"cmsswVersion")
if not release:
logging.info("no release for %s, bailing out", workload.name())
self.assertEqual(release, "test_compops_CMSSW_5_3_6_patch1")
# If all is true, get the run numbers processed by this worklfow
runList = listRunsWorkflow.execute(workflow=workload.name())
self.assertEqual([207214, 207215], runList)
# GO to DQM GUI, get what you want
# https://cmsweb.cern.ch/dqm/offline/jsonfairy/archive/211313/PAMuon/HIRun2013-PromptReco-v1/DQM/DQM/TimerService/event
for dataset in interestingDatasets:
(nothing, PD, procDataSet, dataTier) = dataset.split('/')
worthPoints = {}
for run in runList:
responseJSON = self.getPerformanceFromDQM(dqmUrl, dataset, run)
worthPoints.update(
self.filterInterestingPerfPoints(responseJSON,
perfDashBoardMinLumi,
perfDashBoardMaxLumi))
# Publish dataset performance to DashBoard.
if not self.publishPerformanceDashBoard(dashBoardUrl, PD, release,
worthPoints):
logging.info(
"something went wrong when publishing dataset %s to DashBoard",
dataset)
return
class Tier0FeederPoller(BaseWorkerThread):
def __init__(self, config):
"""
_init_
"""
BaseWorkerThread.__init__(self)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="T0.WMBS",
logger=logging,
dbinterface=myThread.dbi)
self.tier0ConfigFile = config.Tier0Feeder.tier0ConfigFile
self.specDirectory = config.Tier0Feeder.specDirectory
self.dropboxuser = getattr(config.Tier0Feeder, "dropboxuser", None)
self.dropboxpass = getattr(config.Tier0Feeder, "dropboxpass", None)
self.dqmUploadProxy = getattr(config.Tier0Feeder, "dqmUploadProxy",
None)
self.serviceProxy = getattr(config.Tier0Feeder, "serviceProxy", None)
self.localRequestCouchDB = RequestDBWriter(
config.AnalyticsDataCollector.localT0RequestDBURL,
couchapp=config.AnalyticsDataCollector.RequestCouchApp)
self.injectedRuns = set()
hltConfConnectUrl = config.HLTConfDatabase.connectUrl
dbFactoryHltConf = DBFactory(logging,
dburl=hltConfConnectUrl,
options={})
self.dbInterfaceHltConf = dbFactoryHltConf.connect()
daoFactoryHltConf = DAOFactory(package="T0.WMBS",
logger=logging,
dbinterface=self.dbInterfaceHltConf)
self.getHLTConfigDAO = daoFactoryHltConf(
classname="RunConfig.GetHLTConfig")
storageManagerConnectUrl = config.StorageManagerDatabase.connectUrl
dbFactoryStorageManager = DBFactory(logging,
dburl=storageManagerConnectUrl,
options={})
self.dbInterfaceStorageManager = dbFactoryStorageManager.connect()
self.dbInterfaceSMNotify = None
if hasattr(config, "SMNotifyDatabase"):
smNotifyConnectUrl = config.SMNotifyDatabase.connectUrl
dbFactorySMNotify = DBFactory(logging,
dburl=smNotifyConnectUrl,
options={})
self.dbInterfaceSMNotify = dbFactorySMNotify.connect()
self.getExpressReadyRunsDAO = None
if hasattr(config, "PopConLogDatabase"):
popConLogConnectUrl = getattr(config.PopConLogDatabase,
"connectUrl", None)
if popConLogConnectUrl != None:
dbFactoryPopConLog = DBFactory(logging,
dburl=popConLogConnectUrl,
options={})
dbInterfacePopConLog = dbFactoryPopConLog.connect()
daoFactoryPopConLog = DAOFactory(
package="T0.WMBS",
logger=logging,
dbinterface=dbInterfacePopConLog)
self.getExpressReadyRunsDAO = daoFactoryPopConLog(
classname="Tier0Feeder.GetExpressReadyRuns")
self.haveT0DataSvc = False
if hasattr(config, "T0DataSvcDatabase"):
t0datasvcConnectUrl = getattr(config.T0DataSvcDatabase,
"connectUrl", None)
if t0datasvcConnectUrl != None:
self.haveT0DataSvc = True
dbFactoryT0DataSvc = DBFactory(logging,
dburl=t0datasvcConnectUrl,
options={})
dbInterfaceT0DataSvc = dbFactoryT0DataSvc.connect()
self.daoFactoryT0DataSvc = DAOFactory(
package="T0.WMBS",
logger=logging,
dbinterface=dbInterfaceT0DataSvc)
#
# Set deployment ID
#
SetDeploymentIdDAO = self.daoFactory(
classname="Tier0Feeder.SetDeploymentID")
GetDeploymentIdDAO = self.daoFactory(
classname="Tier0Feeder.GetDeploymentID")
try:
self.deployID = GetDeploymentIdDAO.execute()
if self.deployID == 0:
self.deployID = int(
datetime.datetime.now().strftime("%y%m%d%H%M%S"))
SetDeploymentIdDAO.execute(self.deployID)
except:
logging.exception(
"Something went wrong with setting deployment ID")
raise
return
@timeFunction
def algorithm(self, parameters=None):
"""
_algorithm_
"""
logging.debug("Running Tier0Feeder algorithm...")
myThread = threading.currentThread()
findNewRunsDAO = self.daoFactory(classname="Tier0Feeder.FindNewRuns")
findNewRunStreamsDAO = self.daoFactory(
classname="Tier0Feeder.FindNewRunStreams")
findNewExpressRunsDAO = self.daoFactory(
classname="Tier0Feeder.FindNewExpressRuns")
releaseExpressDAO = self.daoFactory(
classname="Tier0Feeder.ReleaseExpress")
feedStreamersDAO = self.daoFactory(
classname="Tier0Feeder.FeedStreamers")
markWorkflowsInjectedDAO = self.daoFactory(
classname="Tier0Feeder.MarkWorkflowsInjected")
tier0Config = None
try:
tier0Config = loadConfigurationFile(self.tier0ConfigFile)
except:
# usually happens when there are syntax errors in the configuration
logging.exception(
"Cannot load Tier0 configuration file, not configuring new runs and run/streams"
)
# only configure new runs and run/streams if we have a valid Tier0 configuration
if tier0Config != None:
#
# we don't inject data if the Tier0Config is unreadable
#
# discover new data from StorageManager and inject into Tier0
# (if the config specifies a list of runs do it only once)
#
# replays call data discovery only once (and ignore data status)
#
try:
if tier0Config.Global.InjectRuns == None:
StorageManagerAPI.injectNewData(
self.dbInterfaceStorageManager,
self.dbInterfaceHltConf,
self.dbInterfaceSMNotify,
streamerPNN=tier0Config.Global.StreamerPNN,
minRun=tier0Config.Global.InjectMinRun,
maxRun=tier0Config.Global.InjectMaxRun)
else:
injectRuns = set()
for injectRun in tier0Config.Global.InjectRuns:
if injectRun not in self.injectedRuns:
injectRuns.add(injectRun)
for injectRun in injectRuns:
StorageManagerAPI.injectNewData(
self.dbInterfaceStorageManager,
self.dbInterfaceHltConf,
self.dbInterfaceSMNotify,
streamerPNN=tier0Config.Global.StreamerPNN,
injectRun=injectRun)
self.injectedRuns.add(injectRun)
except:
# shouldn't happen, just a catch all insurance
logging.exception(
"Something went wrong with data retrieval from StorageManager"
)
#
# Set deployment ID
#
setDeploymentId(tier0Config, self.deployID)
logging.info("Deploy ID: %d" % tier0Config.Global.DeploymentID)
#
# find new runs, setup global run settings and stream/dataset/trigger mapping
#
runHltkeys = findNewRunsDAO.execute(transaction=False)
for run, hltkey in sorted(runHltkeys.items()):
hltConfig = None
# local runs have no hltkey and are configured differently
if hltkey != None:
# retrieve HLT configuration and make sure it's usable
try:
hltConfig = self.getHLTConfigDAO.execute(
hltkey, transaction=False)
if hltConfig['process'] == None or len(
hltConfig['mapping']) == 0:
raise RuntimeError(
"HLTConfDB query returned no process or mapping"
)
except:
logging.exception(
"Can't retrieve hltkey %s for run %d" %
(hltkey, run))
continue
try:
RunConfigAPI.configureRun(tier0Config, run, hltConfig)
except:
logging.exception("Can't configure for run %d" % (run))
#
# find unconfigured run/stream with data
# populate RunConfig, setup workflows/filesets/subscriptions
#
runStreams = findNewRunStreamsDAO.execute(transaction=False)
for run in sorted(runStreams.keys()):
for stream in sorted(runStreams[run]):
try:
RunConfigAPI.configureRunStream(
tier0Config, run, stream, self.specDirectory,
self.dqmUploadProxy)
except:
logging.exception(
"Can't configure for run %d and stream %s" %
(run, stream))
#
# stop and close runs based on RunSummary and StorageManager records
#
RunLumiCloseoutAPI.stopRuns(self.dbInterfaceStorageManager)
RunLumiCloseoutAPI.closeRuns(self.dbInterfaceStorageManager)
#
# release runs for Express
#
runs = findNewExpressRunsDAO.execute(transaction=False)
if len(runs) > 0:
binds = []
for run in runs:
binds.append({'RUN': run})
if self.getExpressReadyRunsDAO != None:
runs = self.getExpressReadyRunsDAO.execute(binds=binds,
transaction=False)
if len(runs) > 0:
binds = []
for run in runs:
binds.append({'RUN': run})
releaseExpressDAO.execute(binds=binds, transaction=False)
#
# release runs for PromptReco
# check PromptRecoStatus first, i.e. big red button
#
if self.getPromptRecoStatusT0DataSvc():
RunConfigAPI.releasePromptReco(tier0Config, self.specDirectory,
self.dqmUploadProxy)
#
# insert express and reco configs into Tier0 Data Service
#
if self.haveT0DataSvc:
self.updateRunConfigT0DataSvc()
self.updateRunStreamDoneT0DataSvc()
self.updateExpressConfigsT0DataSvc()
self.updateRecoConfigsT0DataSvc()
self.updateRecoReleaseConfigsT0DataSvc()
self.lockDatasetsT0DataSvc()
#
# mark express and repack workflows as injected if certain conditions are met
# (we don't do it immediately to prevent the TaskArchiver from cleaning up too early)
#
markWorkflowsInjectedDAO.execute(self.dbInterfaceSMNotify != None,
transaction=False)
#
# close stream/lumis for run/streams that are active (fileset exists and open)
#
RunLumiCloseoutAPI.closeLumiSections(self.dbInterfaceStorageManager)
#
# feed new data into exisiting filesets
#
try:
myThread.transaction.begin()
feedStreamersDAO.execute(conn=myThread.transaction.conn,
transaction=True)
except:
logging.exception("Can't feed data, bailing out...")
raise
else:
myThread.transaction.commit()
#
# run ended and run/stream fileset open
# => check for complete lumi_closed record, all lumis finally closed and all data feed
# => if all conditions satisfied, close the run/stream fileset
#
RunLumiCloseoutAPI.closeRunStreamFilesets()
#
# check and delete active split lumis
#
RunLumiCloseoutAPI.checkActiveSplitLumis()
#
# insert workflows into CouchDB for monitoring
#
self.feedCouchMonitoring()
#
# Update Couch when Repack and Express have closed input filesets (analog to old T0 closeout)
#
self.closeOutRealTimeWorkflows()
#
# send repacked notifications to StorageManager
#
if self.dbInterfaceSMNotify:
StorageManagerAPI.markRepacked(self.dbInterfaceSMNotify)
#
# upload PCL conditions to DropBox
#
ConditionUploadAPI.uploadConditions(self.dropboxuser, self.dropboxpass,
self.serviceProxy)
return
def feedCouchMonitoring(self):
"""
_feedCouchMonitoring_
check for workflows that haven't been uploaded to Couch for monitoring yet
"""
getStreamerWorkflowsForMonitoringDAO = self.daoFactory(
classname="Tier0Feeder.GetStreamerWorkflowsForMonitoring")
getPromptRecoWorkflowsForMonitoringDAO = self.daoFactory(
classname="Tier0Feeder.GetPromptRecoWorkflowsForMonitoring")
markTrackedWorkflowMonitoringDAO = self.daoFactory(
classname="Tier0Feeder.MarkTrackedWorkflowMonitoring")
workflows = getStreamerWorkflowsForMonitoringDAO.execute()
workflows += getPromptRecoWorkflowsForMonitoringDAO.execute()
if len(workflows) == 0:
logging.debug(
"No workflows to publish to couch monitoring, doing nothing")
if workflows:
logging.debug(" Going to publish %d workflows" % len(workflows))
for (workflowId, run, workflowName) in workflows:
logging.info(" Publishing workflow %s to monitoring" %
workflowName)
#TODO: add more information about workflow if there need to be kept longer than
# workflow life cycle.
doc = {}
doc["RequestName"] = workflowName
doc["Run"] = run
response = self.localRequestCouchDB.insertGenericRequest(doc)
if response == "OK" or "EXISTS":
logging.info(" Successfully uploaded request %s" %
workflowName)
# Here we have to trust the insert, if it doesn't happen will be easy to spot on the logs
markTrackedWorkflowMonitoringDAO.execute(workflowId)
return
def closeOutRealTimeWorkflows(self):
"""
_closeOutRealTimeWorkflows_
Updates couch with the closeout status of Repack and Express
PromptReco should be closed out automatically
"""
getNotClosedOutWorkflowsDAO = self.daoFactory(
classname="Tier0Feeder.GetNotClosedOutWorkflows")
workflows = getNotClosedOutWorkflowsDAO.execute()
if len(workflows) == 0:
logging.debug(
"No workflows to publish to couch monitoring, doing nothing")
if workflows:
for workflow in workflows:
(workflowId, filesetId, filesetOpen, workflowName) = workflow
# find returns -1 if the string is not found
if workflowName.find('PromptReco') >= 0:
logging.debug(
"Closing out instantaneously PromptReco Workflow %s" %
workflowName)
self.updateClosedState(workflowName, workflowId)
else:
# Check if fileset (which you already know) is closed or not
# FIXME: No better way to do it? what comes from the DAO is a string, casting bool or int doesn't help much.
# Works like that :
if filesetOpen == '0':
self.updateClosedState(workflowName, workflowId)
return
def updateClosedState(self, workflowName, workflowId):
"""
_updateClosedState_
Mark a workflow as Closed
"""
markCloseoutWorkflowMonitoringDAO = self.daoFactory(
classname="Tier0Feeder.MarkCloseoutWorkflowMonitoring")
response = self.localRequestCouchDB.updateRequestStatus(
workflowName, 'Closed')
if response == "OK" or "EXISTS":
logging.debug("Successfully closed workflow %s" % workflowName)
markCloseoutWorkflowMonitoringDAO.execute(workflowId)
return
def getPromptRecoStatusT0DataSvc(self):
"""
_getPromptRecoStatusDataSvc_
Check the PromptRecoStatus (enabled/disabled) set by the ORM
"""
getPromptRecoStatusDAO = self.daoFactoryT0DataSvc(
classname="T0DataSvc.GetPromptRecoStatus")
status = getPromptRecoStatusDAO.execute(transaction=False)
return status
def updateRunConfigT0DataSvc(self):
"""
_updateRunConfigT0DataSvc_
Check for new runs and push their info into the Tier0 Data Service.
"""
getNewRunDAO = self.daoFactory(classname="T0DataSvc.GetNewRun")
newRun = getNewRunDAO.execute(transaction=False)
if len(newRun) > 0:
binds = []
for runInfo in newRun:
binds.append({
'RUN': runInfo['run'],
'ACQ_ERA': runInfo['acq_era']
})
insertNewRunDAO = self.daoFactoryT0DataSvc(
classname="T0DataSvc.InsertNewRun")
insertNewRunDAO.execute(binds=binds, transaction=False)
for bind in binds:
del bind['ACQ_ERA']
updateNewRunDAO = self.daoFactory(
classname="T0DataSvc.UpdateNewRun")
updateNewRunDAO.execute(binds=binds, transaction=False)
return
def updateRunStreamDoneT0DataSvc(self):
"""
_updateRunStreamDoneT0DataSvc_
Check if a run/stream workflow (express or repack) is finished and
cleaned up and push a completion record into the Tier0 Data Service.
"""
getRunStreamDoneDAO = self.daoFactory(
classname="T0DataSvc.GetRunStreamDone")
runStreamDone = getRunStreamDoneDAO.execute(transaction=False)
if len(runStreamDone) > 0:
binds = []
for runStream in runStreamDone:
binds.append({
'RUN': runStream['run'],
'STREAM': runStream['stream']
})
insertRunStreamDoneDAO = self.daoFactoryT0DataSvc(
classname="T0DataSvc.InsertRunStreamDone")
insertRunStreamDoneDAO.execute(binds=binds, transaction=False)
updateRunStreamDoneDAO = self.daoFactory(
classname="T0DataSvc.UpdateRunStreamDone")
updateRunStreamDoneDAO.execute(binds=binds, transaction=False)
return
def updateExpressConfigsT0DataSvc(self):
"""
_updateExpressConfigsT0DataSvc_
Check which express_config rows are missing
in the Tier0 Data Service and insert them,
also record that fact in t0ast
"""
getExpressConfigsDAO = self.daoFactory(
classname="T0DataSvc.GetExpressConfigs")
expressConfigs = getExpressConfigsDAO.execute(transaction=False)
if len(expressConfigs) > 0:
bindsInsert = []
bindsUpdate = []
for config in expressConfigs:
bindsInsert.append({
'RUN': config['run'],
'STREAM': config['stream'],
'CMSSW': config['cmssw'],
'SCRAM_ARCH': config['scram_arch'],
'RECO_CMSSW': config['reco_cmssw'],
'RECO_SCRAM_ARCH': config['reco_scram_arch'],
'ALCA_SKIM': config['alca_skim'],
'DQM_SEQ': config['dqm_seq'],
'GLOBAL_TAG': config['global_tag'][:50],
'SCENARIO': config['scenario'],
'MULTICORE': config['multicore'],
'WRITE_TIERS': config['write_tiers'],
'WRITE_DQM': config['write_dqm']
})
bindsUpdate.append({
'RUN': config['run'],
'STREAM': config['stream']
})
insertExpressConfigsDAO = self.daoFactoryT0DataSvc(
classname="T0DataSvc.InsertExpressConfigs")
insertExpressConfigsDAO.execute(binds=bindsInsert,
transaction=False)
updateExpressConfigsDAO = self.daoFactory(
classname="T0DataSvc.UpdateExpressConfigs")
updateExpressConfigsDAO.execute(binds=bindsUpdate,
transaction=False)
return
def updateRecoConfigsT0DataSvc(self):
"""
_updateRecoConfigsT0DataSvc_
Check which reco_config rows are missing
in the Tier0 Data Service and insert them,
also record that fact in t0ast
"""
getRecoConfigsDAO = self.daoFactory(
classname="T0DataSvc.GetRecoConfigs")
recoConfigs = getRecoConfigsDAO.execute(transaction=False)
if len(recoConfigs) > 0:
bindsInsert = []
bindsUpdate = []
for config in recoConfigs:
bindsInsert.append({
'RUN': config['run'],
'PRIMDS': config['primds'],
'CMSSW': config['cmssw'],
'SCRAM_ARCH': config['scram_arch'],
'ALCA_SKIM': config['alca_skim'],
'PHYSICS_SKIM': config['physics_skim'],
'DQM_SEQ': config['dqm_seq'],
'GLOBAL_TAG': config['global_tag'][:50],
'SCENARIO': config['scenario'],
'MULTICORE': config['multicore'],
'WRITE_RECO': config['write_reco'],
'WRITE_DQM': config['write_dqm'],
'WRITE_AOD': config['write_aod'],
'WRITE_MINIAOD': config['write_miniaod']
})
bindsUpdate.append({
'RUN': config['run'],
'PRIMDS': config['primds']
})
insertRecoConfigsDAO = self.daoFactoryT0DataSvc(
classname="T0DataSvc.InsertRecoConfigs")
insertRecoConfigsDAO.execute(binds=bindsInsert, transaction=False)
updateRecoConfigsDAO = self.daoFactory(
classname="T0DataSvc.UpdateRecoConfigs")
updateRecoConfigsDAO.execute(binds=bindsUpdate, transaction=False)
return
def updateRecoReleaseConfigsT0DataSvc(self):
"""
_updateRecoReleaseConfigsT0DataSvc_
Insert information about PromptReco release into the Tier0 Data Service.
That means updating the reco_locked records in run granularity (where one
released dataset means the whole run is locked).
Also insert and update the run_primds_done records to track PromptReco status.
"""
getRunDatasetNewDAO = self.daoFactory(
classname="Tier0Feeder.GetRunDatasetNew")
getRunDatasetReleasedDAO = self.daoFactory(
classname="Tier0Feeder.GetRunDatasetReleased")
getRunDatasetDoneDAO = self.daoFactory(
classname="Tier0Feeder.GetRunDatasetDone")
updateRecoReleaseConfigsDAO = self.daoFactory(
classname="Tier0Feeder.UpdateRecoReleaseConfigs")
insertRecoLockedDAO = self.daoFactoryT0DataSvc(
classname="T0DataSvc.InsertRecoLocked")
updateRecoLockedDAO = self.daoFactoryT0DataSvc(
classname="T0DataSvc.UpdateRecoLocked")
insertRunDatasetDoneDAO = self.daoFactoryT0DataSvc(
classname="T0DataSvc.InsertRunDatasetDone")
updateRunDatasetDoneDAO = self.daoFactoryT0DataSvc(
classname="T0DataSvc.UpdateRunDatasetDone")
# first check for records that are completely new
# insert the two Tier0 Data Service records for them
# update reco release records accordingly
runDatasetNew = getRunDatasetNewDAO.execute(transaction=False)
foundRuns = set()
bindsInsertLocked = []
bindsInsertDone = []
bindsUpdate = []
for runDataset in runDatasetNew:
run = runDataset['run']
if run not in foundRuns:
bindsInsertLocked.append({'RUN': run})
foundRuns.add(run)
bindsInsertDone.append({
'RUN': run,
'PRIMDS': runDataset['primds']
})
bindsUpdate.append({
'RUN': run,
'PRIMDS_ID': runDataset['primds_id'],
'IN_DATASVC': 1
})
if len(bindsInsertLocked) > 0:
insertRecoLockedDAO.execute(binds=bindsInsertLocked,
transaction=False)
if len(bindsInsertDone) > 0:
insertRunDatasetDoneDAO.execute(binds=bindsInsertDone,
transaction=False)
if len(bindsUpdate) > 0:
updateRecoReleaseConfigsDAO.execute(binds=bindsUpdate,
transaction=False)
# then check for reco release and lock runs in the Tier0 Data Service
runDatasetReleased = getRunDatasetReleasedDAO.execute(
transaction=False)
foundRuns = set()
bindsUpdateLocked = []
bindsUpdate = []
for runDataset in runDatasetReleased:
run = runDataset['run']
if run not in foundRuns:
bindsUpdateLocked.append({'RUN': run})
foundRuns.add(run)
bindsUpdate.append({
'RUN': run,
'PRIMDS_ID': runDataset['primds_id'],
'IN_DATASVC': 2
})
if len(bindsUpdateLocked) > 0:
updateRecoLockedDAO.execute(binds=bindsUpdateLocked,
transaction=False)
if len(bindsUpdate) > 0:
updateRecoReleaseConfigsDAO.execute(binds=bindsUpdate,
transaction=False)
# finally check for reco completions and mark this in the Tier0 Data Service
runDatasetDone = getRunDatasetDoneDAO.execute(transaction=False)
bindsUpdateDone = []
bindsUpdate = []
for runDataset in runDatasetDone:
run = runDataset['run']
bindsUpdateDone.append({
'RUN': run,
'PRIMDS': runDataset['primds']
})
bindsUpdate.append({
'RUN': run,
'PRIMDS_ID': runDataset['primds_id'],
'IN_DATASVC': 3
})
if len(bindsUpdateDone) > 0:
updateRunDatasetDoneDAO.execute(binds=bindsUpdateDone,
transaction=False)
if len(bindsUpdate) > 0:
updateRecoReleaseConfigsDAO.execute(binds=bindsUpdate,
transaction=False)
return
def lockDatasetsT0DataSvc(self):
"""
_lockDatasetsT0DataSvc_
Publish dataset information into the Tier0 Data Service.
"""
getDatasetLockedDAO = self.daoFactory(
classname="T0DataSvc.GetDatasetLocked")
datasetConfigs = getDatasetLockedDAO.execute(transaction=False)
if len(datasetConfigs) > 0:
bindsInsert = []
bindsUpdate = []
for config in datasetConfigs:
bindsInsert.append({'PATH': config['path']})
bindsUpdate.append({'ID': config['id']})
insertDatasetLockedDAO = self.daoFactoryT0DataSvc(
classname="T0DataSvc.InsertDatasetLocked")
insertDatasetLockedDAO.execute(binds=bindsInsert,
transaction=False)
updateDatasetLockedDAO = self.daoFactory(
classname="T0DataSvc.UpdateDatasetLocked")
updateDatasetLockedDAO.execute(binds=bindsUpdate,
transaction=False)
return
def terminate(self, params):
"""
_terminate_
Kill the code after one final pass when called by the master thread.
"""
logging.debug("terminating immediately")
class Tier0PluginTest(unittest.TestCase):
def setUp(self):
"""
_setUp_
Setup the test environment
"""
self.testInit = TestInit(__file__)
self.testInit.setDatabaseConnection()
self.testInit.setSchema(["WMCore.WMBS"])
self.requestCouchDB = 'wmstats_plugin_t'
self.testInit.setupCouch(self.requestCouchDB, 'T0Request')
self.testDir = self.testInit.generateWorkDir()
reqDBURL = "%s/%s" % (os.environ['COUCHURL'], self.requestCouchDB)
self.requestDBWriter = RequestDBWriter(reqDBURL, couchapp="T0Request")
self.requestDBWriter._setNoStale()
self.stateMap = {}
self.orderedStates = []
self.plugin = None
return
def tearDown(self):
"""
_tearDown_
Clear databases and delete files
"""
self.testInit.tearDownCouch()
self.testInit.clearDatabase()
self.testInit.delWorkDir()
return
def setupRepackWorkflow(self):
"""
_setupRepackWorkflow_
Populate WMBS with a repack-like workflow,
every subscription must be unfinished at first
"""
workflowName = 'Repack_Run481516_StreamZ'
mergeTasks = [
'RepackMergewrite_QuadElectron_RAW',
'RepackMergewrite_TriPhoton_RAW',
'RepackMergewrite_SingleNeutrino_RAW'
]
self.stateMap = {'Merge': [], 'Processing Done': []}
self.orderedStates = ['Merge', 'Processing Done']
# Populate WMStats
self.requestDBWriter.insertGenericRequest(
{'RequestName': workflowName})
self.requestDBWriter.updateRequestStatus(workflowName, 'Closed')
# Create a wmspec in disk
workload = newWorkload(workflowName)
repackTask = workload.newTask('Repack')
for task in mergeTasks:
repackTask.addTask(task)
repackTask.addTask('RepackCleanupUnmergedwrite_QuadElectron_RAW')
specPath = os.path.join(self.testDir, 'Repack.pkl')
workload.save(specPath)
# Populate WMBS
topFileset = Fileset(name='TestStreamerFileset')
topFileset.create()
options = {
'spec': specPath,
'owner': 'ItsAMeMario',
'name': workflowName,
'wfType': 'tier0'
}
topLevelWorkflow = Workflow(task='/%s/Repack' % workflowName,
**options)
topLevelWorkflow.create()
topLevelSub = Subscription(topFileset, topLevelWorkflow)
topLevelSub.create()
self.stateMap['Merge'].append(topFileset)
for task in mergeTasks:
mergeWorkflow = Workflow(task='/%s/Repack/%s' %
(workflowName, task),
**options)
mergeWorkflow.create()
unmergedFileset = Fileset(name='TestUnmergedFileset%s' % task)
unmergedFileset.create()
mergeSub = Subscription(unmergedFileset, mergeWorkflow)
mergeSub.create()
self.stateMap['Processing Done'].append(unmergedFileset)
cleanupWorkflow = Workflow(
task=
'/Repack_Run481516_StreamZ/Repack/RepackCleanupUnmergedwrite_QuadElectron_RAW',
**options)
cleanupWorkflow.create()
unmergedFileset = Fileset(name='TestUnmergedFilesetToCleanup')
unmergedFileset.create()
cleanupSub = Subscription(unmergedFileset, cleanupWorkflow)
cleanupSub.create()
return
def setupExpressWorkflow(self):
"""
_setupExpressWorkflow_
Populate WMBS with a express-like workflow,
every subscription must be unfinished at first
"""
workflowName = 'Express_Run481516_StreamZFast'
secondLevelTasks = [
'ExpressMergewrite_StreamZFast_DQM',
'ExpressMergewrite_ExpressPhysics_FEVT',
'ExpressAlcaSkimwrite_StreamZFast_ALCARECO',
'ExpressCleanupUnmergedwrite_StreamZFast_DQM',
'ExpressCleanupUnmergedwrite_ExpressPhysics_FEVT',
'ExpressCleanupUnmergedwrite_StreamZFast_ALCARECO'
]
alcaHarvestTask = 'ExpressAlcaSkimwrite_StreamZFast_ALCARECOAlcaHarvestALCARECOStreamPromptCalibProd'
dqmHarvestTask = 'ExpressMergewrite_StreamZFast_DQMEndOfRunDQMHarvestMerged'
self.stateMap = {'Merge': [], 'Harvesting': [], 'Processing Done': []}
self.orderedStates = ['Merge', 'Harvesting', 'Processing Done']
# Populate WMStats
self.requestDBWriter.insertGenericRequest(
{'RequestName': workflowName})
self.requestDBWriter.updateRequestStatus(workflowName, 'Closed')
# Create a wmspec in disk
workload = newWorkload(workflowName)
expressTask = workload.newTask('Express')
for task in secondLevelTasks:
secondLevelTask = expressTask.addTask(task)
if task == 'ExpressAlcaSkimwrite_StreamZFast_ALCARECO':
secondLevelTask.addTask(alcaHarvestTask)
elif task == 'ExpressMergewrite_StreamZFast_DQM':
secondLevelTask.addTask(dqmHarvestTask)
specPath = os.path.join(self.testDir, 'Express.pkl')
workload.save(specPath)
# Populate WMBS
sharedFileset = Fileset(name='TestFileset')
sharedFileset.create()
sharedFileset.markOpen(False)
options = {
'spec': specPath,
'owner': 'ItsAMeMario',
'name': workflowName,
'wfType': 'tier0'
}
topLevelWorkflow = Workflow(task='/%s/Express' % workflowName,
**options)
topLevelWorkflow.create()
topLevelSub = Subscription(sharedFileset, topLevelWorkflow)
topLevelSub.create()
self.stateMap['Merge'].append(topLevelSub)
for task in [
x for x in secondLevelTasks if not x.count('CleanupUnmerged')
]:
secondLevelWorkflow = Workflow(task='/%s/Express/%s' %
(workflowName, task),
**options)
secondLevelWorkflow.create()
mergeSub = Subscription(sharedFileset, secondLevelWorkflow)
mergeSub.create()
self.stateMap['Harvesting'].append(mergeSub)
for (parent, child) in [
('ExpressAlcaSkimwrite_StreamZFast_ALCARECO', alcaHarvestTask),
('ExpressMergewrite_StreamZFast_DQM', dqmHarvestTask)
]:
harvestingWorkflow = Workflow(task='/%s/Express/%s/%s' %
(workflowName, parent, child),
**options)
harvestingWorkflow.create()
harvestingSub = Subscription(sharedFileset, harvestingWorkflow)
harvestingSub.create()
self.stateMap['Processing Done'].append(harvestingSub)
return
def setupPromptRecoWorkflow(self):
"""
_setupPromptRecoWorkflow_
Populate WMBS with a real PromptReco workflow,
every subscription must be unfinished at first
"""
# Populate disk and WMBS
testArguments = PromptRecoWorkloadFactory.getTestArguments()
workflowName = 'PromptReco_Run195360_Cosmics'
factory = PromptRecoWorkloadFactory()
testArguments["EnableHarvesting"] = True
testArguments["CouchURL"] = os.environ["COUCHURL"]
workload = factory.factoryWorkloadConstruction(workflowName,
testArguments)
wmbsHelper = WMBSHelper(workload,
'Reco',
'SomeBlock',
cachepath=self.testDir)
wmbsHelper.createTopLevelFileset()
wmbsHelper._createSubscriptionsInWMBS(wmbsHelper.topLevelTask,
wmbsHelper.topLevelFileset)
self.stateMap = {
'AlcaSkim': [],
'Merge': [],
'Harvesting': [],
'Processing Done': []
}
self.orderedStates = [
'AlcaSkim', 'Merge', 'Harvesting', 'Processing Done'
]
# Populate WMStats
self.requestDBWriter.insertGenericRequest(
{'RequestName': workflowName})
self.requestDBWriter.updateRequestStatus(workflowName, 'Closed')
topLevelTask = '/%s/Reco' % workflowName
alcaSkimTask = '%s/AlcaSkim' % topLevelTask
mergeTasks = [
'%s/AlcaSkim/AlcaSkimMergeALCARECOStreamHcalCalHOCosmics',
'%s/AlcaSkim/AlcaSkimMergeALCARECOStreamTkAlCosmics0T',
'%s/AlcaSkim/AlcaSkimMergeALCARECOStreamMuAlGlobalCosmics',
'%s/RecoMergewrite_AOD', '%s/RecoMergewrite_DQM',
'%s/RecoMergewrite_RECO'
]
harvestingTask = '%s/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged' % topLevelTask
self.stateMap['AlcaSkim'].append(wmbsHelper.topLevelSubscription)
alcaSkimWorkflow = Workflow(name=workflowName, task=alcaSkimTask)
alcaSkimWorkflow.load()
alcarecoFileset = Fileset(
name=
'/PromptReco_Run195360_Cosmics/Reco/unmerged-write_ALCARECOALCARECO'
)
alcarecoFileset.load()
alcaSkimSub = Subscription(alcarecoFileset, alcaSkimWorkflow)
alcaSkimSub.load()
self.stateMap['Merge'].append(alcaSkimSub)
for task in mergeTasks:
mergeTask = task % topLevelTask
mergeWorkflow = Workflow(name=workflowName, task=mergeTask)
mergeWorkflow.load()
if 'AlcaSkim' in mergeTask:
stream = mergeTask.split('/')[-1][13:]
unmergedFileset = Fileset(name='%s/unmerged-%sALCARECO' %
(alcaSkimTask, stream))
unmergedFileset.load()
else:
dataTier = mergeTask.split('/')[-1].split('_')[-1]
unmergedFileset = Fileset(name='%s/unmerged-write_%s%s' %
(topLevelTask, dataTier, dataTier))
unmergedFileset.load()
mergeSub = Subscription(unmergedFileset, mergeWorkflow)
mergeSub.load()
self.stateMap['Harvesting'].append(mergeSub)
harvestingWorkflow = Workflow(name=workflowName, task=harvestingTask)
harvestingWorkflow.load()
harvestingFileset = Fileset(
name=
'/PromptReco_Run195360_Cosmics/Reco/RecoMergewrite_DQM/merged-MergedDQM'
)
harvestingFileset.load()
harvestingSub = Subscription(harvestingFileset, harvestingWorkflow)
harvestingSub.load()
self.stateMap['Processing Done'].append(harvestingSub)
return
def verifyStateTransitions(self,
transitionMethod='markFinished',
transitionTrigger=True):
"""
_verifyStateTransitions_
Utility method which goes through the list of states in self.orderedStates and
finishes the tasks that demand a state transition in each step. This according
to the defined transition method and trigger.
It verifies that the request document in WMStats is moving according to the transitions
"""
for idx in range(0, len(self.orderedStates) * 2):
nextState = self.orderedStates[idx // 2]
if (idx // 2) == 0:
currentState = 'Closed'
else:
currentState = self.orderedStates[idx // 2 - 1]
if idx % 2 == 0:
for transitionObject in self.stateMap[nextState][:-1]:
method = getattr(transitionObject, transitionMethod)
method(transitionTrigger)
self.plugin([], self.requestDBWriter, self.requestDBWriter)
currentStateWorkflows = self.requestDBWriter.getRequestByStatus(
[currentState])
nextStateWorkflows = self.requestDBWriter.getRequestByStatus(
[nextState])
self.assertEqual(
len(currentStateWorkflows), 1,
'Workflow moved incorrectly from %s' % currentState)
self.assertEqual(
len(nextStateWorkflows), 0,
'Workflow moved incorrectly to %s' % nextState)
else:
transitionObject = self.stateMap[nextState][-1]
method = getattr(transitionObject, transitionMethod)
method(transitionTrigger)
self.plugin([], self.requestDBWriter, self.requestDBWriter)
currentStateWorkflows = self.requestDBWriter.getRequestByStatus(
[currentState])
nextStateWorkflows = self.requestDBWriter.getRequestByStatus(
[nextState])
self.assertEqual(
len(currentStateWorkflows), 0,
'Workflow did not move correctly from %s' % currentState)
self.assertEqual(
len(nextStateWorkflows), 1,
'Workflow did not move correctly to %s' % nextState)
return
def testA_RepackStates(self):
"""
_testA_RepackStates_
Setup an environment with a Repack workflow
and traverse through the different states.
Check that the transitions are sane.
"""
# Set the environment
self.setupRepackWorkflow()
self.plugin = Tier0Plugin()
# Verify the transitions
self.verifyStateTransitions('markOpen', False)
return
def testB_ExpressStates(self):
"""
_testB_ExpressStates_
Setup an environment with a Express workflow
and traverse through the different states.
Check that the transitions are sane.
"""
# Set the environment
self.setupExpressWorkflow()
self.plugin = Tier0Plugin()
# Verify the transitions
self.verifyStateTransitions()
return
def testC_PromptRecoStates(self):
"""
_testC_PromptRecoStates_
Setup an environment with a PromptReco workflow
and traverse through the different states.
Check that the transitions are sane.
"""
# Set the environment
self.setupPromptRecoWorkflow()
self.plugin = Tier0Plugin()
# Verify the transitions
self.verifyStateTransitions()
return
class Tier0FeederPoller(BaseWorkerThread):
def __init__(self, config):
"""
_init_
"""
BaseWorkerThread.__init__(self)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package = "T0.WMBS",
logger = logging,
dbinterface = myThread.dbi)
self.tier0ConfigFile = config.Tier0Feeder.tier0ConfigFile
self.specDirectory = config.Tier0Feeder.specDirectory
self.dropboxuser = getattr(config.Tier0Feeder, "dropboxuser", None)
self.dropboxpass = getattr(config.Tier0Feeder, "dropboxpass", None)
self.dqmUploadProxy = getattr(config.Tier0Feeder, "dqmUploadProxy", None)
self.serviceProxy = getattr(config.Tier0Feeder, "serviceProxy", None)
self.localRequestCouchDB = RequestDBWriter(config.AnalyticsDataCollector.localT0RequestDBURL,
couchapp = config.AnalyticsDataCollector.RequestCouchApp)
self.injectedRuns = set()
hltConfConnectUrl = config.HLTConfDatabase.connectUrl
dbFactoryHltConf = DBFactory(logging, dburl = hltConfConnectUrl, options = {})
self.dbInterfaceHltConf = dbFactoryHltConf.connect()
daoFactoryHltConf = DAOFactory(package = "T0.WMBS",
logger = logging,
dbinterface = self.dbInterfaceHltConf)
self.getHLTConfigDAO = daoFactoryHltConf(classname = "RunConfig.GetHLTConfig")
storageManagerConnectUrl = config.StorageManagerDatabase.connectUrl
dbFactoryStorageManager = DBFactory(logging, dburl = storageManagerConnectUrl, options = {})
self.dbInterfaceStorageManager = dbFactoryStorageManager.connect()
self.dbInterfaceSMNotify = None
if hasattr(config, "SMNotifyDatabase"):
smNotifyConnectUrl = config.SMNotifyDatabase.connectUrl
dbFactorySMNotify = DBFactory(logging, dburl = smNotifyConnectUrl, options = {})
self.dbInterfaceSMNotify = dbFactorySMNotify.connect()
self.getExpressReadyRunsDAO = None
if hasattr(config, "PopConLogDatabase"):
popConLogConnectUrl = getattr(config.PopConLogDatabase, "connectUrl", None)
if popConLogConnectUrl != None:
dbFactoryPopConLog = DBFactory(logging, dburl = popConLogConnectUrl, options = {})
dbInterfacePopConLog = dbFactoryPopConLog.connect()
daoFactoryPopConLog = DAOFactory(package = "T0.WMBS",
logger = logging,
dbinterface = dbInterfacePopConLog)
self.getExpressReadyRunsDAO = daoFactoryPopConLog(classname = "Tier0Feeder.GetExpressReadyRuns")
self.haveT0DataSvc = False
if hasattr(config, "T0DataSvcDatabase"):
t0datasvcConnectUrl = getattr(config.T0DataSvcDatabase, "connectUrl", None)
if t0datasvcConnectUrl != None:
self.haveT0DataSvc = True
dbFactoryT0DataSvc = DBFactory(logging, dburl = t0datasvcConnectUrl, options = {})
dbInterfaceT0DataSvc = dbFactoryT0DataSvc.connect()
self.daoFactoryT0DataSvc = DAOFactory(package = "T0.WMBS",
logger = logging,
dbinterface = dbInterfaceT0DataSvc)
return
@timeFunction
def algorithm(self, parameters = None):
"""
_algorithm_
"""
logging.debug("Running Tier0Feeder algorithm...")
myThread = threading.currentThread()
findNewRunsDAO = self.daoFactory(classname = "Tier0Feeder.FindNewRuns")
findNewRunStreamsDAO = self.daoFactory(classname = "Tier0Feeder.FindNewRunStreams")
findNewExpressRunsDAO = self.daoFactory(classname = "Tier0Feeder.FindNewExpressRuns")
releaseExpressDAO = self.daoFactory(classname = "Tier0Feeder.ReleaseExpress")
feedStreamersDAO = self.daoFactory(classname = "Tier0Feeder.FeedStreamers")
markWorkflowsInjectedDAO = self.daoFactory(classname = "Tier0Feeder.MarkWorkflowsInjected")
tier0Config = None
try:
tier0Config = loadConfigurationFile(self.tier0ConfigFile)
except:
# usually happens when there are syntax errors in the configuration
logging.exception("Cannot load Tier0 configuration file, not configuring new runs and run/streams")
# only configure new runs and run/streams if we have a valid Tier0 configuration
if tier0Config != None:
#
# we don't inject data if the Tier0Config is unreadable
#
# discover new data from StorageManager and inject into Tier0
# (if the config specifies a list of runs do it only once)
#
# replays call data discovery only once (and ignore data status)
#
try:
if tier0Config.Global.InjectRuns == None:
StorageManagerAPI.injectNewData(self.dbInterfaceStorageManager,
self.dbInterfaceHltConf,
self.dbInterfaceSMNotify,
streamerPNN = tier0Config.Global.StreamerPNN,
minRun = tier0Config.Global.InjectMinRun,
maxRun = tier0Config.Global.InjectMaxRun)
else:
injectRuns = set()
for injectRun in tier0Config.Global.InjectRuns:
if injectRun not in self.injectedRuns:
injectRuns.add(injectRun)
for injectRun in injectRuns:
StorageManagerAPI.injectNewData(self.dbInterfaceStorageManager,
self.dbInterfaceHltConf,
self.dbInterfaceSMNotify,
streamerPNN = tier0Config.Global.StreamerPNN,
injectRun = injectRun)
self.injectedRuns.add(injectRun)
except:
# shouldn't happen, just a catch all insurance
logging.exception("Something went wrong with data retrieval from StorageManager")
#
# find new runs, setup global run settings and stream/dataset/trigger mapping
#
runHltkeys = findNewRunsDAO.execute(transaction = False)
for run, hltkey in sorted(runHltkeys.items()):
hltConfig = None
# local runs have no hltkey and are configured differently
if hltkey != None:
# retrieve HLT configuration and make sure it's usable
try:
hltConfig = self.getHLTConfigDAO.execute(hltkey, transaction = False)
if hltConfig['process'] == None or len(hltConfig['mapping']) == 0:
raise RuntimeError("HLTConfDB query returned no process or mapping")
except:
logging.exception("Can't retrieve hltkey %s for run %d" % (hltkey, run))
continue
try:
RunConfigAPI.configureRun(tier0Config, run, hltConfig)
except:
logging.exception("Can't configure for run %d" % (run))
#
# find unconfigured run/stream with data
# populate RunConfig, setup workflows/filesets/subscriptions
#
runStreams = findNewRunStreamsDAO.execute(transaction = False)
for run in sorted(runStreams.keys()):
for stream in sorted(runStreams[run]):
try:
RunConfigAPI.configureRunStream(tier0Config,
run, stream,
self.specDirectory,
self.dqmUploadProxy)
except:
logging.exception("Can't configure for run %d and stream %s" % (run, stream))
#
# stop and close runs based on RunSummary and StorageManager records
#
RunLumiCloseoutAPI.stopRuns(self.dbInterfaceStorageManager)
RunLumiCloseoutAPI.closeRuns(self.dbInterfaceStorageManager)
#
# release runs for Express
#
runs = findNewExpressRunsDAO.execute(transaction = False)
if len(runs) > 0:
binds = []
for run in runs:
binds.append( { 'RUN' : run } )
if self.getExpressReadyRunsDAO != None:
runs = self.getExpressReadyRunsDAO.execute(binds = binds, transaction = False)
if len(runs) > 0:
binds = []
for run in runs:
binds.append( { 'RUN' : run } )
releaseExpressDAO.execute(binds = binds, transaction = False)
#
# release runs for PromptReco
# check PromptRecoStatus first, i.e. big red button
#
if self.getPromptRecoStatusT0DataSvc():
RunConfigAPI.releasePromptReco(tier0Config,
self.specDirectory,
self.dqmUploadProxy)
#
# insert express and reco configs into Tier0 Data Service
#
if self.haveT0DataSvc:
self.updateRunConfigT0DataSvc()
self.updateRunStreamDoneT0DataSvc()
self.updateExpressConfigsT0DataSvc()
self.updateRecoConfigsT0DataSvc()
self.updateRecoReleaseConfigsT0DataSvc()
self.lockDatasetsT0DataSvc()
#
# mark express and repack workflows as injected if certain conditions are met
# (we don't do it immediately to prevent the TaskArchiver from cleaning up too early)
#
markWorkflowsInjectedDAO.execute(self.dbInterfaceSMNotify != None,
transaction = False)
#
# close stream/lumis for run/streams that are active (fileset exists and open)
#
RunLumiCloseoutAPI.closeLumiSections(self.dbInterfaceStorageManager)
#
# feed new data into exisiting filesets
#
try:
myThread.transaction.begin()
feedStreamersDAO.execute(conn = myThread.transaction.conn, transaction = True)
except:
logging.exception("Can't feed data, bailing out...")
raise
else:
myThread.transaction.commit()
#
# run ended and run/stream fileset open
# => check for complete lumi_closed record, all lumis finally closed and all data feed
# => if all conditions satisfied, close the run/stream fileset
#
RunLumiCloseoutAPI.closeRunStreamFilesets()
#
# check and delete active split lumis
#
RunLumiCloseoutAPI.checkActiveSplitLumis()
#
# insert workflows into CouchDB for monitoring
#
self.feedCouchMonitoring()
#
# Update Couch when Repack and Express have closed input filesets (analog to old T0 closeout)
#
self.closeOutRealTimeWorkflows()
#
# send repacked notifications to StorageManager
#
if self.dbInterfaceSMNotify:
StorageManagerAPI.markRepacked(self.dbInterfaceSMNotify)
#
# upload PCL conditions to DropBox
#
ConditionUploadAPI.uploadConditions(self.dropboxuser, self.dropboxpass, self.serviceProxy)
return
def feedCouchMonitoring(self):
"""
_feedCouchMonitoring_
check for workflows that haven't been uploaded to Couch for monitoring yet
"""
getStreamerWorkflowsForMonitoringDAO = self.daoFactory(classname = "Tier0Feeder.GetStreamerWorkflowsForMonitoring")
getPromptRecoWorkflowsForMonitoringDAO = self.daoFactory(classname = "Tier0Feeder.GetPromptRecoWorkflowsForMonitoring")
markTrackedWorkflowMonitoringDAO = self.daoFactory(classname = "Tier0Feeder.MarkTrackedWorkflowMonitoring")
workflows = getStreamerWorkflowsForMonitoringDAO.execute()
workflows += getPromptRecoWorkflowsForMonitoringDAO.execute()
if len(workflows) == 0:
logging.debug("No workflows to publish to couch monitoring, doing nothing")
if workflows:
logging.debug(" Going to publish %d workflows" % len(workflows))
for (workflowId, run, workflowName) in workflows:
logging.info(" Publishing workflow %s to monitoring" % workflowName)
#TODO: add more information about workflow if there need to be kept longer than
# workflow life cycle.
doc = {}
doc["RequestName"] = workflowName
doc["Run"] = run
response = self.localRequestCouchDB.insertGenericRequest(doc)
if response == "OK" or "EXISTS":
logging.info(" Successfully uploaded request %s" % workflowName)
# Here we have to trust the insert, if it doesn't happen will be easy to spot on the logs
markTrackedWorkflowMonitoringDAO.execute(workflowId)
return
def closeOutRealTimeWorkflows(self):
"""
_closeOutRealTimeWorkflows_
Updates couch with the closeout status of Repack and Express
PromptReco should be closed out automatically
"""
getNotClosedOutWorkflowsDAO = self.daoFactory(classname = "Tier0Feeder.GetNotClosedOutWorkflows")
workflows = getNotClosedOutWorkflowsDAO.execute()
if len(workflows) == 0:
logging.debug("No workflows to publish to couch monitoring, doing nothing")
if workflows:
for workflow in workflows:
(workflowId, filesetId, filesetOpen, workflowName) = workflow
# find returns -1 if the string is not found
if workflowName.find('PromptReco') >= 0:
logging.debug("Closing out instantaneously PromptReco Workflow %s" % workflowName)
self.updateClosedState(workflowName, workflowId)
else :
# Check if fileset (which you already know) is closed or not
# FIXME: No better way to do it? what comes from the DAO is a string, casting bool or int doesn't help much.
# Works like that :
if filesetOpen == '0':
self.updateClosedState(workflowName, workflowId)
return
def updateClosedState(self, workflowName, workflowId):
"""
_updateClosedState_
Mark a workflow as Closed
"""
markCloseoutWorkflowMonitoringDAO = self.daoFactory(classname = "Tier0Feeder.MarkCloseoutWorkflowMonitoring")
response = self.localRequestCouchDB.updateRequestStatus(workflowName, 'Closed')
if response == "OK" or "EXISTS":
logging.debug("Successfully closed workflow %s" % workflowName)
markCloseoutWorkflowMonitoringDAO.execute(workflowId)
return
def getPromptRecoStatusT0DataSvc(self):
"""
_getPromptRecoStatusDataSvc_
Check the PromptRecoStatus (enabled/disabled) set by the ORM
"""
getPromptRecoStatusDAO = self.daoFactoryT0DataSvc(classname = "T0DataSvc.GetPromptRecoStatus")
status = getPromptRecoStatusDAO.execute(transaction = False)
return status
def updateRunConfigT0DataSvc(self):
"""
_updateRunConfigT0DataSvc_
Check for new runs and push their info into the Tier0 Data Service.
"""
getNewRunDAO = self.daoFactory(classname = "T0DataSvc.GetNewRun")
newRun = getNewRunDAO.execute(transaction = False)
if len(newRun) > 0:
binds = []
for runInfo in newRun:
binds.append( { 'RUN' : runInfo['run'],
'ACQ_ERA' : runInfo['acq_era'] } )
insertNewRunDAO = self.daoFactoryT0DataSvc(classname = "T0DataSvc.InsertNewRun")
insertNewRunDAO.execute(binds = binds, transaction = False)
for bind in binds:
del bind['ACQ_ERA']
updateNewRunDAO = self.daoFactory(classname = "T0DataSvc.UpdateNewRun")
updateNewRunDAO.execute(binds = binds, transaction = False)
return
def updateRunStreamDoneT0DataSvc(self):
"""
_updateRunStreamDoneT0DataSvc_
Check if a run/stream workflow (express or repack) is finished and
cleaned up and push a completion record into the Tier0 Data Service.
"""
getRunStreamDoneDAO = self.daoFactory(classname = "T0DataSvc.GetRunStreamDone")
runStreamDone = getRunStreamDoneDAO.execute(transaction = False)
if len(runStreamDone) > 0:
binds = []
for runStream in runStreamDone:
binds.append( { 'RUN' : runStream['run'],
'STREAM' : runStream['stream'] } )
insertRunStreamDoneDAO = self.daoFactoryT0DataSvc(classname = "T0DataSvc.InsertRunStreamDone")
insertRunStreamDoneDAO.execute(binds = binds, transaction = False)
updateRunStreamDoneDAO = self.daoFactory(classname = "T0DataSvc.UpdateRunStreamDone")
updateRunStreamDoneDAO.execute(binds = binds, transaction = False)
return
def updateExpressConfigsT0DataSvc(self):
"""
_updateExpressConfigsT0DataSvc_
Check which express_config rows are missing
in the Tier0 Data Service and insert them,
also record that fact in t0ast
"""
getExpressConfigsDAO = self.daoFactory(classname = "T0DataSvc.GetExpressConfigs")
expressConfigs = getExpressConfigsDAO.execute(transaction = False)
if len(expressConfigs) > 0:
bindsInsert = []
bindsUpdate = []
for config in expressConfigs:
bindsInsert.append( { 'RUN' : config['run'],
'STREAM' : config['stream'],
'CMSSW' : config['cmssw'],
'SCRAM_ARCH' : config['scram_arch'],
'RECO_CMSSW' : config['reco_cmssw'],
'RECO_SCRAM_ARCH' : config['reco_scram_arch'],
'ALCA_SKIM' : config['alca_skim'],
'DQM_SEQ' : config['dqm_seq'],
'GLOBAL_TAG' : config['global_tag'][:50],
'SCENARIO' : config['scenario'],
'MULTICORE' : config['multicore'],
'WRITE_TIERS' : config['write_tiers'],
'WRITE_DQM' : config['write_dqm'] } )
bindsUpdate.append( { 'RUN' : config['run'],
'STREAM' : config['stream'] } )
insertExpressConfigsDAO = self.daoFactoryT0DataSvc(classname = "T0DataSvc.InsertExpressConfigs")
insertExpressConfigsDAO.execute(binds = bindsInsert, transaction = False)
updateExpressConfigsDAO = self.daoFactory(classname = "T0DataSvc.UpdateExpressConfigs")
updateExpressConfigsDAO.execute(binds = bindsUpdate, transaction = False)
return
def updateRecoConfigsT0DataSvc(self):
"""
_updateRecoConfigsT0DataSvc_
Check which reco_config rows are missing
in the Tier0 Data Service and insert them,
also record that fact in t0ast
"""
getRecoConfigsDAO = self.daoFactory(classname = "T0DataSvc.GetRecoConfigs")
recoConfigs = getRecoConfigsDAO.execute(transaction = False)
if len(recoConfigs) > 0:
bindsInsert = []
bindsUpdate = []
for config in recoConfigs:
bindsInsert.append( { 'RUN' : config['run'],
'PRIMDS' : config['primds'],
'CMSSW' : config['cmssw'],
'SCRAM_ARCH' : config['scram_arch'],
'ALCA_SKIM' : config['alca_skim'],
'PHYSICS_SKIM' : config['physics_skim'],
'DQM_SEQ' : config['dqm_seq'],
'GLOBAL_TAG' : config['global_tag'][:50],
'SCENARIO' : config['scenario'],
'MULTICORE' : config['multicore'],
'WRITE_RECO' : config['write_reco'],
'WRITE_DQM' : config['write_dqm'],
'WRITE_AOD' : config['write_aod'],
'WRITE_MINIAOD' : config['write_miniaod'] } )
bindsUpdate.append( { 'RUN' : config['run'],
'PRIMDS' : config['primds'] } )
insertRecoConfigsDAO = self.daoFactoryT0DataSvc(classname = "T0DataSvc.InsertRecoConfigs")
insertRecoConfigsDAO.execute(binds = bindsInsert, transaction = False)
updateRecoConfigsDAO = self.daoFactory(classname = "T0DataSvc.UpdateRecoConfigs")
updateRecoConfigsDAO.execute(binds = bindsUpdate, transaction = False)
return
def updateRecoReleaseConfigsT0DataSvc(self):
"""
_updateRecoReleaseConfigsT0DataSvc_
Insert information about PromptReco release into the Tier0 Data Service.
That means updating the reco_locked records in run granularity (where one
released dataset means the whole run is locked).
Also insert and update the run_primds_done records to track PromptReco status.
"""
getRunDatasetNewDAO = self.daoFactory(classname = "Tier0Feeder.GetRunDatasetNew")
getRunDatasetReleasedDAO = self.daoFactory(classname = "Tier0Feeder.GetRunDatasetReleased")
getRunDatasetDoneDAO = self.daoFactory(classname = "Tier0Feeder.GetRunDatasetDone")
updateRecoReleaseConfigsDAO = self.daoFactory(classname = "Tier0Feeder.UpdateRecoReleaseConfigs")
insertRecoLockedDAO = self.daoFactoryT0DataSvc(classname = "T0DataSvc.InsertRecoLocked")
updateRecoLockedDAO = self.daoFactoryT0DataSvc(classname = "T0DataSvc.UpdateRecoLocked")
insertRunDatasetDoneDAO = self.daoFactoryT0DataSvc(classname = "T0DataSvc.InsertRunDatasetDone")
updateRunDatasetDoneDAO = self.daoFactoryT0DataSvc(classname = "T0DataSvc.UpdateRunDatasetDone")
# first check for records that are completely new
# insert the two Tier0 Data Service records for them
# update reco release records accordingly
runDatasetNew = getRunDatasetNewDAO.execute(transaction = False)
foundRuns = set()
bindsInsertLocked = []
bindsInsertDone = []
bindsUpdate = []
for runDataset in runDatasetNew:
run = runDataset['run']
if run not in foundRuns:
bindsInsertLocked.append( { 'RUN': run } )
foundRuns.add(run)
bindsInsertDone.append( { 'RUN': run,
'PRIMDS': runDataset['primds'] } )
bindsUpdate.append( { 'RUN' : run,
'PRIMDS_ID': runDataset['primds_id'],
'IN_DATASVC' : 1 } )
if len(bindsInsertLocked) > 0:
insertRecoLockedDAO.execute(binds = bindsInsertLocked, transaction = False)
if len(bindsInsertDone) > 0:
insertRunDatasetDoneDAO.execute(binds = bindsInsertDone, transaction = False)
if len(bindsUpdate) > 0:
updateRecoReleaseConfigsDAO.execute(binds = bindsUpdate, transaction = False)
# then check for reco release and lock runs in the Tier0 Data Service
runDatasetReleased = getRunDatasetReleasedDAO.execute(transaction = False)
foundRuns = set()
bindsUpdateLocked = []
bindsUpdate = []
for runDataset in runDatasetReleased:
run = runDataset['run']
if run not in foundRuns:
bindsUpdateLocked.append( { 'RUN': run } )
foundRuns.add(run)
bindsUpdate.append( { 'RUN' : run,
'PRIMDS_ID': runDataset['primds_id'],
'IN_DATASVC' : 2 } )
if len(bindsUpdateLocked) > 0:
updateRecoLockedDAO.execute(binds = bindsUpdateLocked, transaction = False)
if len(bindsUpdate) > 0:
updateRecoReleaseConfigsDAO.execute(binds = bindsUpdate, transaction = False)
# finally check for reco completions and mark this in the Tier0 Data Service
runDatasetDone = getRunDatasetDoneDAO.execute(transaction = False)
bindsUpdateDone = []
bindsUpdate = []
for runDataset in runDatasetDone:
run = runDataset['run']
bindsUpdateDone.append( { 'RUN' : run,
'PRIMDS' : runDataset['primds'] } )
bindsUpdate.append( { 'RUN' : run,
'PRIMDS_ID': runDataset['primds_id'],
'IN_DATASVC' : 3 } )
if len(bindsUpdateDone) > 0:
updateRunDatasetDoneDAO.execute(binds = bindsUpdateDone, transaction = False)
if len(bindsUpdate) > 0:
updateRecoReleaseConfigsDAO.execute(binds = bindsUpdate, transaction = False)
return
def lockDatasetsT0DataSvc(self):
"""
_lockDatasetsT0DataSvc_
Publish dataset information into the Tier0 Data Service.
"""
getDatasetLockedDAO = self.daoFactory(classname = "T0DataSvc.GetDatasetLocked")
datasetConfigs = getDatasetLockedDAO.execute(transaction = False)
if len(datasetConfigs) > 0:
bindsInsert = []
bindsUpdate = []
for config in datasetConfigs:
bindsInsert.append( { 'PATH' : config['path'] } )
bindsUpdate.append( { 'ID' : config['id'] } )
insertDatasetLockedDAO = self.daoFactoryT0DataSvc(classname = "T0DataSvc.InsertDatasetLocked")
insertDatasetLockedDAO.execute(binds = bindsInsert, transaction = False)
updateDatasetLockedDAO = self.daoFactory(classname = "T0DataSvc.UpdateDatasetLocked")
updateDatasetLockedDAO.execute(binds = bindsUpdate, transaction = False)
return
def terminate(self, params):
"""
_terminate_
Kill the code after one final pass when called by the master thread.
"""
logging.debug("terminating immediately")
class RequestDBTest(unittest.TestCase):
"""
"""
def setUp(self):
"""
_setUp_
"""
self.schema = []
self.couchApps = ["ReqMgr"]
self.testInit = TestInitCouchApp('RequestDBServiceTest')
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = self.schema,
useDefault = False)
dbName = 'requsetdb_t'
self.testInit.setupCouch(dbName, *self.couchApps)
reqDBURL = "%s/%s" % (self.testInit.couchUrl, dbName)
self.requestWriter = RequestDBWriter(reqDBURL)
self.requestReader = RequestDBReader(reqDBURL)
self.requestWriter.defaultStale = {}
self.requestReader.defaultStale = {}
return
def tearDown(self):
"""
_tearDown_
Drop all the WMBS tables.
"""
self.testInit.tearDownCouch()
def testRequestDBWriter(self):
# test getWork
schema = generate_reqmgr_schema(3)
result = self.requestWriter.insertGenericRequest(schema[0])
self.assertEqual(len(result), 1, 'insert fail');
self.assertEqual(self.requestWriter.updateRequestStatus(schema[0]['RequestName'], "failed"), 'OK', 'update fail')
self.assertEqual(self.requestWriter.updateRequestStatus("not_exist_schema", "assigned"),
'Error: document not found')
result = self.requestWriter.updateRequestProperty(schema[0]['RequestName'],
{'Teams': ['teamA']})
self.assertEqual(self.requestWriter.updateRequestProperty(schema[0]['RequestName'],
{'Teams': ['teamA']}), 'OK', 'update fail')
self.assertEqual(self.requestWriter.updateRequestProperty("not_exist_schema", {'Teams': 'teamA'}),
'Error: document not found')
result = self.requestReader.getRequestByNames([schema[0]['RequestName']])
self.assertEqual(len(result), 1, "should be 1")
result = self.requestReader.getRequestByStatus(["failed"], False, 1)
self.assertEqual(len(result), 1, "should be 1")
result = self.requestReader.getStatusAndTypeByRequest([schema[0]['RequestName']])
self.assertEqual(result[schema[0]['RequestName']][0], 'failed', "should be failed")
result = self.requestWriter.insertGenericRequest(schema[1])
time.sleep(2)
result = self.requestWriter.insertGenericRequest(schema[2])
endTime = int(time.time()) - 1
result = self.requestReader.getRequestByStatusAndEndTime("new", False, endTime)
self.assertEqual(len(result), 1, "should be 1")
endTime = int(time.time()) + 1
result = self.requestReader.getRequestByStatusAndEndTime("new", False, endTime)
self.assertEqual(len(result), 2, "should be 2")
