def getSplitting(requestName):
reqmgrUrl='https://cmsweb.cern.ch/reqmgr/reqMgr/'
reqmgr = RequestManager(dict = {'endpoint' : reqmgrUrl})
result = reqmgr.getRequest(requestName)
workloadDB = Database(result['CouchWorkloadDBName'], result['CouchURL'])
workloadPickle = workloadDB.getAttachment(requestName, 'spec')
spec = pickle.loads(workloadPickle)
workload = WMWorkloadHelper(spec)
params = workload.getTopLevelTask()[0].jobSplittingParameters()
algo = workload.getTopLevelTask()[0].jobSplittingAlgorithm()
return params['events_per_job']
def main():
demPolicy = Block()
reqmgr = RequestManager(dict = {'endpoint' : 'https://cmsweb.cern.ch/reqmgr/reqMgr'})
result = reqmgr.getRequest('pdmvserv_HIG-Summer12DR53X-01392_T1_ES_PIC_MSS_1_v0__130724_063344_7207')
workloadDB = Database(result['CouchWorkloadDBName'], result['CouchURL'])
workloadPickle = workloadDB.getAttachment('pdmvserv_HIG-Summer12DR53X-01392_T1_ES_PIC_MSS_1_v0__130724_063344_7207', 'spec')
spec = pickle.loads(workloadPickle)
workload = WMWorkloadHelper(spec)
x,y = demPolicy(wmspec = workload, task = workload.getTopLevelTask()[0])
print x
print y
def getSplitting(requestName):
reqmgrUrl = 'https://cmsweb.cern.ch/reqmgr/reqMgr/'
reqmgr = RequestManager(dict={'endpoint': reqmgrUrl})
result = reqmgr.getRequest(requestName)
workloadDB = Database(result['CouchWorkloadDBName'], result['CouchURL'])
workloadPickle = workloadDB.getAttachment(requestName, 'spec')
spec = pickle.loads(workloadPickle)
workload = WMWorkloadHelper(spec)
params = workload.getTopLevelTask()[0].jobSplittingParameters()
algo = workload.getTopLevelTask()[0].jobSplittingAlgorithm()
return params['events_per_job']
def getRequestInformationAndWorkload(requestName, reqmgrUrl):
"""
_getRequestInformationAndWorkload_
Retrieve the request information for assignment
and the full pickled workload.
"""
reqmgr = RequestManager(dict = {'endpoint' : reqmgrUrl})
result = reqmgr.getRequest(requestName)
workloadDB = Database(result['CouchWorkloadDBName'], result['CouchURL'])
workloadPickle = workloadDB.getAttachment(requestName, 'spec')
spec = pickle.loads(workloadPickle)
workload = WMWorkloadHelper(spec)
return workload, result
def getRequestInformationAndWorkload(requestName, reqmgrUrl):
"""
_getRequestInformationAndWorkload_
Retrieve the request information for assignment
and the full pickled workload.
"""
reqmgr = RequestManager(dict={'endpoint': reqmgrUrl})
result = reqmgr.getRequest(requestName)
workloadDB = Database(result['CouchWorkloadDBName'], result['CouchURL'])
workloadPickle = workloadDB.getAttachment(requestName, 'spec')
spec = pickle.loads(workloadPickle)
workload = WMWorkloadHelper(spec)
return workload, result
class RequestManagerTest(RESTBaseUnitTest):
"""
Test RequestMgr Service client
It will start RequestMgr RESTService
Server DB is whatever env is set
This checks whether DS call makes without error and return the results.
This test only test service call returns without error.
The correctness of each function is tested in test/python/RequestManager_t/RequestMgr_t.py
"""
def initialize(self):
self.couchDBName = "reqmgr_t_0"
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
dbUrl = os.environ.get("DATABASE", None)
self.config.setDBUrl(dbUrl)
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName = self.couchDBName)
self.config.setPort(8899)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
def setUp(self):
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache", "ReqMgr")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
# logging stuff from TestInit is broken, setting myself
l = logging.getLogger()
l.setLevel(logging.DEBUG)
self.params = {}
self.params['endpoint'] = self.config.getServerUrl()
self.reqService = RequestManagerDS(self.params)
self.jsonSender = JSONRequests(self.config.getServerUrl())
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
schema['ConfigCacheID'] = self.createConfig()
schema['CouchDBName'] = self.couchDBName
schema['CouchWorkloadDBName'] = self.couchDBName
try:
r = self.jsonSender.put('request', schema)
try:
self.requestName = r[0]['RequestName']
except:
self.requestName = r[0].values()[0]['RequestName']
except Exception as ex:
msg = traceback.format_exc()
print("Exception during set up, reason: %s" % msg)
raise ex
def tearDown(self):
self.config.deleteWorkloadCache()
RESTBaseUnitTest.tearDown(self)
self.testInit.tearDownCouch()
def createConfig(self, bad = False):
"""
_createConfig_
Create a config of some sort that we can load out of ConfigCache
"""
PSetTweak = {'process': {'outputModules_': ['ThisIsAName'],
'ThisIsAName': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}}}}
BadTweak = {'process': {'outputModules_': ['ThisIsAName1', 'ThisIsAName2'],
'ThisIsAName1': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}},
'ThisIsAName2': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}}}}
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = self.couchDBName)
configCache.createUserGroup(groupname = "testGroup", username = '******')
if bad:
configCache.setPSetTweaks(PSetTweak = BadTweak)
else:
configCache.setPSetTweaks(PSetTweak = PSetTweak)
configCache.save()
return configCache.getCouchID()
@attr("integration")
def testA_RequestManagerService(self):
requestName = self.requestName
request = self.reqService.getRequest(requestName)
# minimal test : it's return type and the some value inside
self.assertEqual(type(request), dict)
self.assertTrue(len(request) > 0)
# Test putTeam
self.reqService.putTeam("team_usa")
self.assertTrue('team_usa' in self.jsonSender.get('team')[0])
self.jsonSender.put('assignment/%s/%s' % ("team_usa", requestName))
request = self.reqService.getAssignment(teamName = "team_usa")
self.assertEqual(type(request), list)
self.assertTrue(len(request) > 0)
request = self.reqService.getAssignment(request = requestName)
self.assertEqual(type(request), list)
self.assertTrue(len(request) > 0)
self.reqService.sendMessage(requestName,"error")
self.reqService.putWorkQueue(requestName, "http://test_url")
self.reqService.reportRequestProgress(requestName,
percent_complete = 100, percent_success = 90)
self.reqService.updateRequestStatus(requestName, "running-open")
class RequestManagerTest(RESTBaseUnitTest):
"""
Test RequestMgr Service client
It will start RequestMgr RESTService
Server DB is whatever env is set
This checks whether DS call makes without error and return the results.
This test only test service call returns without error.
The correctness of each function is tested in test/python/RequestManager_t/RequestMgr_t.py
"""
def initialize(self):
self.couchDBName = "reqmgr_t_0"
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
dbUrl = os.environ.get("DATABASE", None)
self.config.setDBUrl(dbUrl)
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName=self.couchDBName)
self.config.setPort(8899)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
def setUp(self):
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser",
"ConfigCache", "ReqMgr")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
# logging stuff from TestInit is broken, setting myself
l = logging.getLogger()
l.setLevel(logging.DEBUG)
self.params = {}
self.params['endpoint'] = self.config.getServerUrl()
self.reqService = RequestManagerDS(self.params)
self.jsonSender = JSONRequests(self.config.getServerUrl())
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
schema['ConfigCacheID'] = self.createConfig()
schema['CouchDBName'] = self.couchDBName
schema['CouchWorkloadDBName'] = self.couchDBName
try:
r = self.jsonSender.put('request', schema)
try:
self.requestName = r[0]['RequestName']
except:
self.requestName = r[0].values()[0]['RequestName']
except Exception as ex:
msg = traceback.format_exc()
print "Exception during set up, reason: %s" % msg
raise ex
def tearDown(self):
self.config.deleteWorkloadCache()
RESTBaseUnitTest.tearDown(self)
self.testInit.tearDownCouch()
def createConfig(self, bad=False):
"""
_createConfig_
Create a config of some sort that we can load out of ConfigCache
"""
PSetTweak = {
'process': {
'outputModules_': ['ThisIsAName'],
'ThisIsAName': {
'dataset': {
'dataTier': 'RECO',
'filterName': 'Filter'
}
}
}
}
BadTweak = {
'process': {
'outputModules_': ['ThisIsAName1', 'ThisIsAName2'],
'ThisIsAName1': {
'dataset': {
'dataTier': 'RECO',
'filterName': 'Filter'
}
},
'ThisIsAName2': {
'dataset': {
'dataTier': 'RECO',
'filterName': 'Filter'
}
}
}
}
configCache = ConfigCache(os.environ["COUCHURL"],
couchDBName=self.couchDBName)
configCache.createUserGroup(groupname="testGroup", username='******')
if bad:
configCache.setPSetTweaks(PSetTweak=BadTweak)
else:
configCache.setPSetTweaks(PSetTweak=PSetTweak)
configCache.save()
return configCache.getCouchID()
@attr("integration")
def testA_RequestManagerService(self):
requestName = self.requestName
request = self.reqService.getRequest(requestName)
# minimal test : it's return type and the some value inside
self.assertEqual(type(request), dict)
self.assertTrue(len(request) > 0)
# Test putTeam
self.reqService.putTeam("team_usa")
self.assertTrue('team_usa' in self.jsonSender.get('team')[0])
self.jsonSender.put('assignment/%s/%s' % ("team_usa", requestName))
request = self.reqService.getAssignment(teamName="team_usa")
self.assertEqual(type(request), list)
self.assertTrue(len(request) > 0)
request = self.reqService.getAssignment(request=requestName)
self.assertEqual(type(request), list)
self.assertTrue(len(request) > 0)
self.reqService.sendMessage(requestName, "error")
self.reqService.putWorkQueue(requestName, "http://test_url")
self.reqService.reportRequestProgress(requestName,
percent_complete=100,
percent_success=90)
self.reqService.updateRequestStatus(requestName, "running-open")
class JobUpdaterPoller(BaseWorkerThread):
"""
_JobUpdaterPoller_
Poller class for the JobUpdater
"""
def __init__(self, config):
"""
__init__
"""
BaseWorkerThread.__init__(self)
self.config = config
self.bossAir = BossAirAPI(config = self.config)
self.reqmgr = RequestManager({'endpoint' : self.config.JobUpdater.reqMgrUrl})
self.workqueue = WorkQueue(self.config.WorkQueueManager.couchurl,
self.config.WorkQueueManager.dbname)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.listWorkflowsDAO = self.daoFactory(classname = "Workflow.ListForJobUpdater")
self.updateWorkflowPrioDAO = self.daoFactory(classname = "Workflow.UpdatePriority")
self.executingJobsDAO = self.daoFactory(classname = "Jobs.GetNumberOfJobsForWorkflowTaskStatus")
def setup(self, parameters = None):
"""
_setup_
"""
pass
def terminate(self, parameters = None):
"""
_terminate_
Terminate gracefully.
"""
pass
def algorithm(self, parameters = None):
"""
_algorithm_
"""
logging.info("Synchronizing priorities with ReqMgr...")
self.synchronizeJobPriority()
def synchronizeJobPriority(self):
"""
_synchronizeJobPriority_
Check WMBS and WorkQueue for active workflows and compare with the
ReqMgr for priority changes. If a priority change occurs
then update the job priority in the batch system and
the elements in the local queue that have not been injected yet.
"""
# Update the priority of workflows that are not in WMBS and just in local queue
priorityCache = {}
workflowsToUpdate = {}
workflowsToCheck = [x for x in self.workqueue.getAvailableWorkflows()]
for workflow, priority in workflowsToCheck:
if workflow not in priorityCache:
try:
priorityCache[workflow] = self.reqmgr.getRequest(workflow)['RequestPriority']
except Exception, ex:
logging.error("Couldn't retrieve the priority of request %s" % workflow)
logging.error("Error: %s" % ex)
continue
if priority != priorityCache[workflow]:
workflowsToUpdate[workflow] = priorityCache[workflow]
for workflow in workflowsToUpdate:
self.workqueue.updatePriority(workflow, workflowsToUpdate[workflow])
# Check the workflows in WMBS
priorityCache = {}
workflowsToUpdateWMBS = {}
workflowsToCheck = self.listWorkflowsDAO.execute()
for workflowEntry in workflowsToCheck:
workflow = workflowEntry['name']
if workflow not in priorityCache:
try:
priorityCache[workflow] = self.reqmgr.getRequest(workflow)['RequestPriority']
except Exception, ex:
logging.error("Couldn't retrieve the priority of request %s" % workflow)
logging.error("Error: %s" % ex)
continue
requestPriority = priorityCache[workflow]
if requestPriority != workflowEntry['workflow_priority']:
# Update the workqueue priority for the Available elements
self.workqueue.updatePriority(workflow, priorityCache[workflow])
# Check if there are executing jobs for this particular task
if self.executingJobsDAO.execute(workflow, workflowEntry['task']) > 0:
self.bossAir.updateJobInformation(workflow, workflowEntry['task'],
requestPriority = priorityCache[workflow],
taskPriority = workflowEntry['task_priority'])
workflowsToUpdateWMBS[workflow] = priorityCache[workflow]
class RequestManagerTest(RESTBaseUnitTest):
"""
Test RequestMgr Service client
It will start RequestMgr RESTService
Server DB is whatever env is set
This checks whether DS call makes without error and return the results.
This test only test service call returns without error.
The correctness of each function is tested in test/python/RequestManager_t/RequestMgr_t.py
"""
def initialize(self):
self.couchDBName = "reqmgr_t_0"
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
dbUrl = os.environ.get("DATABASE", None)
self.config.setDBUrl(dbUrl)
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName = self.couchDBName)
self.config.setPort(8888)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
return
def setUp(self):
"""
setUP global values
"""
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName,
"GroupUser", "ConfigCache")
self.params = {}
self.params['endpoint'] = self.config.getServerUrl()
self.reqService = RequestManagerDS(self.params)
self.jsonSender = JSONRequests(self.config.getServerUrl())
self.requestSchema = getRequestSchema()
self.jsonSender.put('group/PeopleLikeMe')
self.jsonSender.put('user/[email protected]')
self.jsonSender.put('group/PeopleLikeMe/me')
self.jsonSender.put('version/CMSSW_3_5_8')
r = self.jsonSender.put('request/' + self.requestSchema['RequestName'],
self.requestSchema)
self.requestName = r[0]['RequestName']
def tearDown(self):
self.config.deleteWorkloadCache()
RESTBaseUnitTest.tearDown(self)
self.testInit.tearDownCouch()
@attr("integration")
def testA_RequestManagerService(self):
requestName = self.requestName
request = self.reqService.getRequest(requestName)
# minimal test : it's return type and the some value inside
self.assertEqual(type(request), dict)
self.assertTrue(len(request) > 0)
# Test putTeam
self.reqService.putTeam("team_usa")
self.assertTrue('team_usa' in self.jsonSender.get('team')[0])
self.jsonSender.put('assignment/%s/%s' % ("team_usa", requestName))
request = self.reqService.getAssignment(teamName = "team_usa")
self.assertEqual(type(request), list)
self.assertTrue(len(request) > 0)
request = self.reqService.getAssignment(request = requestName)
self.assertEqual(type(request), list)
self.assertTrue(len(request) > 0)
self.reqService.sendMessage(requestName,"error")
self.reqService.putWorkQueue(requestName, "http://test_url")
self.reqService.reportRequestProgress(requestName)
self.reqService.reportRequestProgress(requestName,
percent_complete = 100, percent_success = 90)
self.reqService.reportRequestStatus(requestName, "running")
return
