def main():
"""
_main_
"""
if 'WMAGENT_CONFIG' not in os.environ:
os.environ['WMAGENT_CONFIG'] = '/data/srv/wmagent/current/config/wmagent/config.py'
config = loadConfigurationFile(os.environ["WMAGENT_CONFIG"])
# Instantiating central reqmgr and local workqueue
print "ReqMgr2 URL : %s" % sanitizeURL(config.JobUpdater.reqMgr2Url)['url']
print "WorkQueue URL: %s and dbname %s" % (sanitizeURL(config.WorkQueueManager.couchurl)['url'],
config.WorkQueueManager.dbname)
reqmgr2 = ReqMgr(config.JobUpdater.reqMgr2Url)
workqueue = WorkQueue(config.WorkQueueManager.couchurl, config.WorkQueueManager.dbname)
print "\nFirst attempt to update prio of wfs that are not in WMBS and only in local queue"
priorityCache = {}
workflowsToUpdate = {}
workflowsToCheck = [x for x in workqueue.getAvailableWorkflows()]
print "Retrieved %d workflows from workqueue" % len(workflowsToCheck)
for workflow, priority in workflowsToCheck:
if workflow not in priorityCache:
try:
priorityCache[workflow] = reqmgr2.getRequestByNames(workflow)[workflow]['RequestPriority']
except Exception, ex:
print "Couldn't retrieve the priority of request %s" % workflow
print "Error: %s" % ex
continue
if priority != priorityCache[workflow]:
workflowsToUpdate[workflow] = priorityCache[workflow]
def main():
"""
_main_
"""
if 'WMAGENT_CONFIG' not in os.environ:
os.environ[
'WMAGENT_CONFIG'] = '/data/srv/wmagent/current/config/wmagent/config.py'
config = loadConfigurationFile(os.environ["WMAGENT_CONFIG"])
# Instantiating central reqmgr and local workqueue
print "ReqMgr2 URL : %s" % sanitizeURL(
config.JobUpdater.reqMgr2Url)['url']
print "WorkQueue URL: %s and dbname %s" % (sanitizeURL(
config.WorkQueueManager.couchurl)['url'],
config.WorkQueueManager.dbname)
reqmgr2 = ReqMgr(config.JobUpdater.reqMgr2Url)
workqueue = WorkQueue(config.WorkQueueManager.couchurl,
config.WorkQueueManager.dbname)
print "\nFirst attempt to update prio of wfs that are not in WMBS and only in local queue"
priorityCache = {}
workflowsToUpdate = {}
workflowsToCheck = [x for x in workqueue.getAvailableWorkflows()]
print "Retrieved %d workflows from workqueue" % len(workflowsToCheck)
for workflow, priority in workflowsToCheck:
if workflow not in priorityCache:
try:
priorityCache[workflow] = reqmgr2.getRequestByNames(
workflow)[workflow]['RequestPriority']
except Exception, ex:
print "Couldn't retrieve the priority of request %s" % workflow
print "Error: %s" % ex
continue
if priority != priorityCache[workflow]:
workflowsToUpdate[workflow] = priorityCache[workflow]
class WorkQueueReqMgrInterface(object):
"""Helper class for ReqMgr interaction"""
def __init__(self, **kwargs):
if not kwargs.get('logger'):
import logging
kwargs['logger'] = logging
self.logger = kwargs['logger']
# this will break all in one test
self.reqMgr2 = ReqMgr(kwargs.get("reqmgr2_endpoint", None))
centralurl = kwargs.get("central_logdb_url", "")
identifier = kwargs.get("log_reporter", "")
# set the thread name before creat the log db.
# only sets that when it is not set already
myThread = threading.currentThread()
if myThread.getName() == "MainThread":
myThread.setName(self.__class__.__name__)
self.logdb = LogDB(centralurl, identifier, logger=self.logger)
self.previous_state = {}
def __call__(self, queue):
"""Synchronize WorkQueue and RequestManager"""
msg = ''
try: # pull in new work
work = self.queueNewRequests(queue)
msg += "New Work: %d\n" % work
except Exception:
self.logger.exception("Error caught during RequestManager pull")
try: # get additional open-running work
extraWork = self.addNewElementsToOpenRequests(queue)
msg += "Work added: %d\n" % extraWork
except Exception:
self.logger.exception("Error caught during RequestManager split")
try: # report back to ReqMgr
uptodate_elements = self.report(queue)
msg += "Updated ReqMgr status for: %s\n" % ", ".join([x['RequestName'] for x in uptodate_elements])
except Exception:
self.logger.exception("Error caught during RequestManager update")
else:
try: # Delete finished requests from WorkQueue
self.deleteFinishedWork(queue, uptodate_elements)
except Exception:
self.logger.exception("Error caught during work deletion")
queue.backend.recordTaskActivity('reqmgr_sync', msg)
def queueNewRequests(self, queue):
"""Get requests from regMgr and queue to workqueue"""
self.logger.info("Contacting Request manager for more work")
work = 0
workLoads = []
if queue.params['DrainMode']:
self.logger.info('Draining queue: Skip requesting work from ReqMgr')
return 0
try:
workLoads = self.getAvailableRequests()
except Exception as ex:
traceMsg = traceback.format_exc()
msg = "Error contacting RequestManager: %s" % traceMsg
self.logger.warning(msg)
return 0
for team, reqName, workLoadUrl in workLoads:
try:
try:
Lexicon.couchurl(workLoadUrl)
except Exception as ex: # can throw many errors e.g. AttributeError, AssertionError etc.
# check its not a local file
if not os.path.exists(workLoadUrl):
error = WorkQueueWMSpecError(None, "Workflow url validation error: %s" % str(ex))
raise error
self.logger.info("Processing request %s at %s" % (reqName, workLoadUrl))
units = queue.queueWork(workLoadUrl, request=reqName, team=team)
self.logdb.delete(reqName, "error", this_thread=True)
except TERMINAL_EXCEPTIONS as ex:
# fatal error - report back to ReqMgr
self.logger.error('Permanent failure processing request "%s": %s' % (reqName, str(ex)))
self.logger.info("Marking request %s as failed in ReqMgr" % reqName)
self.reportRequestStatus(reqName, 'Failed', message=str(ex))
continue
except (IOError, socket.error, CouchError, CouchConnectionError) as ex:
# temporary problem - try again later
msg = 'Error processing request "%s": will try again later.' % reqName
msg += '\nError: "%s"' % str(ex)
self.logger.info(msg)
self.logdb.post(reqName, msg, 'error')
continue
except Exception as ex:
# Log exception as it isnt a communication problem
msg = 'Error processing request "%s": will try again later.' % reqName
msg += '\nSee log for details.\nError: "%s"' % str(ex)
self.logger.exception('Unknown error processing %s' % reqName)
self.logdb.post(reqName, msg, 'error')
continue
try:
self.reportRequestStatus(reqName, "acquired")
except Exception as ex:
self.logger.warning("Unable to update ReqMgr state: %s" % str(ex))
self.logger.warning('Will try again later')
self.logger.info('%s units(s) queued for "%s"' % (units, reqName))
work += units
self.logger.info("%s element(s) obtained from RequestManager" % work)
return work
def report(self, queue):
"""Report queue status to ReqMgr."""
new_state = {}
uptodate_elements = []
now = time.time()
elements = queue.statusInbox(dictKey="RequestName")
if not elements:
return new_state
for ele in elements:
ele = elements[ele][0] # 1 element tuple
try:
request = self.reqMgr2.getRequestByNames(ele['RequestName'])
if not request:
msg = 'Failed to get request "%s" from ReqMgr2. Will try again later.' % ele['RequestName']
self.logger.warning(msg)
continue
request = request[0][ele['RequestName']]
if request['RequestStatus'] in ('failed', 'completed', 'announced',
'epic-FAILED', 'closed-out', 'rejected'):
# requests can be done in reqmgr but running in workqueue
# if request has been closed but agent cleanup actions
# haven't been run (or agent has been retired)
# Prune out obviously too old ones to avoid build up
if queue.params.get('reqmgrCompleteGraceTime', -1) > 0:
if (now - float(ele.updatetime)) > queue.params['reqmgrCompleteGraceTime']:
# have to check all elements are at least running and are old enough
request_elements = queue.statusInbox(WorkflowName=request['RequestName'])
if not any(
[x for x in request_elements if x['Status'] != 'Running' and not x.inEndState()]):
last_update = max([float(x.updatetime) for x in request_elements])
if (now - last_update) > queue.params['reqmgrCompleteGraceTime']:
self.logger.info(
"Finishing request %s as it is done in reqmgr" % request['RequestName'])
queue.doneWork(WorkflowName=request['RequestName'])
continue
else:
pass # assume workqueue status will catch up later
elif request['RequestStatus'] in ['aborted', 'force-complete']:
queue.cancelWork(WorkflowName=request['RequestName'])
# Check consistency of running-open/closed and the element closure status
elif request['RequestStatus'] == 'running-open' and not ele.get('OpenForNewData', False):
self.reportRequestStatus(ele['RequestName'], 'running-closed')
elif request['RequestStatus'] == 'running-closed' and ele.get('OpenForNewData', False):
queue.closeWork(ele['RequestName'])
# we do not want to move the request to 'failed' status
elif ele['Status'] == 'Failed':
continue
elif ele['Status'] not in self._reqMgrToWorkQueueStatus(request['RequestStatus']):
self.reportElement(ele)
uptodate_elements.append(ele)
except Exception as ex:
msg = 'Error talking to ReqMgr about request "%s": %s' % (ele['RequestName'], str(ex))
self.logger.exception(msg)
return uptodate_elements
def deleteFinishedWork(self, queue, elements):
"""Delete work from queue that is finished in ReqMgr"""
finished = []
for element in elements:
if element.inEndState() and self._workQueueToReqMgrStatus(element['Status']) in ('aborted', 'failed', 'completed',
'announced', 'epic-FAILED',
'closed-out', 'rejected'):
finished.append(element['RequestName'])
return queue.deleteWorkflows(*finished)
def getAvailableRequests(self):
"""
Get available requests and sort by team and priority
returns [(team, request_name, request_spec_url)]
"""
tempResults = self.reqMgr2.getRequestByStatus("assigned")
filteredResults = []
for requests in tempResults:
for request in requests.values():
filteredResults.append(request)
filteredResults.sort(key=itemgetter('RequestPriority'), reverse=True)
filteredResults.sort(key=lambda r: r["Teams"][0])
results = [(x["Teams"][0], x["RequestName"], x["RequestWorkflow"]) for x in filteredResults]
return results
def reportRequestStatus(self, request, status, message=None):
"""Change state in RequestManager
Optionally, take a message to append to the request
"""
if message:
self.logdb.post(request, str(message), 'info')
reqmgrStatus = self._workQueueToReqMgrStatus(status)
if reqmgrStatus: # only send known states
try:
self.reqMgr2.updateRequestStatus(request, reqmgrStatus)
except Exception as ex:
msg = "%s : fail to update status will try later: %s" % (request, str(ex))
msg += traceback.format_exc()
self.logdb.post(request, msg, 'warning')
raise ex
return
def _workQueueToReqMgrStatus(self, status):
"""Map WorkQueue Status to that reported to ReqMgr"""
statusMapping = {'Acquired': 'acquired',
'Running': 'running-open',
'Failed': 'failed',
'Canceled': 'aborted',
'CancelRequested': 'aborted',
'Done': 'completed'
}
if status in statusMapping:
# if wq status passed convert to reqmgr status
return statusMapping[status]
elif status in REQUEST_STATE_LIST:
# if reqmgr status passed return reqmgr status
return status
else:
# unknown status
return None
def _reqMgrToWorkQueueStatus(self, status):
"""Map ReqMgr status to that in a WorkQueue element, it is not a 1-1 relation"""
statusMapping = {'acquired': ['Acquired'],
'running': ['Running'],
'running-open': ['Running'],
'running-closed': ['Running'],
'failed': ['Failed'],
'aborted': ['Canceled', 'CancelRequested'],
'force-complete': ['Canceled', 'CancelRequested'],
'completed': ['Done']}
if status in statusMapping:
return statusMapping[status]
else:
return []
def reportElement(self, element):
"""Report element to ReqMgr"""
self.reportRequestStatus(element['RequestName'], element['Status'])
def addNewElementsToOpenRequests(self, queue):
"""Add new elements to open requests which are in running-open state, only works adding new blocks from the input dataset"""
self.logger.info("Checking Request Manager for open requests and closing old ones")
# First close any open inbox element which hasn't found anything new in a while
queue.closeWork()
self.report(queue)
work = 0
requests = []
# Drain mode, don't pull any work into open requests. They will be closed if the queue stays in drain long enough
if queue.params['DrainMode']:
self.logger.info('Draining queue: Skip requesting work from ReqMgr')
return 0
try:
requests = self.reqMgr2.getRequestByStatus("running-open", detail=False)
except Exception as ex:
traceMsg = traceback.format_exc()
msg = "Error contacting RequestManager: %s" % traceMsg
self.logger.warning(msg)
return 0
for reqName in requests:
try:
self.logger.info("Processing request %s" % (reqName))
units = queue.addWork(requestName=reqName)
self.logdb.delete(reqName, 'error', True)
except (WorkQueueWMSpecError, WorkQueueNoWorkError) as ex:
# fatal error - but at least it was split the first time. Log and skip.
msg = 'Error adding further work to request "%s". Will try again later' % reqName
msg += '\nError: "%s"' % str(ex)
self.logger.info(msg)
self.logdb.post(reqName, msg, 'error')
continue
except (IOError, socket.error, CouchError, CouchConnectionError) as ex:
# temporary problem - try again later
msg = 'Error processing request "%s": will try again later.' % reqName
msg += '\nError: "%s"' % str(ex)
self.logger.info(msg)
self.logdb.post(reqName, msg, 'error')
continue
except Exception as ex:
# Log exception as it isnt a communication problem
msg = 'Error processing request "%s": will try again later.' % reqName
msg += '\nSee log for details.\nError: "%s"' % str(ex)
traceMsg = traceback.format_exc()
msg = "%s\n%s" % (msg, traceMsg)
self.logger.exception('Unknown error processing %s' % reqName)
self.logdb.post(reqName, msg, 'error')
continue
self.logger.info('%s units(s) queued for "%s"' % (units, reqName))
work += units
self.logger.info("%s element(s) added to open requests" % work)
return work
class ReqMgrTest(RESTBaseUnitTestWithDBBackend):
"""
Test WorkQueue Service client
It will start WorkQueue RESTService
Server DB sets from environment variable.
Client DB sets from environment variable.
This checks whether DS call makes without error and return the results.
Not the correctness of functions. That will be tested in different module.
"""
def setFakeDN(self):
# put into ReqMgr auxiliary database under "software" document scram/cmsms
# which we'll need a little for request injection
#Warning: this assumes the same structure in jenkins wmcore_root/test
self.admin_header = getAuthHeader(self.test_authz_key.data, ADMIN_PERMISSION)
self.create_header = getAuthHeader(self.test_authz_key.data, CREATE_PERMISSION)
self.default_header = getAuthHeader(self.test_authz_key.data, DEFAULT_PERMISSION)
self.assign_header = getAuthHeader(self.test_authz_key.data, ASSIGN_PERMISSION)
self.default_status_header = getAuthHeader(self.test_authz_key.data, DEFAULT_STATUS_PERMISSION)
def setUp(self):
self.setConfig(config)
self.setCouchDBs([(config.views.data.couch_reqmgr_db, "ReqMgr"),
(config.views.data.couch_reqmgr_aux_db, None)])
self.setSchemaModules([])
RESTBaseUnitTestWithDBBackend.setUp(self)
self.setFakeDN()
requestPath = os.path.join(getWMBASE(), "test", "data", "ReqMgr", "requests", "DMWM")
rerecoFile = open(os.path.join(requestPath, "ReReco.json"), 'r')
rerecoArgs = JsonWrapper.load(rerecoFile)
self.rerecoCreateArgs = rerecoArgs["createRequest"]
self.rerecoAssignArgs = rerecoArgs["assignRequest"]
cmsswDoc = {"_id": "software"}
cmsswDoc[self.rerecoCreateArgs["ScramArch"]] = []
cmsswDoc[self.rerecoCreateArgs["ScramArch"]].append(self.rerecoCreateArgs["CMSSWVersion"])
insertDataToCouch(os.getenv("COUCHURL"), config.views.data.couch_reqmgr_aux_db, cmsswDoc)
self.reqSvc = ReqMgr(self.jsonSender["host"])
self.reqSvc._noStale = True
self.reqSvc['requests'].additionalHeaders = self.create_header
def tearDown(self):
RESTBaseUnitTestWithDBBackend.tearDown(self)
def testRequestSimpleCycle(self):
"""
test request cycle with one request without composite get condition.
post, get, put
"""
# test post method
response = self.reqSvc.insertRequests(self.rerecoCreateArgs)
self.assertEqual(len(response), 1)
requestName = response[0]['request']
## test get method
# get by name
response = self.reqSvc.getRequestByNames(requestName)
self.assertEqual(response[requestName]['RequestPriority'], 10000)
self.assertEqual(len(response), 1)
# get by status
response = self.reqSvc.getRequestByStatus('new')
self.assertEqual(len(response), 1)
print(response)
self.reqSvc.updateRequestStatus(requestName, 'assignment-approved')
response = self.reqSvc.getRequestByStatus('assignment-approved')
self.assertEqual(len(response), 1)
self.reqSvc.updateRequestProperty(requestName, {'RequestStatus': 'assigned',
"AcquisitionEra": "TEST_ERA",
"Team": "unittest",
"SiteWhitelist": ["T1_US_CBS"],
"SiteBlacklist": ["T1_US_FOX"]})
response = self.reqSvc.getRequestByStatus('assignment-approved')
self.assertEqual(len(response), 0)
response = self.reqSvc.getRequestByStatus('assigned')
self.assertEqual(len(response), 1)
self.assertEqual(response.values()[0]["SiteWhitelist"], ["T1_US_CBS"])
self.reqSvc.updateRequestStats(requestName, {'total_jobs': 100, 'input_lumis': 100,
'input_events': 100, 'input_num_files': 100})
class WorkQueueReqMgrInterface(object):
"""Helper class for ReqMgr interaction"""
def __init__(self, **kwargs):
if not kwargs.get('logger'):
import logging
kwargs['logger'] = logging
self.logger = kwargs['logger']
# this will break all in one test
self.reqMgr2 = ReqMgr(kwargs.get("reqmgr2_endpoint", None))
centralurl = kwargs.get("central_logdb_url", "")
identifier = kwargs.get("log_reporter", "")
# set the thread name before creat the log db.
# only sets that when it is not set already
myThread = threading.currentThread()
if myThread.getName() == "MainThread":
myThread.setName(self.__class__.__name__)
self.logdb = LogDB(centralurl, identifier, logger=self.logger)
self.previous_state = {}
def __call__(self, queue):
"""Synchronize WorkQueue and RequestManager"""
msg = ''
try: # pull in new work
self.logger.info("queueing new work")
work = self.queueNewRequests(queue)
msg += "New Work: %d\n" % work
except Exception as ex:
errorMsg = "Error caught during RequestManager pull"
self.logger.exception("%s: %s", errorMsg, str(ex))
try: # get additional open-running work
self.logger.info("adding new element to open requests")
extraWork = self.addNewElementsToOpenRequests(queue)
msg += "Work added: %d\n" % extraWork
except Exception as ex:
errorMsg = "Error caught during RequestManager split"
self.logger.exception("%s: %s", errorMsg, str(ex))
try: # report back to ReqMgr
self.logger.info("cancel aborted requests")
count = self.cancelWork(queue)
self.logger.info("finised canceling requests")
msg += "Work canceled: %s " % count
except Exception as ex:
errorMsg = "Error caught during canceling the request"
self.logger.exception("%s: %s", errorMsg, str(ex))
queue.backend.recordTaskActivity('reqmgr_sync', msg)
def queueNewRequests(self, queue):
"""Get requests from regMgr and queue to workqueue"""
self.logger.info("Contacting Request manager for more work")
work = 0
workLoads = []
try:
workLoads = self.getAvailableRequests()
except Exception as ex:
traceMsg = traceback.format_exc()
msg = "Error contacting RequestManager: %s" % traceMsg
self.logger.warning(msg)
return 0
for team, reqName, workLoadUrl in workLoads:
try:
try:
Lexicon.couchurl(workLoadUrl)
except Exception as ex: # can throw many errors e.g. AttributeError, AssertionError etc.
# check its not a local file
if not os.path.exists(workLoadUrl):
error = WorkQueueWMSpecError(
None,
"Workflow url validation error: %s" % str(ex))
raise error
self.logger.info("Processing request %s at %s" %
(reqName, workLoadUrl))
units = queue.queueWork(workLoadUrl,
request=reqName,
team=team)
self.logdb.delete(reqName,
"error",
this_thread=True,
agent=False)
except TERMINAL_EXCEPTIONS as ex:
# fatal error - report back to ReqMgr
self.logger.error(
'Permanent failure processing request "%s": %s' %
(reqName, str(ex)))
self.logger.info("Marking request %s as failed in ReqMgr" %
reqName)
self.reportRequestStatus(reqName, 'Failed', message=str(ex))
continue
except (IOError, socket.error, CouchError,
CouchConnectionError) as ex:
# temporary problem - try again later
msg = 'Error processing request "%s": will try again later.' % reqName
msg += '\nError: "%s"' % str(ex)
self.logger.info(msg)
self.logdb.post(reqName, msg, 'error')
continue
except Exception as ex:
# Log exception as it isnt a communication problem
msg = 'Error processing request "%s": will try again later.' % reqName
msg += '\nSee log for details.\nError: "%s"' % str(ex)
self.logger.exception('Unknown error processing %s' % reqName)
self.logdb.post(reqName, msg, 'error')
continue
self.logger.info('%s units(s) queued for "%s"' % (units, reqName))
work += units
self.logger.info("%s element(s) obtained from RequestManager" %
work)
return work
def cancelWork(self, queue):
requests = self.reqMgr2.getRequestByStatus(
['aborted', 'force-complete'], detail=False)
count = 0
for req in requests:
try:
queue.cancelWork(req)
count += 1
except Exception as ex:
msg = 'Error to cancel the request "%s": %s' % (req, str(ex))
self.logger.exception(msg)
return count
def report(self, queue):
"""Report queue status to ReqMgr."""
new_state = {}
uptodate_elements = []
now = time.time()
elements = queue.statusInbox(dictKey="RequestName")
if not elements:
return new_state
for ele in elements:
ele = elements[ele][0] # 1 element tuple
try:
request = self.reqMgr2.getRequestByNames(ele['RequestName'])
if not request:
msg = 'Failed to get request "%s" from ReqMgr2. Will try again later.' % ele[
'RequestName']
self.logger.warning(msg)
continue
request = request[0][ele['RequestName']]
if request['RequestStatus'] in ('failed', 'completed',
'announced', 'closed-out',
'rejected'):
# requests can be done in reqmgr but running in workqueue
# if request has been closed but agent cleanup actions
# haven't been run (or agent has been retired)
# Prune out obviously too old ones to avoid build up
if queue.params.get('reqmgrCompleteGraceTime', -1) > 0:
if (now - float(ele.updatetime)
) > queue.params['reqmgrCompleteGraceTime']:
# have to check all elements are at least running and are old enough
request_elements = queue.statusInbox(
WorkflowName=request['RequestName'])
if not any([
x for x in request_elements
if x['Status'] != 'Running'
and not x.inEndState()
]):
last_update = max([
float(x.updatetime)
for x in request_elements
])
if (
now - last_update
) > queue.params['reqmgrCompleteGraceTime']:
self.logger.info(
"Finishing request %s as it is done in reqmgr"
% request['RequestName'])
queue.doneWork(
WorkflowName=request['RequestName'])
continue
else:
pass # assume workqueue status will catch up later
elif request['RequestStatus'] in ['aborted', 'force-complete']:
queue.cancelWork(WorkflowName=request['RequestName'])
# Check consistency of running-open/closed and the element closure status
elif request['RequestStatus'] == 'running-open' and not ele.get(
'OpenForNewData', False):
self.reportRequestStatus(ele['RequestName'],
'running-closed')
elif request['RequestStatus'] == 'running-closed' and ele.get(
'OpenForNewData', False):
queue.closeWork(ele['RequestName'])
# we do not want to move the request to 'failed' status
elif ele['Status'] == 'Failed':
continue
elif ele['Status'] not in self._reqMgrToWorkQueueStatus(
request['RequestStatus']):
self.reportElement(ele)
uptodate_elements.append(ele)
except Exception as ex:
msg = 'Error talking to ReqMgr about request "%s": %s' % (
ele['RequestName'], str(ex))
self.logger.exception(msg)
return uptodate_elements
def deleteFinishedWork(self, queue, elements):
"""Delete work from queue that is finished in ReqMgr"""
finished = []
for element in elements:
if element.inEndState():
finished.append(element['RequestName'])
return queue.deleteWorkflows(*finished)
def getAvailableRequests(self):
"""
Get available requests and sort by team and priority
returns [(team, request_name, request_spec_url)]
"""
tempResults = self.reqMgr2.getRequestByStatus("staged")
filteredResults = []
for requests in tempResults:
for request in requests.values():
filteredResults.append(request)
filteredResults.sort(key=itemgetter('RequestPriority'), reverse=True)
filteredResults.sort(key=lambda r: r["Team"])
results = [(x["Team"], x["RequestName"], x["RequestWorkflow"])
for x in filteredResults]
return results
def reportRequestStatus(self, request, status, message=None):
"""Change state in RequestManager
Optionally, take a message to append to the request
"""
if message:
logType = "error" if status == "Failed" else "info"
self.logdb.post(request, str(message), logType)
reqmgrStatus = self._workQueueToReqMgrStatus(status)
if reqmgrStatus: # only send known states
try:
self.reqMgr2.updateRequestStatus(request, reqmgrStatus)
except Exception as ex:
msg = "%s : fail to update status will try later: %s" % (
request, str(ex))
msg += traceback.format_exc()
self.logdb.post(request, msg, 'warning')
raise ex
return
def _workQueueToReqMgrStatus(self, status):
"""Map WorkQueue Status to that reported to ReqMgr"""
statusMapping = {
'Acquired': 'acquired',
'Running': 'running-open',
'Failed': 'failed',
'Canceled': 'aborted',
'CancelRequested': 'aborted',
'Done': 'completed'
}
if status in statusMapping:
# if wq status passed convert to reqmgr status
return statusMapping[status]
elif status in REQUEST_STATE_LIST:
# if reqmgr status passed return reqmgr status
return status
else:
# unknown status
return None
def _reqMgrToWorkQueueStatus(self, status):
"""Map ReqMgr status to that in a WorkQueue element, it is not a 1-1 relation"""
statusMapping = {
'acquired': ['Acquired'],
'running': ['Running'],
'running-open': ['Running'],
'running-closed': ['Running'],
'failed': ['Failed'],
'aborted': ['Canceled', 'CancelRequested'],
'force-complete': ['Canceled', 'CancelRequested'],
'completed': ['Done']
}
if status in statusMapping:
return statusMapping[status]
else:
return []
def reportElement(self, element):
"""Report element to ReqMgr"""
self.reportRequestStatus(element['RequestName'], element['Status'])
def addNewElementsToOpenRequests(self, queue):
"""Add new elements to open requests which are in running-open state, only works adding new blocks from the input dataset"""
self.logger.info(
"Checking Request Manager for open requests and closing old ones")
work = 0
requests = []
try:
requests = self.reqMgr2.getRequestByStatus("running-open",
detail=False)
except Exception as ex:
traceMsg = traceback.format_exc()
msg = "Error contacting RequestManager: %s" % traceMsg
self.logger.warning(msg)
return 0
for reqName in requests:
try:
self.logger.info("Processing request %s" % (reqName))
units = queue.addWork(requestName=reqName)
self.logdb.delete(reqName, 'error', True, agent=False)
except (WorkQueueWMSpecError, WorkQueueNoWorkError) as ex:
# fatal error - but at least it was split the first time. Log and skip.
msg = 'Error adding further work to request "%s". Will try again later' % reqName
msg += '\nError: "%s"' % str(ex)
self.logger.info(msg)
self.logdb.post(reqName, msg, 'error')
continue
except (IOError, socket.error, CouchError,
CouchConnectionError) as ex:
# temporary problem - try again later
msg = 'Error processing request "%s": will try again later.' % reqName
msg += '\nError: "%s"' % str(ex)
self.logger.info(msg)
self.logdb.post(reqName, msg, 'error')
continue
except Exception as ex:
# Log exception as it isnt a communication problem
msg = 'Error processing request "%s": will try again later.' % reqName
msg += '\nSee log for details.\nError: "%s"' % str(ex)
traceMsg = traceback.format_exc()
msg = "%s\n%s" % (msg, traceMsg)
self.logger.exception('Unknown error processing %s' % reqName)
self.logdb.post(reqName, msg, 'error')
continue
self.logger.info('%s units(s) queued for "%s"' % (units, reqName))
work += units
self.logger.info("%s element(s) added to open requests" % work)
return work
class ReqMgrService(TemplatedPage):
"""
Request Manager web service class
"""
def __init__(self, app, config, mount):
self.base = config.base
self.rootdir = '/'.join(WMCore.__file__.split('/')[:-1])
if config and not isinstance(config, dict):
web_config = config.dictionary_()
if not config:
web_config = {'base': self.base}
TemplatedPage.__init__(self, web_config)
imgdir = os.environ.get('RM_IMAGESPATH', os.getcwd() + '/images')
self.imgdir = web_config.get('imgdir', imgdir)
cssdir = os.environ.get('RM_CSSPATH', os.getcwd() + '/css')
self.cssdir = web_config.get('cssdir', cssdir)
jsdir = os.environ.get('RM_JSPATH', os.getcwd() + '/js')
self.jsdir = web_config.get('jsdir', jsdir)
spdir = os.environ.get('RM_SPECPATH', os.getcwd() + '/specs')
self.spdir = web_config.get('spdir', spdir)
# read scripts area and initialize data-ops scripts
self.sdir = os.environ.get('RM_SCRIPTS', os.getcwd() + '/scripts')
self.sdir = web_config.get('sdir', self.sdir)
self.sdict_thr = web_config.get('sdict_thr',
600) # put reasonable 10 min interval
self.sdict = {'ts': time.time()} # placeholder for data-ops scripts
self.update_scripts(force=True)
# To be filled at run time
self.cssmap = {}
self.jsmap = {}
self.imgmap = {}
self.yuimap = {}
std_specs_dir = os.path.join(self.rootdir, 'WMSpec/StdSpecs')
self.std_specs = spec_list(std_specs_dir, 'WMSpec.StdSpecs')
self.std_specs.sort()
# Update CherryPy configuration
mime_types = ['text/css']
mime_types += [
'application/javascript', 'text/javascript',
'application/x-javascript', 'text/x-javascript'
]
cherryconf.update({
'tools.encode.on': True,
'tools.gzip.on': True,
'tools.gzip.mime_types': mime_types,
})
self._cache = {}
# initialize rest API
statedir = '/tmp'
app = RESTMain(config, statedir) # REST application
mount = '/rest' # mount point for cherrypy service
api = RestApiHub(app, config.reqmgr, mount)
# initialize access to reqmgr2 APIs
self.reqmgr = ReqMgr(config.reqmgr.reqmgr2_url)
# only gets current view (This might cause to reponse time much longer,
# If upto date view is not needed overwrite Fale)
self.reqmgr._noStale = True
# admin helpers
self.admin_info = Info(app, api, config.reqmgr, mount=mount + '/info')
self.admin_group = Group(app,
api,
config.reqmgr,
mount=mount + '/group')
self.admin_team = Team(app, api, config.reqmgr, mount=mount + '/team')
# get fields which we'll use in templates
cdict = config.reqmgr.dictionary_()
self.couch_url = cdict.get('couch_host', '')
self.couch_dbname = cdict.get('couch_reqmgr_db', '')
self.couch_wdbname = cdict.get('couch_workload_summary_db', '')
self.acdc_url = cdict.get('acdc_host', '')
self.acdc_dbname = cdict.get('acdc_db', '')
self.configcache_url = cdict.get('couch_config_cache_url',
self.couch_url)
self.dbs_url = cdict.get('dbs_url', '')
self.dqm_url = cdict.get('dqm_url', '')
self.sw_ver = cdict.get('default_sw_version', 'CMSSW_5_2_5')
self.sw_arch = cdict.get('default_sw_scramarch', 'slc5_amd64_gcc434')
def user(self):
"""
Return user name associated with this instance.
"""
try:
return cherrypy.request.user['login']
except:
return 'testuser'
def user_dn(self):
"Return user DN"
try:
return cherrypy.request.user['dn']
except:
return '/CN/bla/foo'
def update_scripts(self, force=False):
"Update scripts dict"
if force or abs(time.time() - self.sdict['ts']) > self.sdict_thr:
for item in os.listdir(self.sdir):
with open(os.path.join(self.sdir, item), 'r') as istream:
self.sdict[item.split('.')[0]] = istream.read()
self.sdict['ts'] = time.time()
def abs_page(self, tmpl, content):
"""generate abstract page"""
menu = self.templatepage('menu', menus=menus(), tmpl=tmpl)
body = self.templatepage('generic', menu=menu, content=content)
page = self.templatepage('main', content=body, user=self.user())
return page
def page(self, content):
"""
Provide page wrapped with top/bottom templates.
"""
return self.templatepage('main', content=content)
def error(self, content):
"Generate common error page"
content = self.templatepage('error', content=content)
return self.abs_page('error', content)
@expose
def index(self, **kwds):
"""Main page"""
content = self.templatepage('index',
requests=ACTIVE_STATUS,
rdict=REQUEST_STATE_TRANSITION)
return self.abs_page('main', content)
@expose
def home(self, **kwds):
"""Main page"""
return self.index(**kwds)
### Admin actions ###
@expose
def admin(self, **kwds):
"""admin page"""
print "\n### ADMIN PAGE"
rows = self.admin_info.get()
print "rows", [r for r in rows]
content = self.templatepage('admin')
return self.abs_page('admin', content)
@expose
def add_user(self, **kwds):
"""add_user action"""
rid = genid(kwds)
status = "ok" # chagne to whatever it would be
content = self.templatepage('confirm',
ticket=rid,
user=self.user(),
status=status)
return self.abs_page('admin', content)
@expose
def add_group(self, **kwds):
"""add_group action"""
rows = self.admin_group.get()
print "\n### GROUPS", [r for r in rows]
rid = genid(kwds)
status = "ok" # chagne to whatever it would be
content = self.templatepage('confirm',
ticket=rid,
user=self.user(),
status=status)
return self.abs_page('admin', content)
@expose
def add_team(self, **kwds):
"""add_team action"""
rows = self.admin_team.get()
print "\n### TEAMS", kwds, [r for r in rows]
print "request to add", kwds
rid = genid(kwds)
status = "ok" # chagne to whatever it would be
content = self.templatepage('confirm',
ticket=rid,
user=self.user(),
status=status)
return self.abs_page('admin', content)
### Request actions ###
@expose
@checkargs(['status', 'sort'])
def assign(self, **kwds):
"""assign page"""
if not kwds:
kwds = {}
if 'status' not in kwds:
kwds.update({'status': 'assignment-approved'})
docs = []
attrs = [
'RequestName', 'RequestDate', 'Group', 'Requestor', 'RequestStatus'
]
data = self.reqmgr.getRequestByStatus(statusList=[kwds['status']])
for key, val in data.items():
docs.append(request_attr(val, attrs))
sortby = kwds.get('sort', 'status')
docs = [r for r in sort(docs, sortby)]
misc_json = {
'CMSSW Releases': releases(),
'CMSSW architectures': architectures(),
'SubscriptionPriority': ['Low', 'Normal', 'High'],
'CustodialSubType': ['Move', 'Replica'],
'NonCustodialSubType': ['Move', 'Replica'],
'MinMergeSize': 2147483648,
'MaxMergeSize': 4294967296,
'MaxMergeEvents': 50000,
'MaxRSS': 20411724,
'MaxVSize': 20411724,
'SoftTimeout': 129600,
'GracePeriod': 300,
'BlockCloseMaxWaitTime': 66400,
'BlockCloseMaxFiles': 500,
'BlockCloseMaxEvents': 250000000,
'BlockCloseMaxSize': 5000000000000,
'AcquisitionEra': '',
'ProcessingVersion': 1,
'ProcessingString': '',
'MergedLFNBase': lfn_bases(),
'UnmergedLFNBase': lfn_unmerged_bases(),
}
filter_sort = self.templatepage('filter_sort')
content = self.templatepage('assign',
sort=sortby,
filter_sort_table=filter_sort,
sites=sites(),
site_white_list=site_white_list(),
site_black_list=site_black_list(),
user=self.user(),
user_dn=self.user_dn(),
requests=docs,
misc_table=json2table(
misc_json, web_ui_names()),
misc_json=json2form(misc_json,
indent=2,
keep_first_value=True))
return self.abs_page('assign', content)
@expose
@checkargs(['status', 'sort'])
def approve(self, **kwds):
"""
Approve page: get list of request associated with user DN.
Fetch their status list from ReqMgr and display if requests
were seen by data-ops.
"""
if not kwds:
kwds = {}
if 'status' not in kwds:
kwds.update({'status': 'new'})
kwds.update({'_nostale': True})
docs = []
attrs = [
'RequestName', 'RequestDate', 'Group', 'Requestor', 'RequestStatus'
]
data = self.reqmgr.getRequestByStatus(statusList=[kwds['status']])
for key, val in data.items():
docs.append(request_attr(val, attrs))
sortby = kwds.get('sort', 'status')
docs = [r for r in sort(docs, sortby)]
filter_sort = self.templatepage('filter_sort')
content = self.templatepage('approve',
requests=docs,
date=tstamp(),
sort=sortby,
filter_sort_table=filter_sort)
return self.abs_page('approve', content)
@expose
def create(self, **kwds):
"""create page"""
# get list of standard specs from WMCore and new ones from local area
#loc_specs_dir = os.path.join(self.spdir, 'Specs') # local specs
#loc_specs = spec_list(loc_specs_dir, 'Specs')
#all_specs = list(set(self.std_specs + loc_specs))
#all_specs.sort()
all_specs = self.std_specs
spec = kwds.get('form', '')
if not spec:
spec = self.std_specs[0]
# make spec first in all_specs list
if spec in all_specs:
all_specs.remove(spec)
all_specs = [spec] + all_specs
jsondata = get_request_template_from_type(spec)
# create templatized page out of provided forms
self.update_scripts()
content = self.templatepage(
'create',
table=json2table(jsondata, web_ui_names()),
jsondata=json2form(jsondata, indent=2, keep_first_value=True),
name=spec,
scripts=[s for s in self.sdict.keys() if s != 'ts'],
specs=all_specs)
return self.abs_page('create', content)
def generate_objs(self, script, jsondict):
"""Generate objects from givem JSON template"""
self.update_scripts()
code = self.sdict.get(script, '')
if code.find('def genobjs(jsondict)') == -1:
return self.error(
"Improper python snippet, your code should start with <b>def genobjs(jsondict)</b> function"
)
exec(code) # code snippet must starts with genobjs function
return [r for r in genobjs(jsondict)]
@expose
def fetch(self, rid, **kwds):
"Fetch document for given id"
rid = rid.replace('request-', '')
doc = self.reqmgr.getRequestByNames(rid)
transitions = []
if len(doc) == 1:
try:
doc = doc[rid]
except:
pass
name = doc.get('RequestName', 'NA')
title = 'Request %s' % name
status = doc.get('RequestStatus', '')
transitions = REQUEST_STATE_TRANSITION.get(status, [])
if status in transitions:
transitions.remove(status)
content = self.templatepage('doc',
title=title,
status=status,
name=name,
table=json2table(doc, web_ui_names()),
jsondata=json2form(
doc,
indent=2,
keep_first_value=False),
transitions=transitions)
elif len(doc) > 1:
jsondata = [pprint.pformat(d) for d in doc]
content = self.templatepage('doc',
title='Series of docs: %s' % rid,
table="",
jsondata=jsondata,
transitions=transitions)
else:
doc = 'No request found for name=%s' % rid
return self.abs_page('request', content)
@expose
def requests(self, **kwds):
"""Page showing requests"""
if not kwds:
kwds = {}
if 'status' not in kwds:
kwds.update({'status': 'acquired'})
results = self.reqmgr.getRequestByStatus(kwds['status'])
docs = []
for key, doc in results.items():
docs.append(request_attr(doc))
sortby = kwds.get('sort', 'status')
docs = [r for r in sort(docs, sortby)]
filter_sort = self.templatepage('filter_sort')
content = self.templatepage('requests',
requests=docs,
sort=sortby,
status=kwds['status'],
filter_sort_table=filter_sort)
return self.abs_page('requests', content)
@expose
def request(self, **kwargs):
"Get data example and expose it as json"
dataset = kwargs.get('uinput', '')
if not dataset:
return {'error': 'no input dataset'}
url = 'https://cmsweb.cern.ch/reqmgr/rest/outputdataset/%s' % dataset
params = {}
headers = {'Accept': 'application/json;text/json'}
wdata = getdata(url, params)
wdict = dict(date=time.ctime(),
team='Team-A',
status='Running',
ID=genid(wdata))
winfo = self.templatepage('workflow',
wdict=wdict,
dataset=dataset,
code=pprint.pformat(wdata))
content = self.templatepage('search', content=winfo)
return self.abs_page('request', content)
@expose
def batch(self, **kwds):
"""batch page"""
# TODO: we need a template for batch attributes
# and read it from separate area, like DASMaps
name = kwds.get('name', '')
batch = {}
if name:
# batch = self.reqmgr.getBatchesByName(name)
batch = {
'Name': 'Batch1',
'Description': 'Bla-bla',
'Creator': 'valya',
'Group': 'test',
'Workflows': ['workflow1', 'workflow2'],
'Attributes': {
'HeavyIon': ['true', 'false']
}
}
attributes = batch.get('Attributes', {})
workflows = batch.get('Workflows', [])
description = batch.get('Description', '')
creator = batch.get('Creator', self.user_dn())
content = self.templatepage('batch',
name=name,
attributes=json2table(
attributes, web_ui_names()),
workflows=workflows,
creator=creator,
description=description)
return self.abs_page('batch', content)
@expose
def batches(self, **kwds):
"""Page showing batches"""
if not kwds:
kwds = {}
if 'name' not in kwds:
kwds.update({'name': ''})
sortby = kwds.get('sort', 'name')
# results = self.reqmgr.getBatchesByName(kwds['name'])
results = [
{
'Name': 'Batch1',
'Description': 'Bla-bla',
'Creator': 'valya',
'Group': 'test',
'Workflows': ['workflow1', 'workflow2'],
'Date': 'Fri Feb 13 10:36:41 EST 2015',
'Attributes': {
'HeavyIon': ['true', 'false']
}
},
{
'Name': 'Batch2',
'Description': 'lksdjflksjdf',
'Creator': 'valya',
'Group': 'test',
'Workflows': ['workflow1', 'workflow2'],
'Date': 'Fri Feb 10 10:36:41 EST 2015',
'Attributes': {
'HeavyIon': ['true', 'false']
}
},
]
docs = [r for r in sort(results, sortby)]
filter_sort = self.templatepage('filter_sort')
content = self.templatepage('batches',
batches=docs,
sort=sortby,
filter_sort_table=filter_sort)
return self.abs_page('batches', content)
### Aux methods ###
@expose
def put_request(self, *args, **kwds):
"PUT request callback to reqmgr server, should be used in AJAX"
reqname = kwds.get('RequestName', '')
status = kwds.get('RequestStatus', '')
if not reqname:
msg = 'Unable to update request status, empty request name'
raise cherrypy.HTTPError(406, msg)
if not status:
msg = 'Unable to update request status, empty status value'
raise cherrypy.HTTPError(406, msg)
return self.reqmgr.updateRequestStatus(reqname, status)
@expose
def images(self, *args, **kwargs):
"""
Serve static images.
"""
args = list(args)
self.check_scripts(args, self.imgmap, self.imgdir)
mime_types = [
'*/*', 'image/gif', 'image/png', 'image/jpg', 'image/jpeg'
]
accepts = cherrypy.request.headers.elements('Accept')
for accept in accepts:
if accept.value in mime_types and len(args) == 1 \
and args[0] in self.imgmap:
image = self.imgmap[args[0]]
# use image extension to pass correct content type
ctype = 'image/%s' % image.split('.')[-1]
cherrypy.response.headers['Content-type'] = ctype
return serve_file(image, content_type=ctype)
def serve(self, kwds, imap, idir, datatype='', minimize=False):
"Serve files for high level APIs (yui/css/js)"
args = []
for key, val in kwds.items():
if key == 'f': # we only look-up files from given kwds dict
if isinstance(val, list):
args += val
else:
args.append(val)
scripts = self.check_scripts(args, imap, idir)
return self.serve_files(args, scripts, imap, datatype, minimize)
@exposecss
@tools.gzip()
def css(self, **kwargs):
"""
Serve provided CSS files. They can be passed as
f=file1.css&f=file2.css
"""
resource = kwargs.get('resource', 'css')
if resource == 'css':
return self.serve(kwargs, self.cssmap, self.cssdir, 'css', True)
@exposejs
@tools.gzip()
def js(self, **kwargs):
"""
Serve provided JS scripts. They can be passed as
f=file1.js&f=file2.js with optional resource parameter
to speficy type of JS files, e.g. resource=yui.
"""
resource = kwargs.get('resource', 'js')
if resource == 'js':
return self.serve(kwargs, self.jsmap, self.jsdir)
def serve_files(self,
args,
scripts,
resource,
datatype='',
minimize=False):
"""
Return asked set of files for JS, YUI, CSS.
"""
idx = "-".join(scripts)
if idx not in self._cache.keys():
data = ''
if datatype == 'css':
data = '@CHARSET "UTF-8";'
for script in args:
path = os.path.join(sys.path[0], resource[script])
path = os.path.normpath(path)
ifile = open(path)
data = "\n".join ([data, ifile.read().\
replace('@CHARSET "UTF-8";', '')])
ifile.close()
if datatype == 'css':
set_headers("text/css")
if minimize:
self._cache[idx] = minify(data)
else:
self._cache[idx] = data
return self._cache[idx]
def check_scripts(self, scripts, resource, path):
"""
Check a script is known to the resource map
and that the script actually exists
"""
for script in scripts:
if script not in resource.keys():
spath = os.path.normpath(os.path.join(path, script))
if os.path.isfile(spath):
resource.update({script: spath})
return scripts
class JobUpdaterPoller(BaseWorkerThread):
"""
_JobUpdaterPoller_
Poller class for the JobUpdater
"""
def __init__(self, config):
"""
__init__
"""
BaseWorkerThread.__init__(self)
self.config = config
self.bossAir = BossAirAPI(config=self.config)
self.reqmgr2 = ReqMgr(self.config.JobUpdater.reqMgr2Url)
self.workqueue = WorkQueue(self.config.WorkQueueManager.couchurl,
self.config.WorkQueueManager.dbname)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.listWorkflowsDAO = self.daoFactory(classname="Workflow.ListForJobUpdater")
self.updateWorkflowPrioDAO = self.daoFactory(classname="Workflow.UpdatePriority")
self.executingJobsDAO = self.daoFactory(classname="Jobs.GetNumberOfJobsForWorkflowTaskStatus")
def setup(self, parameters=None):
"""
_setup_
"""
pass
def terminate(self, parameters=None):
"""
_terminate_
Terminate gracefully.
"""
pass
def algorithm(self, parameters=None):
"""
_algorithm_
"""
try:
logging.info("Synchronizing priorities with ReqMgr...")
self.synchronizeJobPriority()
logging.info("Priorities were synchronized, wait until the next cycle")
except CouchConnectionError as ex:
msg = "Caught CouchConnectionError exception in JobUpdater\n"
msg += "transactions postponed until the next polling cycle\n"
msg += str(ex)
logging.exception(msg)
except CouchConflictError as ex:
msg = "Caught CouchConflictError exception in JobUpdater\n"
msg += "transactions postponed until the next polling cycle\n"
msg += str(ex)
logging.exception(msg)
except Exception as ex:
if 'Connection refused' in str(ex):
logging.warn("Failed to sync priorities. Trying in the next cycle")
else:
msg = "Caught unexpected exception in JobUpdater: %s\n" % str(ex)
logging.exception(msg)
raise JobUpdaterException(msg)
def synchronizeJobPriority(self):
"""
_synchronizeJobPriority_
Check WMBS and WorkQueue for active workflows and compare with the
ReqMgr for priority changes. If a priority change occurs
then update the job priority in the batch system and
the elements in the local queue that have not been injected yet.
"""
# Update the priority of workflows that are not in WMBS and just in local queue
priorityCache = {}
workflowsToUpdate = {}
workflowsToCheck = [x for x in self.workqueue.getAvailableWorkflows()]
for workflow, priority in workflowsToCheck:
if workflow not in priorityCache:
try:
priorityCache[workflow] = self.reqmgr2.getRequestByNames(workflow)[workflow]['RequestPriority']
except Exception as ex:
logging.error("Couldn't retrieve the priority of request %s", workflow)
logging.error("Error: %s", str(ex))
continue
if priority != priorityCache[workflow]:
workflowsToUpdate[workflow] = priorityCache[workflow]
logging.info("Found %d workflows to update in workqueue", len(workflowsToUpdate))
for workflow in workflowsToUpdate:
self.workqueue.updatePriority(workflow, workflowsToUpdate[workflow])
# Check the workflows in WMBS
priorityCache = {}
workflowsToUpdateWMBS = {}
workflowsToCheck = self.listWorkflowsDAO.execute()
for workflowEntry in workflowsToCheck:
workflow = workflowEntry['name']
if workflow not in priorityCache:
try:
priorityCache[workflow] = self.reqmgr2.getRequestByNames(workflow)[workflow]['RequestPriority']
except Exception as ex:
logging.error("Couldn't retrieve the priority of request %s", workflow)
logging.error("Error: %s", str(ex))
continue
requestPriority = int(priorityCache[workflow])
if requestPriority != int(workflowEntry['workflow_priority']):
# Update the workqueue priority for the Available elements
self.workqueue.updatePriority(workflow, requestPriority)
# Check if there are executing jobs for this particular task
if self.executingJobsDAO.execute(workflow, workflowEntry['task']) > 0:
self.bossAir.updateJobInformation(workflow, workflowEntry['task'],
requestPriority=priorityCache[workflow],
taskPriority=workflowEntry['task_priority'])
workflowsToUpdateWMBS[workflow] = priorityCache[workflow]
if workflowsToUpdateWMBS:
logging.info("Updating %d workflows in WMBS.", len(workflowsToUpdateWMBS))
self.updateWorkflowPrioDAO.execute(workflowsToUpdateWMBS)
class JobUpdaterPoller(BaseWorkerThread):
"""
_JobUpdaterPoller_
Poller class for the JobUpdater
"""
def __init__(self, config):
"""
__init__
"""
BaseWorkerThread.__init__(self)
self.config = config
self.bossAir = BossAirAPI(config=self.config)
self.reqmgr2 = ReqMgr(self.config.General.ReqMgr2ServiceURL)
self.workqueue = WorkQueue(self.config.WorkQueueManager.couchurl,
self.config.WorkQueueManager.dbname)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.listWorkflowsDAO = self.daoFactory(
classname="Workflow.ListForJobUpdater")
self.updateWorkflowPrioDAO = self.daoFactory(
classname="Workflow.UpdatePriority")
self.executingJobsDAO = self.daoFactory(
classname="Jobs.GetNumberOfJobsForWorkflowTaskStatus")
def setup(self, parameters=None):
"""
_setup_
"""
pass
def terminate(self, parameters=None):
"""
_terminate_
Terminate gracefully.
"""
pass
@timeFunction
def algorithm(self, parameters=None):
"""
_algorithm_
"""
try:
logging.info("Synchronizing priorities with ReqMgr...")
self.synchronizeJobPriority()
logging.info(
"Priorities were synchronized, wait until the next cycle")
except CouchConnectionError as ex:
msg = "Caught CouchConnectionError exception in JobUpdater\n"
msg += "transactions postponed until the next polling cycle\n"
msg += str(ex)
logging.exception(msg)
except CouchConflictError as ex:
msg = "Caught CouchConflictError exception in JobUpdater\n"
msg += "transactions postponed until the next polling cycle\n"
msg += str(ex)
logging.exception(msg)
except Exception as ex:
if 'Connection refused' in str(ex):
logging.warn(
"Failed to sync priorities. Trying in the next cycle")
else:
msg = "Caught unexpected exception in JobUpdater: %s\n" % str(
ex)
logging.exception(msg)
raise JobUpdaterException(msg)
def synchronizeJobPriority(self):
"""
_synchronizeJobPriority_
Check WMBS and WorkQueue for active workflows and compare with the
ReqMgr for priority changes. If a priority change occurs
then update the job priority in the batch system and
the elements in the local queue that have not been injected yet.
"""
# Update the priority of workflows that are not in WMBS and just in local queue
priorityCache = {}
workflowsToUpdate = {}
workflowsToCheck = [x for x in self.workqueue.getAvailableWorkflows()]
for workflow, priority in workflowsToCheck:
if workflow not in priorityCache:
try:
result = self.reqmgr2.getRequestByNames(workflow)[0]
priorityCache[workflow] = result[workflow][
'RequestPriority']
except Exception as ex:
logging.error(
"Couldn't retrieve the priority of request %s",
workflow)
logging.error("Error: %s", str(ex))
continue
if priority != priorityCache[workflow]:
workflowsToUpdate[workflow] = priorityCache[workflow]
logging.info("Found %d workflows to update in workqueue",
len(workflowsToUpdate))
for workflow in workflowsToUpdate:
self.workqueue.updatePriority(workflow,
workflowsToUpdate[workflow])
# Check the workflows in WMBS
priorityCache = {}
workflowsToUpdateWMBS = {}
workflowsToCheck = self.listWorkflowsDAO.execute()
for workflowEntry in workflowsToCheck:
workflow = workflowEntry['name']
if workflow not in priorityCache:
try:
result = self.reqmgr2.getRequestByNames(workflow)[0]
priorityCache[workflow] = result[workflow][
'RequestPriority']
except Exception as ex:
logging.error(
"Couldn't retrieve the priority of request %s",
workflow)
logging.error("Error: %s", str(ex))
continue
requestPriority = int(priorityCache[workflow])
if requestPriority != int(workflowEntry['workflow_priority']):
# Update the workqueue priority for the Available elements
self.workqueue.updatePriority(workflow, requestPriority)
# Check if there are executing jobs for this particular task
if self.executingJobsDAO.execute(workflow,
workflowEntry['task']) > 0:
self.bossAir.updateJobInformation(
workflow,
workflowEntry['task'],
requestPriority=priorityCache[workflow],
taskPriority=workflowEntry['task_priority'])
workflowsToUpdateWMBS[workflow] = priorityCache[workflow]
if workflowsToUpdateWMBS:
logging.info("Updating %d workflows in WMBS.",
len(workflowsToUpdateWMBS))
self.updateWorkflowPrioDAO.execute(workflowsToUpdateWMBS)
class WorkQueueReqMgrInterface():
"""Helper class for ReqMgr interaction"""
def __init__(self, **kwargs):
if not kwargs.get('logger'):
import logging
kwargs['logger'] = logging
self.logger = kwargs['logger']
#TODO: (reqmgr2Only - remove this line when reqmgr is replaced)
self.reqMgr = RequestManager(kwargs)
#this will break all in one test
self.reqMgr2 = ReqMgr(kwargs.get("reqmgr2_endpoint", None))
centralurl = kwargs.get("central_logdb_url", "")
identifier = kwargs.get("log_reporter", "")
# set the thread name before creat the log db.
# only sets that when it is not set already
myThread = threading.currentThread()
if myThread.getName() == "MainThread":
myThread.setName(self.__class__.__name__)
self.logdb = LogDB(centralurl, identifier, logger=self.logger)
self.previous_state = {}
def __call__(self, queue):
"""Synchronize WorkQueue and RequestManager"""
msg = ''
try: # pull in new work
work = self.queueNewRequests(queue)
msg += "New Work: %d\n" % work
except Exception:
self.logger.exception("Error caught during RequestManager pull")
try: # get additional open-running work
extraWork = self.addNewElementsToOpenRequests(queue)
msg += "Work added: %d\n" % extraWork
except Exception:
self.logger.exception("Error caught during RequestManager split")
try: # report back to ReqMgr
uptodate_elements = self.report(queue)
msg += "Updated ReqMgr status for: %s\n" % ", ".join(
[x['RequestName'] for x in uptodate_elements])
except Exception:
self.logger.exception("Error caught during RequestManager update")
else:
try: # Delete finished requests from WorkQueue
self.deleteFinishedWork(queue, uptodate_elements)
except Exception:
self.logger.exception("Error caught during work deletion")
queue.backend.recordTaskActivity('reqmgr_sync', msg)
def queueNewRequests(self, queue):
"""Get requests from regMgr and queue to workqueue"""
self.logger.info("Contacting Request manager for more work")
work = 0
workLoads = []
if queue.params['DrainMode']:
self.logger.info(
'Draining queue: Skip requesting work from ReqMgr')
return 0
try:
workLoads = self.getAvailableRequests(queue.params['Teams'])
except Exception as ex:
traceMsg = traceback.format_exc()
msg = "Error contacting RequestManager: %s" % traceMsg
self.logger.warning(msg)
return 0
for team, reqName, workLoadUrl in workLoads:
# try:
# self.reportRequestStatus(reqName, "negotiating")
# except Exception, ex:
# self.logger.error("""
# Unable to update ReqMgr state to negotiating: %s
# Ignoring this request: %s""" % (str(ex), reqName))
# continue
try:
try:
Lexicon.couchurl(workLoadUrl)
except Exception as ex: # can throw many errors e.g. AttributeError, AssertionError etc.
# check its not a local file
if not os.path.exists(workLoadUrl):
error = WorkQueueWMSpecError(
None,
"Workflow url validation error: %s" % str(ex))
raise error
self.logger.info("Processing request %s at %s" %
(reqName, workLoadUrl))
units = queue.queueWork(workLoadUrl,
request=reqName,
team=team)
self.logdb.delete(reqName, "error", this_thread=True)
except (WorkQueueWMSpecError, WorkQueueNoWorkError) as ex:
# fatal error - report back to ReqMgr
self.logger.info(
'Permanent failure processing request "%s": %s' %
(reqName, str(ex)))
self.logger.info("Marking request %s as failed in ReqMgr" %
reqName)
self.reportRequestStatus(reqName, 'Failed', message=str(ex))
continue
except (IOError, socket.error, CouchError,
CouchConnectionError) as ex:
# temporary problem - try again later
msg = 'Error processing request "%s": will try again later.' \
'\nError: "%s"' % (reqName, str(ex))
self.logger.info(msg)
self.logdb.post(reqName, msg, 'error')
continue
except Exception as ex:
# Log exception as it isnt a communication problem
msg = 'Error processing request "%s": will try again later.' \
'\nSee log for details.\nError: "%s"' % (reqName, str(ex))
self.logger.exception('Unknown error processing %s' % reqName)
self.logdb.post(reqName, msg, 'error')
continue
try:
self.reportRequestStatus(reqName, "acquired")
except Exception as ex:
self.logger.warning("Unable to update ReqMgr state: %s" %
str(ex))
self.logger.warning('Will try again later')
self.logger.info('%s units(s) queued for "%s"' % (units, reqName))
work += units
self.logger.info("%s element(s) obtained from RequestManager" %
work)
return work
def report(self, queue):
"""Report queue status to ReqMgr."""
new_state = {}
uptodate_elements = []
now = time.time()
elements = queue.statusInbox(dictKey="RequestName")
if not elements:
return new_state
for ele in elements:
ele = elements[ele][0] # 1 element tuple
try:
request = self.reqMgr2.getRequestByNames(ele['RequestName'])
if not request:
msg = 'Failed to get request "%s" from ReqMgr2. Will try again later.' % ele[
'RequestName']
self.logger.warning(msg)
continue
request = request[ele['RequestName']]
if request['RequestStatus'] in ('failed', 'completed',
'announced', 'epic-FAILED',
'closed-out', 'rejected'):
# requests can be done in reqmgr but running in workqueue
# if request has been closed but agent cleanup actions
# haven't been run (or agent has been retired)
# Prune out obviously too old ones to avoid build up
if queue.params.get('reqmgrCompleteGraceTime', -1) > 0:
if (now - float(ele.updatetime)
) > queue.params['reqmgrCompleteGraceTime']:
# have to check all elements are at least running and are old enough
request_elements = queue.statusInbox(
WorkflowName=request['RequestName'])
if not any([
x for x in request_elements
if x['Status'] != 'Running'
and not x.inEndState()
]):
last_update = max([
float(x.updatetime)
for x in request_elements
])
if (
now - last_update
) > queue.params['reqmgrCompleteGraceTime']:
self.logger.info(
"Finishing request %s as it is done in reqmgr"
% request['RequestName'])
queue.doneWork(
WorkflowName=request['RequestName'])
continue
else:
pass # assume workqueue status will catch up later
elif request['RequestStatus'] == 'aborted' or request[
'RequestStatus'] == 'force-complete':
queue.cancelWork(WorkflowName=request['RequestName'])
# Check consistency of running-open/closed and the element closure status
elif request['RequestStatus'] == 'running-open' and not ele.get(
'OpenForNewData', False):
self.reportRequestStatus(ele['RequestName'],
'running-closed')
elif request['RequestStatus'] == 'running-closed' and ele.get(
'OpenForNewData', False):
queue.closeWork(ele['RequestName'])
# update request status if necessary
elif ele['Status'] not in self._reqMgrToWorkQueueStatus(
request['RequestStatus']):
self.reportElement(ele)
uptodate_elements.append(ele)
except Exception as ex:
msg = 'Error talking to ReqMgr about request "%s": %s'
traceMsg = traceback.format_exc()
self.logger.error(msg % (ele['RequestName'], traceMsg))
return uptodate_elements
def deleteFinishedWork(self, queue, elements):
"""Delete work from queue that is finished in ReqMgr"""
finished = []
for element in elements:
if self._workQueueToReqMgrStatus(element['Status']) in ('aborted', 'failed', 'completed', 'announced',
'epic-FAILED', 'closed-out', 'rejected') \
and element.inEndState():
finished.append(element['RequestName'])
return queue.deleteWorkflows(*finished)
def _getRequestsByTeamsAndStatus(self, status, teams=[]):
"""
TODO: now it assumes one team per requests - check whether this assumption is correct
Check whether we actually use the team for this.
Also switch to byteamandstatus couch call instead of
"""
requests = self.reqMgr2.getRequestByStatus(status)
#Then sort by Team name then sort by Priority
#https://docs.python.org/2/howto/sorting.html
if teams and len(teams) > 0:
results = {}
for reqName, value in requests.items():
if value["Teams"][0] in teams:
results[reqName] = value
return results
else:
return requests
def getAvailableRequests(self, teams):
"""
Get available requests for the given teams and sort by team and priority
returns [(team, request_name, request_spec_url)]
"""
tempResults = self._getRequestsByTeamsAndStatus("assigned",
teams).values()
filteredResults = []
for request in tempResults:
if "Teams" in request and len(request["Teams"]) == 1:
filteredResults.append(request)
self.logdb.delete(request["RequestName"],
"error",
this_thread=True)
else:
msg = "no team or more than one team (%s) are assigined: %s" % (
request.get("Teams", None), request["RequestName"])
self.logger.error(msg)
self.logdb.post(request["RequestName"], msg, 'error')
filteredResults.sort(key=itemgetter('RequestPriority'), reverse=True)
filteredResults.sort(key=lambda r: r["Teams"][0])
results = [(x["Teams"][0], x["RequestName"], x["RequestWorkflow"])
for x in filteredResults]
return results
def reportRequestStatus(self, request, status, message=None):
"""Change state in RequestManager
Optionally, take a message to append to the request
"""
if message:
self.logdb.post(request, str(message), 'info')
reqmgrStatus = self._workQueueToReqMgrStatus(status)
if reqmgrStatus: # only send known states
try:
#TODO: try reqmgr1 call if it fails (reqmgr2Only - remove this line when reqmgr is replaced)
self.reqMgr.reportRequestStatus(request, reqmgrStatus)
# And replace with this (remove all Exceptins)
#self.reqMgr2.updateRequestStatus(request, reqmgrStatus)
except HTTPException as ex:
# If we get an HTTPException of 404 means reqmgr2 request
if ex.status == 404:
# try reqmgr2 call
msg = "%s : reqmgr2 request: %s" % (request, str(ex))
self.logdb.post(request, msg, 'info')
self.reqMgr2.updateRequestStatus(request, reqmgrStatus)
else:
msg = "%s : fail to update status with HTTP error: %s" % (
request, str(ex))
self.logdb.post(request, msg, 'warning')
raise ex
except Exception as ex:
msg = "%s : fail to update status will try later: %s" % (
request, str(ex))
self.logdb.post(request, msg, 'warning')
raise ex
def markAcquired(self, request, url=None):
"""Mark request acquired"""
self.reqMgr.putWorkQueue(request, url)
def _workQueueToReqMgrStatus(self, status):
"""Map WorkQueue Status to that reported to ReqMgr"""
statusMapping = {
'Acquired': 'acquired',
'Running': 'running-open',
'Failed': 'failed',
'Canceled': 'aborted',
'CancelRequested': 'aborted',
'Done': 'completed'
}
if status in statusMapping:
# if wq status passed convert to reqmgr status
return statusMapping[status]
elif status in REQUEST_STATE_LIST:
# if reqmgr status passed return reqmgr status
return status
else:
# unknown status
return None
def _reqMgrToWorkQueueStatus(self, status):
"""Map ReqMgr status to that in a WorkQueue element, it is not a 1-1 relation"""
statusMapping = {
'acquired': ['Acquired'],
'running': ['Running'],
'running-open': ['Running'],
'running-closed': ['Running'],
'failed': ['Failed'],
'aborted': ['Canceled', 'CancelRequested'],
'force-complete': ['Canceled', 'CancelRequested'],
'completed': ['Done']
}
if status in statusMapping:
return statusMapping[status]
else:
return []
def reportElement(self, element):
"""Report element to ReqMgr"""
self.reportRequestStatus(element['RequestName'], element['Status'])
def addNewElementsToOpenRequests(self, queue):
"""Add new elements to open requests which are in running-open state, only works adding new blocks from the input dataset"""
self.logger.info(
"Checking Request Manager for open requests and closing old ones")
# First close any open inbox element which hasn't found anything new in a while
queue.closeWork()
self.report(queue)
work = 0
requests = []
# Drain mode, don't pull any work into open requests. They will be closed if the queue stays in drain long enough
if queue.params['DrainMode']:
self.logger.info(
'Draining queue: Skip requesting work from ReqMgr')
return 0
try:
requests = self._getRequestsByTeamsAndStatus(
"running-open", queue.params['Teams']).keys()
except Exception as ex:
traceMsg = traceback.format_exc()
msg = "Error contacting RequestManager: %s" % traceMsg
self.logger.warning(msg)
return 0
for reqName in requests:
try:
self.logger.info("Processing request %s" % (reqName))
units = queue.addWork(requestName=reqName)
self.logdb.delete(request["RequestName"], 'error', True)
except (WorkQueueWMSpecError, WorkQueueNoWorkError) as ex:
# fatal error - but at least it was split the first time. Log and skip.
msg = 'Error adding further work to request "%s". Will try again later' \
'\nError: "%s"' % (reqName, str(ex))
self.logger.info(msg)
self.logdb.post(reqName, msg, 'error')
continue
except (IOError, socket.error, CouchError,
CouchConnectionError) as ex:
# temporary problem - try again later
msg = 'Error processing request "%s": will try again later.' \
'\nError: "%s"' % (reqName, str(ex))
self.logger.info(msg)
self.logdb.post(reqName, msg, 'error')
continue
except Exception as ex:
# Log exception as it isnt a communication problem
msg = 'Error processing request "%s": will try again later.' \
'\nSee log for details.\nError: "%s"' % (reqName, str(ex))
self.logger.exception('Unknown error processing %s' % reqName)
self.logdb.post(reqName, msg, 'error')
continue
self.logger.info('%s units(s) queued for "%s"' % (units, reqName))
work += units
self.logger.info("%s element(s) added to open requests" % work)
return work
class ReqMgrService(TemplatedPage):
"""
Request Manager web service class
"""
def __init__(self, app, config, mount):
self.base = config.base
self.rootdir = '/'.join(WMCore.__file__.split('/')[:-1])
if config and not isinstance(config, dict):
web_config = config.dictionary_()
if not config:
web_config = {'base': self.base}
TemplatedPage.__init__(self, web_config)
imgdir = os.environ.get('RM_IMAGESPATH', os.getcwd()+'/images')
self.imgdir = web_config.get('imgdir', imgdir)
cssdir = os.environ.get('RM_CSSPATH', os.getcwd()+'/css')
self.cssdir = web_config.get('cssdir', cssdir)
jsdir = os.environ.get('RM_JSPATH', os.getcwd()+'/js')
self.jsdir = web_config.get('jsdir', jsdir)
spdir = os.environ.get('RM_SPECPATH', os.getcwd()+'/specs')
self.spdir = web_config.get('spdir', spdir)
# read scripts area and initialize data-ops scripts
self.sdir = os.environ.get('RM_SCRIPTS', os.getcwd()+'/scripts')
self.sdir = web_config.get('sdir', self.sdir)
self.sdict_thr = web_config.get('sdict_thr', 600) # put reasonable 10 min interval
self.sdict = {'ts':time.time()} # placeholder for data-ops scripts
self.update_scripts(force=True)
# To be filled at run time
self.cssmap = {}
self.jsmap = {}
self.imgmap = {}
self.yuimap = {}
std_specs_dir = os.path.join(self.rootdir, 'WMSpec/StdSpecs')
self.std_specs = spec_list(std_specs_dir, 'WMSpec.StdSpecs')
self.std_specs.sort()
# Update CherryPy configuration
mime_types = ['text/css']
mime_types += ['application/javascript', 'text/javascript',
'application/x-javascript', 'text/x-javascript']
cherryconf.update({'tools.encode.on': True,
'tools.gzip.on': True,
'tools.gzip.mime_types': mime_types,
})
self._cache = {}
# initialize rest API
statedir = '/tmp'
app = RESTMain(config, statedir) # REST application
mount = '/rest' # mount point for cherrypy service
api = RestApiHub(app, config.reqmgr, mount)
# initialize access to reqmgr2 APIs
self.reqmgr = ReqMgr(config.reqmgr.reqmgr2_url)
# only gets current view (This might cause to reponse time much longer,
# If upto date view is not needed overwrite Fale)
self.reqmgr._noStale = True
# admin helpers
self.admin_info = Info(app, api, config.reqmgr, mount=mount+'/info')
self.admin_group = Group(app, api, config.reqmgr, mount=mount+'/group')
self.admin_team = Team(app, api, config.reqmgr, mount=mount+'/team')
# get fields which we'll use in templates
cdict = config.reqmgr.dictionary_()
self.couch_url = cdict.get('couch_host', '')
self.couch_dbname = cdict.get('couch_reqmgr_db', '')
self.couch_wdbname = cdict.get('couch_workload_summary_db', '')
self.acdc_url = cdict.get('acdc_host', '')
self.acdc_dbname = cdict.get('acdc_db', '')
self.configcache_url = cdict.get('couch_config_cache_url', self.couch_url)
self.dbs_url = cdict.get('dbs_url', '')
self.dqm_url = cdict.get('dqm_url', '')
self.sw_ver = cdict.get('default_sw_version', 'CMSSW_5_2_5')
self.sw_arch = cdict.get('default_sw_scramarch', 'slc5_amd64_gcc434')
def user(self):
"""
Return user name associated with this instance.
"""
try:
return cherrypy.request.user['login']
except:
return 'testuser'
def user_dn(self):
"Return user DN"
try:
return cherrypy.request.user['dn']
except:
return '/CN/bla/foo'
def update_scripts(self, force=False):
"Update scripts dict"
if force or abs(time.time()-self.sdict['ts']) > self.sdict_thr:
for item in os.listdir(self.sdir):
with open(os.path.join(self.sdir, item), 'r') as istream:
self.sdict[item.split('.')[0]] = istream.read()
self.sdict['ts'] = time.time()
def abs_page(self, tmpl, content):
"""generate abstract page"""
menu = self.templatepage('menu', menus=menus(), tmpl=tmpl)
body = self.templatepage('generic', menu=menu, content=content)
page = self.templatepage('main', content=body, user=self.user())
return page
def page(self, content):
"""
Provide page wrapped with top/bottom templates.
"""
return self.templatepage('main', content=content)
def error(self, content):
"Generate common error page"
content = self.templatepage('error', content=content)
return self.abs_page('error', content)
@expose
def index(self, **kwds):
"""Main page"""
content = self.templatepage('index', requests=ACTIVE_STATUS, rdict=REQUEST_STATE_TRANSITION)
return self.abs_page('main', content)
@expose
def home(self, **kwds):
"""Main page"""
return self.index(**kwds)
### Admin actions ###
@expose
def admin(self, **kwds):
"""admin page"""
print "\n### ADMIN PAGE"
rows = self.admin_info.get()
print "rows", [r for r in rows]
content = self.templatepage('admin')
return self.abs_page('admin', content)
@expose
def add_user(self, **kwds):
"""add_user action"""
rid = genid(kwds)
status = "ok" # chagne to whatever it would be
content = self.templatepage('confirm', ticket=rid, user=self.user(), status=status)
return self.abs_page('admin', content)
@expose
def add_group(self, **kwds):
"""add_group action"""
rows = self.admin_group.get()
print "\n### GROUPS", [r for r in rows]
rid = genid(kwds)
status = "ok" # chagne to whatever it would be
content = self.templatepage('confirm', ticket=rid, user=self.user(), status=status)
return self.abs_page('admin', content)
@expose
def add_team(self, **kwds):
"""add_team action"""
rows = self.admin_team.get()
print "\n### TEAMS", kwds, [r for r in rows]
print "request to add", kwds
rid = genid(kwds)
status = "ok" # chagne to whatever it would be
content = self.templatepage('confirm', ticket=rid, user=self.user(), status=status)
return self.abs_page('admin', content)
### Request actions ###
@expose
@checkargs(['status', 'sort'])
def assign(self, **kwds):
"""assign page"""
if not kwds:
kwds = {}
if 'status' not in kwds:
kwds.update({'status': 'assignment-approved'})
docs = []
attrs = ['RequestName', 'RequestDate', 'Group', 'Requestor', 'RequestStatus']
data = self.reqmgr.getRequestByStatus(statusList=[kwds['status']])
for key, val in data.items():
docs.append(request_attr(val, attrs))
sortby = kwds.get('sort', 'status')
docs = [r for r in sort(docs, sortby)]
misc_json = {'CMSSW Releases':releases(),
'CMSSW architectures':architectures(),
'SubscriptionPriority':['Low', 'Normal', 'High'],
'CustodialSubType':['Move', 'Replica'],
'NonCustodialSubType':['Move', 'Replica'],
'MinMergeSize':2147483648,
'MaxMergeSize':4294967296,
'MaxMergeEvents':50000,
'MaxRSS':20411724,
'MaxVSize':20411724,
'SoftTimeout':129600,
'GracePeriod':300,
'BlockCloseMaxWaitTime':66400,
'BlockCloseMaxFiles':500,
'BlockCloseMaxEvents':250000000,
'BlockCloseMaxSize':5000000000000,
'AcquisitionEra':'',
'ProcessingVersion':1,
'ProcessingString':'',
'MergedLFNBase':lfn_bases(),
'UnmergedLFNBase':lfn_unmerged_bases(),}
filter_sort = self.templatepage('filter_sort')
content = self.templatepage('assign', sort=sortby,
filter_sort_table=filter_sort,
sites=sites(),
site_white_list=site_white_list(),
site_black_list=site_black_list(),
user=self.user(), user_dn=self.user_dn(), requests=docs,
misc_table=json2table(misc_json, web_ui_names()),
misc_json=json2form(misc_json, indent=2, keep_first_value=True))
return self.abs_page('assign', content)
@expose
@checkargs(['status', 'sort'])
def approve(self, **kwds):
"""
Approve page: get list of request associated with user DN.
Fetch their status list from ReqMgr and display if requests
were seen by data-ops.
"""
if not kwds:
kwds = {}
if 'status' not in kwds:
kwds.update({'status': 'new'})
kwds.update({'_nostale':True})
docs = []
attrs = ['RequestName', 'RequestDate', 'Group', 'Requestor', 'RequestStatus']
data = self.reqmgr.getRequestByStatus(statusList=[kwds['status']])
for key, val in data.items():
docs.append(request_attr(val, attrs))
sortby = kwds.get('sort', 'status')
docs = [r for r in sort(docs, sortby)]
filter_sort = self.templatepage('filter_sort')
content = self.templatepage('approve', requests=docs, date=tstamp(),
sort=sortby, filter_sort_table=filter_sort)
return self.abs_page('approve', content)
@expose
def create(self, **kwds):
"""create page"""
# get list of standard specs from WMCore and new ones from local area
#loc_specs_dir = os.path.join(self.spdir, 'Specs') # local specs
#loc_specs = spec_list(loc_specs_dir, 'Specs')
#all_specs = list(set(self.std_specs + loc_specs))
#all_specs.sort()
all_specs = self.std_specs
spec = kwds.get('form', '')
if not spec:
spec = self.std_specs[0]
# make spec first in all_specs list
if spec in all_specs:
all_specs.remove(spec)
all_specs = [spec] + all_specs
jsondata = get_request_template_from_type(spec)
# create templatized page out of provided forms
self.update_scripts()
content = self.templatepage('create', table=json2table(jsondata, web_ui_names()),
jsondata=json2form(jsondata, indent=2, keep_first_value=True), name=spec,
scripts=[s for s in self.sdict.keys() if s!='ts'],
specs=all_specs)
return self.abs_page('create', content)
def generate_objs(self, script, jsondict):
"""Generate objects from givem JSON template"""
self.update_scripts()
code = self.sdict.get(script, '')
if code.find('def genobjs(jsondict)') == -1:
return self.error("Improper python snippet, your code should start with <b>def genobjs(jsondict)</b> function")
exec(code) # code snippet must starts with genobjs function
return [r for r in genobjs(jsondict)]
@expose
def fetch(self, rid, **kwds):
"Fetch document for given id"
rid = rid.replace('request-', '')
doc = self.reqmgr.getRequestByNames(rid)
transitions = []
if len(doc) == 1:
try:
doc = doc[rid]
except:
pass
name = doc.get('RequestName', 'NA')
title = 'Request %s' % name
status = doc.get('RequestStatus', '')
transitions = REQUEST_STATE_TRANSITION.get(status, [])
if status in transitions:
transitions.remove(status)
content = self.templatepage('doc', title=title, status=status, name=name,
table=json2table(doc, web_ui_names()),
jsondata=json2form(doc, indent=2, keep_first_value=False),
transitions=transitions)
elif len(doc) > 1:
jsondata = [pprint.pformat(d) for d in doc]
content = self.templatepage('doc', title='Series of docs: %s' % rid,
table="", jsondata=jsondata,
transitions=transitions)
else:
doc = 'No request found for name=%s' % rid
return self.abs_page('request', content)
@expose
def requests(self, **kwds):
"""Page showing requests"""
if not kwds:
kwds = {}
if 'status' not in kwds:
kwds.update({'status': 'acquired'})
results = self.reqmgr.getRequestByStatus(kwds['status'])
docs = []
for key, doc in results.items():
docs.append(request_attr(doc))
sortby = kwds.get('sort', 'status')
docs = [r for r in sort(docs, sortby)]
filter_sort = self.templatepage('filter_sort')
content = self.templatepage('requests', requests=docs, sort=sortby,
status=kwds['status'], filter_sort_table=filter_sort)
return self.abs_page('requests', content)
@expose
def request(self, **kwargs):
"Get data example and expose it as json"
dataset = kwargs.get('uinput', '')
if not dataset:
return {'error':'no input dataset'}
url = 'https://cmsweb.cern.ch/reqmgr/rest/outputdataset/%s' % dataset
params = {}
headers = {'Accept': 'application/json;text/json'}
wdata = getdata(url, params)
wdict = dict(date=time.ctime(), team='Team-A', status='Running', ID=genid(wdata))
winfo = self.templatepage('workflow', wdict=wdict,
dataset=dataset, code=pprint.pformat(wdata))
content = self.templatepage('search', content=winfo)
return self.abs_page('request', content)
@expose
def batch(self, **kwds):
"""batch page"""
# TODO: we need a template for batch attributes
# and read it from separate area, like DASMaps
name = kwds.get('name', '')
batch = {}
if name:
# batch = self.reqmgr.getBatchesByName(name)
batch = {'Name':'Batch1', 'Description': 'Bla-bla', 'Creator':'valya', 'Group':'test',
'Workflows':['workflow1', 'workflow2'],
'Attributes':{'HeavyIon':['true', 'false']}}
attributes = batch.get('Attributes', {})
workflows = batch.get('Workflows', [])
description = batch.get('Description', '')
creator = batch.get('Creator', self.user_dn())
content = self.templatepage('batch', name=name,
attributes=json2table(attributes, web_ui_names()),
workflows=workflows, creator=creator,
description=description)
return self.abs_page('batch', content)
@expose
def batches(self, **kwds):
"""Page showing batches"""
if not kwds:
kwds = {}
if 'name' not in kwds:
kwds.update({'name': ''})
sortby = kwds.get('sort', 'name')
# results = self.reqmgr.getBatchesByName(kwds['name'])
results = [
{'Name':'Batch1', 'Description': 'Bla-bla', 'Creator':'valya', 'Group':'test',
'Workflows':['workflow1', 'workflow2'],
'Date': 'Fri Feb 13 10:36:41 EST 2015',
'Attributes':{'HeavyIon':['true', 'false']}},
{'Name':'Batch2', 'Description': 'lksdjflksjdf', 'Creator':'valya', 'Group':'test',
'Workflows':['workflow1', 'workflow2'],
'Date': 'Fri Feb 10 10:36:41 EST 2015',
'Attributes':{'HeavyIon':['true', 'false']}},
]
docs = [r for r in sort(results, sortby)]
filter_sort = self.templatepage('filter_sort')
content = self.templatepage('batches', batches=docs, sort=sortby,
filter_sort_table=filter_sort)
return self.abs_page('batches', content)
### Aux methods ###
@expose
def put_request(self, *args, **kwds):
"PUT request callback to reqmgr server, should be used in AJAX"
reqname = kwds.get('RequestName', '')
status = kwds.get('RequestStatus', '')
if not reqname:
msg = 'Unable to update request status, empty request name'
raise cherrypy.HTTPError(406, msg)
if not status:
msg = 'Unable to update request status, empty status value'
raise cherrypy.HTTPError(406, msg)
return self.reqmgr.updateRequestStatus(reqname, status)
@expose
def images(self, *args, **kwargs):
"""
Serve static images.
"""
args = list(args)
self.check_scripts(args, self.imgmap, self.imgdir)
mime_types = ['*/*', 'image/gif', 'image/png',
'image/jpg', 'image/jpeg']
accepts = cherrypy.request.headers.elements('Accept')
for accept in accepts:
if accept.value in mime_types and len(args) == 1 \
and args[0] in self.imgmap:
image = self.imgmap[args[0]]
# use image extension to pass correct content type
ctype = 'image/%s' % image.split('.')[-1]
cherrypy.response.headers['Content-type'] = ctype
return serve_file(image, content_type=ctype)
def serve(self, kwds, imap, idir, datatype='', minimize=False):
"Serve files for high level APIs (yui/css/js)"
args = []
for key, val in kwds.items():
if key == 'f': # we only look-up files from given kwds dict
if isinstance(val, list):
args += val
else:
args.append(val)
scripts = self.check_scripts(args, imap, idir)
return self.serve_files(args, scripts, imap, datatype, minimize)
@exposecss
@tools.gzip()
def css(self, **kwargs):
"""
Serve provided CSS files. They can be passed as
f=file1.css&f=file2.css
"""
resource = kwargs.get('resource', 'css')
if resource == 'css':
return self.serve(kwargs, self.cssmap, self.cssdir, 'css', True)
@exposejs
@tools.gzip()
def js(self, **kwargs):
"""
Serve provided JS scripts. They can be passed as
f=file1.js&f=file2.js with optional resource parameter
to speficy type of JS files, e.g. resource=yui.
"""
resource = kwargs.get('resource', 'js')
if resource == 'js':
return self.serve(kwargs, self.jsmap, self.jsdir)
def serve_files(self, args, scripts, resource, datatype='', minimize=False):
"""
Return asked set of files for JS, YUI, CSS.
"""
idx = "-".join(scripts)
if idx not in self._cache.keys():
data = ''
if datatype == 'css':
data = '@CHARSET "UTF-8";'
for script in args:
path = os.path.join(sys.path[0], resource[script])
path = os.path.normpath(path)
ifile = open(path)
data = "\n".join ([data, ifile.read().\
replace('@CHARSET "UTF-8";', '')])
ifile.close()
if datatype == 'css':
set_headers("text/css")
if minimize:
self._cache[idx] = minify(data)
else:
self._cache[idx] = data
return self._cache[idx]
def check_scripts(self, scripts, resource, path):
"""
Check a script is known to the resource map
and that the script actually exists
"""
for script in scripts:
if script not in resource.keys():
spath = os.path.normpath(os.path.join(path, script))
if os.path.isfile(spath):
resource.update({script: spath})
return scripts
class ReqMgrService(TemplatedPage):
"""
Request Manager web service class
"""
def __init__(self, app, config, mount):
self.base = config.base
self.rootdir = '/'.join(WMCore.__file__.split('/')[:-1])
if config and not isinstance(config, dict):
web_config = config.dictionary_()
if not config:
web_config = {'base': self.base}
TemplatedPage.__init__(self, web_config)
imgdir = os.environ.get('RM_IMAGESPATH', os.getcwd() + '/images')
self.imgdir = web_config.get('imgdir', imgdir)
cssdir = os.environ.get('RM_CSSPATH', os.getcwd() + '/css')
self.cssdir = web_config.get('cssdir', cssdir)
jsdir = os.environ.get('RM_JSPATH', os.getcwd() + '/js')
self.jsdir = web_config.get('jsdir', jsdir)
spdir = os.environ.get('RM_SPECPATH', os.getcwd() + '/specs')
self.spdir = web_config.get('spdir', spdir)
# read scripts area and initialize data-ops scripts
self.sdir = os.environ.get('RM_SCRIPTS', os.getcwd() + '/scripts')
self.sdir = web_config.get('sdir', self.sdir)
self.sdict_thr = web_config.get('sdict_thr',
600) # put reasonable 10 min interval
self.sdict = {'ts': time.time()} # placeholder for data-ops scripts
self.update_scripts(force=True)
# To be filled at run time
self.cssmap = {}
self.jsmap = {}
self.imgmap = {}
self.yuimap = {}
std_specs_dir = os.path.join(self.rootdir, 'WMSpec/StdSpecs')
self.std_specs = spec_list(std_specs_dir)
self.std_specs.sort()
# Update CherryPy configuration
mime_types = ['text/css']
mime_types += [
'application/javascript', 'text/javascript',
'application/x-javascript', 'text/x-javascript'
]
cherryconf.update({
'tools.encode.on': True,
'tools.gzip.on': True,
'tools.gzip.mime_types': mime_types,
})
self._cache = {}
# initialize access to reqmgr2 APIs
self.reqmgr_url = config.reqmgr.reqmgr2_url
self.reqmgr = ReqMgr(self.reqmgr_url)
# only gets current view (This might cause to reponse time much longer,
# If upto date view is not needed overwrite Fale)
self.reqmgr._noStale = True
# get fields which we'll use in templates
cdict = config.reqmgr.dictionary_()
self.couch_url = cdict.get('couch_host', '')
self.couch_dbname = cdict.get('couch_reqmgr_db', '')
self.couch_wdbname = cdict.get('couch_workload_summary_db', '')
self.acdc_url = cdict.get('acdc_host', '')
self.acdc_dbname = cdict.get('acdc_db', '')
self.configcache_url = cdict.get('couch_config_cache_url',
self.couch_url)
self.dbs_url = cdict.get('dbs_url', '')
self.dqm_url = cdict.get('dqm_url', '')
self.sw_ver = cdict.get('default_sw_version', 'CMSSW_7_6_1')
self.sw_arch = cdict.get('default_sw_scramarch', 'slc6_amd64_gcc493')
# LogDB holder
centralurl = cdict.get("central_logdb_url", "")
identifier = cdict.get("log_reporter", "reqmgr2")
self.logdb = LogDB(centralurl, identifier)
# local team cache which will request data from wmstats
base, uri = self.reqmgr_url.split('://')
base_url = '%s://%s' % (base, uri.split('/')[0])
self.wmstatsurl = cdict.get('wmstats_url',
'%s/wmstatsserver' % base_url)
if not self.wmstatsurl:
raise Exception(
'ReqMgr2 configuration file does not provide wmstats url')
self.team_cache = []
# fetch assignment arguments specification from StdBase
self.assignArgs = StdBase().getWorkloadAssignArgs()
self.assignArgs = {
key: val['default']
for key, val in self.assignArgs.items()
}
def getTeams(self):
"Helper function to get teams from wmstats or local cache"
teams = self.team_cache
url = '%s/data/teams' % self.wmstatsurl
params = {}
headers = {'Accept': 'application/json'}
try:
data = getdata(url, params, headers)
if 'error' in data:
print("WARNING: fail to get teams from %s" % url)
print(data)
teams = data.get('result', [])
self.team_cache = teams
except Exception as exp:
print("WARNING: fail to get teams from %s" % url)
print(str(exp))
return teams
def update_scripts(self, force=False):
"Update scripts dict"
if force or abs(time.time() - self.sdict['ts']) > self.sdict_thr:
for item in os.listdir(self.sdir):
with open(os.path.join(self.sdir, item), 'r') as istream:
self.sdict[item.split('.')[0]] = istream.read()
self.sdict['ts'] = time.time()
def abs_page(self, tmpl, content):
"""generate abstract page"""
menu = self.templatepage('menu', menus=menus(), tmpl=tmpl)
body = self.templatepage('generic', menu=menu, content=content)
page = self.templatepage('main', content=body, user=user())
return page
def page(self, content):
"""
Provide page wrapped with top/bottom templates.
"""
return self.templatepage('main', content=content)
def error(self, content):
"Generate common error page"
content = self.templatepage('error', content=content)
return self.abs_page('error', content)
@expose
def index(self):
"""Main page"""
content = self.templatepage('index',
requests=ACTIVE_STATUS,
rdict=REQUEST_STATE_TRANSITION)
return self.abs_page('main', content)
@expose
def home(self, **kwds):
"""Main page"""
return self.index(**kwds)
### Request actions ###
@expose
@checkargs(['status', 'sort'])
def assign(self, **kwds):
"""assign page"""
if not kwds:
kwds = {}
if 'status' not in kwds:
kwds.update({'status': 'assignment-approved'})
docs = []
attrs = [
'RequestName', 'RequestDate', 'Group', 'Requestor', 'RequestStatus'
]
dataResult = self.reqmgr.getRequestByStatus(
statusList=[kwds['status']])
for data in dataResult:
for val in data.values():
docs.append(request_attr(val, attrs))
sortby = kwds.get('sort', 'status')
docs = [r for r in sort(docs, sortby)]
assignDict = deepcopy(self.assignArgs)
assignDict.update(getPropValueMap())
assignDict['Team'] = self.getTeams()
filter_sort = self.templatepage('filter_sort')
content = self.templatepage('assign',
sort=sortby,
filter_sort_table=filter_sort,
sites=SITE_CACHE.getData(),
site_white_list=site_white_list(),
site_black_list=site_black_list(),
user=user(),
user_dn=user_dn(),
requests=toString(docs),
misc_table=json2table(
assignDict, web_ui_names(),
"all_attributes"),
misc_json=json2form(assignDict,
indent=2,
keep_first_value=True))
return self.abs_page('assign', content)
@expose
@checkargs(['status', 'sort'])
def approve(self, **kwds):
"""
Approve page: get list of request associated with user DN.
Fetch their status list from ReqMgr and display if requests
were seen by data-ops.
"""
if not kwds:
kwds = {}
if 'status' not in kwds:
kwds.update({'status': 'new'})
kwds.update({'_nostale': True})
docs = []
attrs = [
'RequestName', 'RequestDate', 'Group', 'Requestor',
'RequestStatus', 'Campaign'
]
dataResult = self.reqmgr.getRequestByStatus(
statusList=[kwds['status']])
for data in dataResult:
for val in data.values():
docs.append(request_attr(val, attrs))
sortby = kwds.get('sort', 'status')
docs = [r for r in sort(docs, sortby)]
filter_sort = self.templatepage('filter_sort')
content = self.templatepage('approve',
requests=toString(docs),
date=tstamp(),
sort=sortby,
filter_sort_table=filter_sort,
gen_color=gen_color)
return self.abs_page('approve', content)
@expose
def create(self, **kwds):
"""create page"""
# get list of standard specs from WMCore and new ones from local area
# loc_specs_dir = os.path.join(self.spdir, 'Specs') # local specs
# loc_specs = spec_list(loc_specs_dir, 'Specs')
# all_specs = list(set(self.std_specs + loc_specs))
# all_specs.sort()
all_specs = list(self.std_specs)
spec = kwds.get('form', '')
if not spec:
spec = self.std_specs[0]
# make spec first in all_specs list
if spec in all_specs:
all_specs.remove(spec)
all_specs = [spec] + all_specs
jsondata = get_request_template_from_type(spec)
# create templatized page out of provided forms
self.update_scripts()
content = self.templatepage(
'create',
table=json2table(jsondata, web_ui_names(), jsondata),
jsondata=json2form(jsondata, indent=2, keep_first_value=True),
name=spec,
scripts=[s for s in self.sdict.keys() if s != 'ts'],
specs=all_specs)
return self.abs_page('create', content)
def generate_objs(self, script, jsondict):
"""Generate objects from givem JSON template"""
self.update_scripts()
code = self.sdict.get(script, '')
if code.find('def genobjs(jsondict)') == -1:
return self.error(
"Improper python snippet, your code should start with <b>def genobjs(jsondict)</b> function"
)
exec(code) # code snippet must starts with genobjs function
return [r for r in genobjs(jsondict)]
@expose
def config(self, name):
"Fetch config for given request name"
result = self.reqmgr.getConfig(name)
if len(result) == 1:
result = result[0]
else:
result = 'Configuration not found for: %s' % name
return result.replace('\n', '<br/>')
@expose
def fetch(self, rid):
"Fetch document for given id"
rid = rid.replace('request-', '')
doc = self.reqmgr.getRequestByNames(rid)
transitions = []
tst = time.time()
# get request tasks
tasks = self.reqmgr.getRequestTasks(rid)
if len(doc) == 1:
try:
doc = doc[0][rid]
except:
pass
name = doc.get('RequestName', 'NA')
title = 'Request %s' % name
status = doc.get('RequestStatus', '')
transitions = REQUEST_STATE_TRANSITION.get(status, [])
if status in transitions:
transitions.remove(status)
visible_attrs = get_modifiable_properties(status)
filterout_attrs = get_protected_properties()
# extend filterout list with "RequestStatus" since it is passed separately
filterout_attrs.append("RequestStatus")
for key, val in self.assignArgs.items():
if not doc.get(key):
doc[key] = val
if visible_attrs == "all_attributes":
filteredDoc = doc
for prop in filterout_attrs:
if prop in filteredDoc:
del filteredDoc[prop]
else:
filteredDoc = {}
for prop in visible_attrs:
filteredDoc[prop] = doc.get(prop, "")
propValueMap = getPropValueMap()
propValueMap['Team'] = self.getTeams()
selected = {}
for prop in propValueMap:
if prop in filteredDoc:
filteredDoc[prop], selected[prop] = reorder_list(
propValueMap[prop], filteredDoc[prop])
content = self.templatepage(
'doc',
title=title,
status=status,
name=name,
rid=rid,
tasks=json2form(tasks, indent=2, keep_first_value=False),
table=json2table(filteredDoc, web_ui_names(), visible_attrs,
selected),
jsondata=json2form(doc, indent=2, keep_first_value=False),
doc=json.dumps(doc),
time=time,
tasksConfigs=tasks_configs(doc, html=True),
sTransition=state_transition(doc),
pTransition=priority_transition(doc),
transitions=transitions,
ts=tst,
user=user(),
userdn=user_dn())
elif len(doc) > 1:
jsondata = [pprint.pformat(d) for d in doc]
content = self.templatepage('doc',
title='Series of docs: %s' % rid,
table="",
jsondata=jsondata,
time=time,
tasksConfigs=tasks_configs(doc,
html=True),
sTransition=state_transition(doc),
pTransition=priority_transition(doc),
transitions=transitions,
ts=tst,
user=user(),
userdn=user_dn())
else:
doc = 'No request found for name=%s' % rid
return self.abs_page('request', content)
@expose
def record2logdb(self, **kwds):
"""LogDB submission page"""
print(kwds)
request = kwds['request']
msg = kwds['message']
self.logdb.post(request, msg)
msg = '<h6>Confirmation</h6>Your request has been entered to LogDB.'
return self.abs_page('generic', msg)
@expose
def requests(self, **kwds):
"""Page showing requests"""
if not kwds:
kwds = {}
if 'status' not in kwds:
kwds.update({'status': 'acquired'})
dataResult = self.reqmgr.getRequestByStatus(kwds['status'])
attrs = [
'RequestName', 'RequestDate', 'Group', 'Requestor',
'RequestStatus', 'Campaign'
]
docs = []
for data in dataResult:
for doc in data.values():
docs.append(request_attr(doc, attrs))
sortby = kwds.get('sort', 'status')
docs = [r for r in sort(docs, sortby)]
filter_sort = self.templatepage('filter_sort')
content = self.templatepage('requests',
requests=toString(docs),
sort=sortby,
status=kwds['status'],
filter_sort_table=filter_sort)
return self.abs_page('requests', content)
@expose
def request(self, **kwargs):
"Get data example and expose it as json"
dataset = kwargs.get('uinput', '')
if not dataset:
return {'error': 'no input dataset'}
url = 'https://cmsweb.cern.ch/reqmgr2/data/request?outputdataset=%s' % dataset
params = {}
headers = {'Accept': 'application/json'}
wdata = getdata(url, params, headers)
wdict = dict(date=time.ctime(),
team='Team-A',
status='Running',
ID=genid(wdata))
winfo = self.templatepage('workflow',
wdict=wdict,
dataset=dataset,
code=pprint.pformat(wdata))
content = self.templatepage('search', content=winfo)
return self.abs_page('request', content)
@expose
def batch(self, **kwds):
"""batch page"""
# TODO: we need a template for batch attributes
# and read it from separate area, like DASMaps
name = kwds.get('name', '')
batch = {}
if name:
# batch = self.reqmgr.getBatchesByName(name)
batch = {
'Name': 'Batch1',
'Description': 'Bla-bla',
'Creator': 'valya',
'Group': 'test',
'Workflows': ['workflow1', 'workflow2'],
'Attributes': {
'HeavyIon': ['true', 'false']
}
}
attributes = batch.get('Attributes', {})
workflows = batch.get('Workflows', [])
description = batch.get('Description', '')
creator = batch.get('Creator', user_dn())
content = self.templatepage('batch',
name=name,
attributes=json2table(
attributes, web_ui_names()),
workflows=workflows,
creator=creator,
description=description)
return self.abs_page('batch', content)
@expose
def batches(self, **kwds):
"""Page showing batches"""
if not kwds:
kwds = {}
if 'name' not in kwds:
kwds.update({'name': ''})
sortby = kwds.get('sort', 'name')
# results = self.reqmgr.getBatchesByName(kwds['name'])
results = [
{
'Name': 'Batch1',
'Description': 'Bla-bla',
'Creator': 'valya',
'Group': 'test',
'Workflows': ['workflow1', 'workflow2'],
'Date': 'Fri Feb 13 10:36:41 EST 2015',
'Attributes': {
'HeavyIon': ['true', 'false']
}
},
{
'Name': 'Batch2',
'Description': 'lksdjflksjdf',
'Creator': 'valya',
'Group': 'test',
'Workflows': ['workflow1', 'workflow2'],
'Date': 'Fri Feb 10 10:36:41 EST 2015',
'Attributes': {
'HeavyIon': ['true', 'false']
}
},
]
docs = [r for r in sort(results, sortby)]
filter_sort = self.templatepage('filter_sort')
content = self.templatepage('batches',
batches=docs,
sort=sortby,
filter_sort_table=filter_sort)
return self.abs_page('batches', content)
### Aux methods ###
@expose
def put_request(self, **kwds):
"PUT request callback to reqmgr server, should be used in AJAX"
reqname = kwds.get('RequestName', '')
status = kwds.get('RequestStatus', '')
if not reqname:
msg = 'Unable to update request status, empty request name'
raise cherrypy.HTTPError(406, msg)
if not status:
msg = 'Unable to update request status, empty status value'
raise cherrypy.HTTPError(406, msg)
return self.reqmgr.updateRequestStatus(reqname, status)
@expose
def images(self, *args):
"""
Serve static images.
"""
args = list(args)
check_scripts(args, self.imgmap, self.imgdir)
mime_types = [
'*/*', 'image/gif', 'image/png', 'image/jpg', 'image/jpeg'
]
accepts = cherrypy.request.headers.elements('Accept')
for accept in accepts:
if accept.value in mime_types and len(args) == 1 \
and args[0] in self.imgmap:
image = self.imgmap[args[0]]
# use image extension to pass correct content type
ctype = 'image/%s' % image.split('.')[-1]
cherrypy.response.headers['Content-type'] = ctype
return serve_file(image, content_type=ctype)
def serve(self, kwds, imap, idir, datatype='', minimize=False):
"Serve files for high level APIs (yui/css/js)"
args = []
for key, val in kwds.items():
if key == 'f': # we only look-up files from given kwds dict
if isinstance(val, list):
args += val
else:
args.append(val)
scripts = check_scripts(args, imap, idir)
return self.serve_files(args, scripts, imap, datatype, minimize)
@exposecss
@tools.gzip()
def css(self, **kwargs):
"""
Serve provided CSS files. They can be passed as
f=file1.css&f=file2.css
"""
resource = kwargs.get('resource', 'css')
if resource == 'css':
return self.serve(kwargs, self.cssmap, self.cssdir, 'css', True)
@exposejs
@tools.gzip()
def js(self, **kwargs):
"""
Serve provided JS scripts. They can be passed as
f=file1.js&f=file2.js with optional resource parameter
to speficy type of JS files, e.g. resource=yui.
"""
resource = kwargs.get('resource', 'js')
if resource == 'js':
return self.serve(kwargs, self.jsmap, self.jsdir)
def serve_files(self,
args,
scripts,
resource,
datatype='',
minimize=False):
"""
Return asked set of files for JS, YUI, CSS.
"""
idx = "-".join(scripts)
if idx not in self._cache.keys():
data = ''
if datatype == 'css':
data = '@CHARSET "UTF-8";'
for script in args:
path = os.path.join(sys.path[0], resource[script])
path = os.path.normpath(path)
ifile = open(path)
data = "\n".join([data, ifile.read(). \
replace('@CHARSET "UTF-8";', '')])
ifile.close()
if datatype == 'css':
set_headers("text/css")
if minimize:
self._cache[idx] = minify(data)
else:
self._cache[idx] = data
return self._cache[idx]
class ReqMgrService(TemplatedPage):
"""
Request Manager web service class
"""
def __init__(self, app, config, mount):
self.base = config.base
self.rootdir = '/'.join(WMCore.__file__.split('/')[:-1])
if config and not isinstance(config, dict):
web_config = config.dictionary_()
if not config:
web_config = {'base': self.base}
TemplatedPage.__init__(self, web_config)
imgdir = os.environ.get('RM_IMAGESPATH', os.getcwd() + '/images')
self.imgdir = web_config.get('imgdir', imgdir)
cssdir = os.environ.get('RM_CSSPATH', os.getcwd() + '/css')
self.cssdir = web_config.get('cssdir', cssdir)
jsdir = os.environ.get('RM_JSPATH', os.getcwd() + '/js')
self.jsdir = web_config.get('jsdir', jsdir)
spdir = os.environ.get('RM_SPECPATH', os.getcwd() + '/specs')
self.spdir = web_config.get('spdir', spdir)
# read scripts area and initialize data-ops scripts
self.sdir = os.environ.get('RM_SCRIPTS', os.getcwd() + '/scripts')
self.sdir = web_config.get('sdir', self.sdir)
self.sdict_thr = web_config.get('sdict_thr', 600) # put reasonable 10 min interval
self.sdict = {'ts': time.time()} # placeholder for data-ops scripts
self.update_scripts(force=True)
# To be filled at run time
self.cssmap = {}
self.jsmap = {}
self.imgmap = {}
self.yuimap = {}
std_specs_dir = os.path.join(self.rootdir, 'WMSpec/StdSpecs')
self.std_specs = spec_list(std_specs_dir)
self.std_specs.sort()
# Update CherryPy configuration
mime_types = ['text/css']
mime_types += ['application/javascript', 'text/javascript',
'application/x-javascript', 'text/x-javascript']
cherryconf.update({'tools.encode.on': True,
'tools.gzip.on': True,
'tools.gzip.mime_types': mime_types,
})
self._cache = {}
# initialize access to reqmgr2 APIs
self.reqmgr_url = config.reqmgr.reqmgr2_url
self.reqmgr = ReqMgr(self.reqmgr_url)
# only gets current view (This might cause to reponse time much longer,
# If upto date view is not needed overwrite Fale)
self.reqmgr._noStale = True
# get fields which we'll use in templates
cdict = config.reqmgr.dictionary_()
self.couch_url = cdict.get('couch_host', '')
self.couch_dbname = cdict.get('couch_reqmgr_db', '')
self.couch_wdbname = cdict.get('couch_workload_summary_db', '')
self.acdc_url = cdict.get('acdc_host', '')
self.acdc_dbname = cdict.get('acdc_db', '')
self.configcache_url = cdict.get('couch_config_cache_url', self.couch_url)
self.dbs_url = cdict.get('dbs_url', '')
self.dqm_url = cdict.get('dqm_url', '')
self.sw_ver = cdict.get('default_sw_version', 'CMSSW_7_6_1')
self.sw_arch = cdict.get('default_sw_scramarch', 'slc6_amd64_gcc493')
# LogDB holder
centralurl = cdict.get("central_logdb_url", "")
identifier = cdict.get("log_reporter", "reqmgr2")
self.logdb = LogDB(centralurl, identifier)
# local team cache which will request data from wmstats
base, uri = self.reqmgr_url.split('://')
base_url = '%s://%s' % (base, uri.split('/')[0])
self.wmstatsurl = cdict.get('wmstats_url', '%s/wmstatsserver' % base_url)
if not self.wmstatsurl:
raise Exception('ReqMgr2 configuration file does not provide wmstats url')
self.team_cache = []
# fetch assignment arguments specification from StdBase
self.assignArgs = StdBase().getWorkloadAssignArgs()
self.assignArgs = {key: val['default'] for key, val in self.assignArgs.items()}
def getTeams(self):
"Helper function to get teams from wmstats or local cache"
teams = self.team_cache
url = '%s/data/teams' % self.wmstatsurl
params = {}
headers = {'Accept': 'application/json'}
try:
data = getdata(url, params, headers)
if 'error' in data:
print("WARNING: fail to get teams from %s" % url)
print(data)
teams = data.get('result', [])
self.team_cache = teams
except Exception as exp:
print("WARNING: fail to get teams from %s" % url)
print(str(exp))
return teams
def update_scripts(self, force=False):
"Update scripts dict"
if force or abs(time.time() - self.sdict['ts']) > self.sdict_thr:
for item in os.listdir(self.sdir):
with open(os.path.join(self.sdir, item), 'r') as istream:
self.sdict[item.split('.')[0]] = istream.read()
self.sdict['ts'] = time.time()
def abs_page(self, tmpl, content):
"""generate abstract page"""
menu = self.templatepage('menu', menus=menus(), tmpl=tmpl)
body = self.templatepage('generic', menu=menu, content=content)
page = self.templatepage('main', content=body, user=user())
return page
def page(self, content):
"""
Provide page wrapped with top/bottom templates.
"""
return self.templatepage('main', content=content)
def error(self, content):
"Generate common error page"
content = self.templatepage('error', content=content)
return self.abs_page('error', content)
@expose
def index(self):
"""Main page"""
content = self.templatepage('index', requests=ACTIVE_STATUS, rdict=REQUEST_STATE_TRANSITION)
return self.abs_page('main', content)
@expose
def home(self, **kwds):
"""Main page"""
return self.index(**kwds)
### Request actions ###
@expose
@checkargs(['status', 'sort'])
def assign(self, **kwds):
"""assign page"""
if not kwds:
kwds = {}
if 'status' not in kwds:
kwds.update({'status': 'assignment-approved'})
docs = []
attrs = ['RequestName', 'RequestDate', 'Group', 'Requestor', 'RequestStatus']
dataResult = self.reqmgr.getRequestByStatus(statusList=[kwds['status']])
for data in dataResult:
for val in data.values():
docs.append(request_attr(val, attrs))
sortby = kwds.get('sort', 'status')
docs = [r for r in sort(docs, sortby)]
assignDict = deepcopy(self.assignArgs)
assignDict.update(getPropValueMap())
assignDict['Team'] = self.getTeams()
filter_sort = self.templatepage('filter_sort')
content = self.templatepage('assign', sort=sortby,
filter_sort_table=filter_sort,
sites=SITE_CACHE.getData(),
site_white_list=site_white_list(),
site_black_list=site_black_list(),
user=user(), user_dn=user_dn(), requests=toString(docs),
misc_table=json2table(assignDict, web_ui_names(), "all_attributes"),
misc_json=json2form(assignDict, indent=2, keep_first_value=True))
return self.abs_page('assign', content)
@expose
@checkargs(['status', 'sort'])
def approve(self, **kwds):
"""
Approve page: get list of request associated with user DN.
Fetch their status list from ReqMgr and display if requests
were seen by data-ops.
"""
if not kwds:
kwds = {}
if 'status' not in kwds:
kwds.update({'status': 'new'})
kwds.update({'_nostale': True})
docs = []
attrs = ['RequestName', 'RequestDate', 'Group', 'Requestor', 'RequestStatus', 'Campaign']
dataResult = self.reqmgr.getRequestByStatus(statusList=[kwds['status']])
for data in dataResult:
for val in data.values():
docs.append(request_attr(val, attrs))
sortby = kwds.get('sort', 'status')
docs = [r for r in sort(docs, sortby)]
filter_sort = self.templatepage('filter_sort')
content = self.templatepage('approve', requests=toString(docs), date=tstamp(),
sort=sortby, filter_sort_table=filter_sort,
gen_color=gen_color)
return self.abs_page('approve', content)
@expose
def create(self, **kwds):
"""create page"""
# get list of standard specs from WMCore and new ones from local area
# loc_specs_dir = os.path.join(self.spdir, 'Specs') # local specs
# loc_specs = spec_list(loc_specs_dir, 'Specs')
# all_specs = list(set(self.std_specs + loc_specs))
# all_specs.sort()
all_specs = list(self.std_specs)
spec = kwds.get('form', '')
if not spec:
spec = self.std_specs[0]
# make spec first in all_specs list
if spec in all_specs:
all_specs.remove(spec)
all_specs = [spec] + all_specs
jsondata = get_request_template_from_type(spec)
# create templatized page out of provided forms
self.update_scripts()
content = self.templatepage('create', table=json2table(jsondata, web_ui_names(), jsondata),
jsondata=json2form(jsondata, indent=2, keep_first_value=True), name=spec,
scripts=[s for s in self.sdict.keys() if s != 'ts'],
specs=all_specs)
return self.abs_page('create', content)
def generate_objs(self, script, jsondict):
"""Generate objects from givem JSON template"""
self.update_scripts()
code = self.sdict.get(script, '')
if code.find('def genobjs(jsondict)') == -1:
return self.error(
"Improper python snippet, your code should start with <b>def genobjs(jsondict)</b> function")
exec (code) # code snippet must starts with genobjs function
return [r for r in genobjs(jsondict)]
@expose
def config(self, name):
"Fetch config for given request name"
result = self.reqmgr.getConfig(name)
if len(result) == 1:
result = result[0]
else:
result = 'Configuration not found for: %s' % name
return result.replace('\n', '<br/>')
@expose
def fetch(self, rid):
"Fetch document for given id"
rid = rid.replace('request-', '')
doc = self.reqmgr.getRequestByNames(rid)
transitions = []
tst = time.time()
# get request tasks
tasks = self.reqmgr.getRequestTasks(rid)
if len(doc) == 1:
try:
doc = doc[0][rid]
except:
pass
name = doc.get('RequestName', 'NA')
title = 'Request %s' % name
status = doc.get('RequestStatus', '')
transitions = REQUEST_STATE_TRANSITION.get(status, [])
if status in transitions:
transitions.remove(status)
visible_attrs = get_modifiable_properties(status)
filterout_attrs = get_protected_properties()
# extend filterout list with "RequestStatus" since it is passed separately
filterout_attrs.append("RequestStatus")
for key, val in self.assignArgs.items():
if not doc.get(key):
doc[key] = val
if visible_attrs == "all_attributes":
filteredDoc = doc
for prop in filterout_attrs:
if prop in filteredDoc:
del filteredDoc[prop]
else:
filteredDoc = {}
for prop in visible_attrs:
filteredDoc[prop] = doc.get(prop, "")
propValueMap = getPropValueMap()
propValueMap['Team'] = self.getTeams()
selected = {}
for prop in propValueMap:
if prop in filteredDoc:
filteredDoc[prop], selected[prop] = reorder_list(propValueMap[prop], filteredDoc[prop])
content = self.templatepage('doc', title=title, status=status, name=name, rid=rid,
tasks=json2form(tasks, indent=2, keep_first_value=False),
table=json2table(filteredDoc, web_ui_names(), visible_attrs, selected),
jsondata=json2form(doc, indent=2, keep_first_value=False),
doc=json.dumps(doc), time=time,
tasksConfigs=tasks_configs(doc, html=True),
sTransition=state_transition(doc),
pTransition=priority_transition(doc),
transitions=transitions, humanStates=REQUEST_HUMAN_STATES,
ts=tst, user=user(), userdn=user_dn())
elif len(doc) > 1:
jsondata = [pprint.pformat(d) for d in doc]
content = self.templatepage('doc', title='Series of docs: %s' % rid,
table="", jsondata=jsondata, time=time,
tasksConfigs=tasks_configs(doc, html=True),
sTransition=state_transition(doc),
pTransition=priority_transition(doc),
transitions=transitions, humanStates=REQUEST_HUMAN_STATES,
ts=tst, user=user(), userdn=user_dn())
else:
doc = 'No request found for name=%s' % rid
return self.abs_page('request', content)
@expose
def record2logdb(self, **kwds):
"""LogDB submission page"""
print(kwds)
request = kwds['request']
msg = kwds['message']
self.logdb.post(request, msg)
msg = '<h6>Confirmation</h6>Your request has been entered to LogDB.'
return self.abs_page('generic', msg)
@expose
def requests(self, **kwds):
"""Page showing requests"""
if not kwds:
kwds = {}
if 'status' not in kwds:
kwds.update({'status': 'acquired'})
dataResult = self.reqmgr.getRequestByStatus(kwds['status'])
attrs = ['RequestName', 'RequestDate', 'Group', 'Requestor', 'RequestStatus', 'Campaign']
docs = []
for data in dataResult:
for doc in data.values():
docs.append(request_attr(doc, attrs))
sortby = kwds.get('sort', 'status')
docs = [r for r in sort(docs, sortby)]
filter_sort = self.templatepage('filter_sort')
content = self.templatepage('requests', requests=toString(docs), sort=sortby,
status=kwds['status'], filter_sort_table=filter_sort)
return self.abs_page('requests', content)
@expose
def request(self, **kwargs):
"Get data example and expose it as json"
dataset = kwargs.get('uinput', '')
if not dataset:
return {'error': 'no input dataset'}
url = 'https://cmsweb.cern.ch/reqmgr2/data/request?outputdataset=%s' % dataset
params = {}
headers = {'Accept': 'application/json'}
wdata = getdata(url, params, headers)
wdict = dict(date=time.ctime(), team='Team-A', status='Running', ID=genid(wdata))
winfo = self.templatepage('workflow', wdict=wdict,
dataset=dataset, code=pprint.pformat(wdata))
content = self.templatepage('search', content=winfo)
return self.abs_page('request', content)
@expose
def batch(self, **kwds):
"""batch page"""
# TODO: we need a template for batch attributes
# and read it from separate area, like DASMaps
name = kwds.get('name', '')
batch = {}
if name:
# batch = self.reqmgr.getBatchesByName(name)
batch = {'Name': 'Batch1', 'Description': 'Bla-bla', 'Creator': 'valya', 'Group': 'test',
'Workflows': ['workflow1', 'workflow2'],
'Attributes': {'HeavyIon': ['true', 'false']}}
attributes = batch.get('Attributes', {})
workflows = batch.get('Workflows', [])
description = batch.get('Description', '')
creator = batch.get('Creator', user_dn())
content = self.templatepage('batch', name=name,
attributes=json2table(attributes, web_ui_names()),
workflows=workflows, creator=creator,
description=description)
return self.abs_page('batch', content)
@expose
def batches(self, **kwds):
"""Page showing batches"""
if not kwds:
kwds = {}
if 'name' not in kwds:
kwds.update({'name': ''})
sortby = kwds.get('sort', 'name')
# results = self.reqmgr.getBatchesByName(kwds['name'])
results = [
{'Name': 'Batch1', 'Description': 'Bla-bla', 'Creator': 'valya', 'Group': 'test',
'Workflows': ['workflow1', 'workflow2'],
'Date': 'Fri Feb 13 10:36:41 EST 2015',
'Attributes': {'HeavyIon': ['true', 'false']}},
{'Name': 'Batch2', 'Description': 'lksdjflksjdf', 'Creator': 'valya', 'Group': 'test',
'Workflows': ['workflow1', 'workflow2'],
'Date': 'Fri Feb 10 10:36:41 EST 2015',
'Attributes': {'HeavyIon': ['true', 'false']}},
]
docs = [r for r in sort(results, sortby)]
filter_sort = self.templatepage('filter_sort')
content = self.templatepage('batches', batches=docs, sort=sortby,
filter_sort_table=filter_sort)
return self.abs_page('batches', content)
### Aux methods ###
@expose
def put_request(self, **kwds):
"PUT request callback to reqmgr server, should be used in AJAX"
reqname = kwds.get('RequestName', '')
status = kwds.get('RequestStatus', '')
if not reqname:
msg = 'Unable to update request status, empty request name'
raise cherrypy.HTTPError(406, msg)
if not status:
msg = 'Unable to update request status, empty status value'
raise cherrypy.HTTPError(406, msg)
return self.reqmgr.updateRequestStatus(reqname, status)
@expose
def images(self, *args):
"""
Serve static images.
"""
args = list(args)
check_scripts(args, self.imgmap, self.imgdir)
mime_types = ['*/*', 'image/gif', 'image/png',
'image/jpg', 'image/jpeg']
accepts = cherrypy.request.headers.elements('Accept')
for accept in accepts:
if accept.value in mime_types and len(args) == 1 \
and args[0] in self.imgmap:
image = self.imgmap[args[0]]
# use image extension to pass correct content type
ctype = 'image/%s' % image.split('.')[-1]
cherrypy.response.headers['Content-type'] = ctype
return serve_file(image, content_type=ctype)
def serve(self, kwds, imap, idir, datatype='', minimize=False):
"Serve files for high level APIs (yui/css/js)"
args = []
for key, val in kwds.items():
if key == 'f': # we only look-up files from given kwds dict
if isinstance(val, list):
args += val
else:
args.append(val)
scripts = check_scripts(args, imap, idir)
return self.serve_files(args, scripts, imap, datatype, minimize)
@exposecss
@tools.gzip()
def css(self, **kwargs):
"""
Serve provided CSS files. They can be passed as
f=file1.css&f=file2.css
"""
resource = kwargs.get('resource', 'css')
if resource == 'css':
return self.serve(kwargs, self.cssmap, self.cssdir, 'css', True)
@exposejs
@tools.gzip()
def js(self, **kwargs):
"""
Serve provided JS scripts. They can be passed as
f=file1.js&f=file2.js with optional resource parameter
to speficy type of JS files, e.g. resource=yui.
"""
resource = kwargs.get('resource', 'js')
if resource == 'js':
return self.serve(kwargs, self.jsmap, self.jsdir)
def serve_files(self, args, scripts, resource, datatype='', minimize=False):
"""
Return asked set of files for JS, YUI, CSS.
"""
idx = "-".join(scripts)
if idx not in self._cache.keys():
data = ''
if datatype == 'css':
data = '@CHARSET "UTF-8";'
for script in args:
path = os.path.join(sys.path[0], resource[script])
path = os.path.normpath(path)
with open(path) as ifile:
data = "\n".join([data, ifile.read().replace('@CHARSET "UTF-8";', '')])
if datatype == 'css':
set_headers("text/css")
if minimize:
self._cache[idx] = minify(data)
else:
self._cache[idx] = data
return self._cache[idx]
class ReqMgrTest(RESTBaseUnitTestWithDBBackend):
"""
Test WorkQueue Service client
It will start WorkQueue RESTService
Server DB sets from environment variable.
Client DB sets from environment variable.
This checks whether DS call makes without error and return the results.
Not the correctness of functions. That will be tested in different module.
"""
def setFakeDN(self):
# put into ReqMgr auxiliary database under "software" document scram/cmsms
# which we'll need a little for request injection
#Warning: this assumes the same structure in jenkins wmcore_root/test
self.admin_header = getAuthHeader(self.test_authz_key.data,
ADMIN_PERMISSION)
self.create_header = getAuthHeader(self.test_authz_key.data,
CREATE_PERMISSION)
self.default_header = getAuthHeader(self.test_authz_key.data,
DEFAULT_PERMISSION)
self.assign_header = getAuthHeader(self.test_authz_key.data,
ASSIGN_PERMISSION)
self.default_status_header = getAuthHeader(self.test_authz_key.data,
DEFAULT_STATUS_PERMISSION)
def setUp(self):
self.setConfig(config)
self.setCouchDBs([(config.views.data.couch_reqmgr_db, "ReqMgr"),
(config.views.data.couch_reqmgr_aux_db, None)])
self.setSchemaModules([])
RESTBaseUnitTestWithDBBackend.setUp(self)
self.setFakeDN()
requestPath = os.path.join(getWMBASE(), "test", "data", "ReqMgr",
"requests", "DMWM")
rerecoFile = open(os.path.join(requestPath, "ReReco.json"), 'r')
rerecoArgs = json.load(rerecoFile)
self.rerecoCreateArgs = rerecoArgs["createRequest"]
self.rerecoAssignArgs = rerecoArgs["assignRequest"]
cmsswDoc = {"_id": "software"}
cmsswDoc[self.rerecoCreateArgs["ScramArch"]] = []
cmsswDoc[self.rerecoCreateArgs["ScramArch"]].append(
self.rerecoCreateArgs["CMSSWVersion"])
insertDataToCouch(os.getenv("COUCHURL"),
config.views.data.couch_reqmgr_aux_db, cmsswDoc)
self.reqSvc = ReqMgr(self.jsonSender["host"])
self.reqSvc._noStale = True
self.reqSvc['requests'].additionalHeaders = self.create_header
def tearDown(self):
RESTBaseUnitTestWithDBBackend.tearDown(self)
def testRequestSimpleCycle(self):
"""
test request cycle with one request without composite get condition.
post, get, put
"""
# test post method
response = self.reqSvc.insertRequests(self.rerecoCreateArgs)
self.assertEqual(len(response), 1)
requestName = response[0]['request']
## test get method
# get by name
response = self.reqSvc.getRequestByNames(requestName)
self.assertEqual(response[requestName]['RequestPriority'], 10000)
self.assertEqual(len(response), 1)
# get by status
response = self.reqSvc.getRequestByStatus('new')
self.assertEqual(len(response), 1)
print(response)
self.reqSvc.updateRequestStatus(requestName, 'assignment-approved')
response = self.reqSvc.getRequestByStatus('assignment-approved')
self.assertEqual(len(response), 1)
self.reqSvc.updateRequestProperty(
requestName, {
'RequestStatus': 'assigned',
"AcquisitionEra": "TEST_ERA",
"Team": "unittest",
"SiteWhitelist": ["T1_US_CBS"],
"SiteBlacklist": ["T1_US_FOX"]
})
response = self.reqSvc.getRequestByStatus('assignment-approved')
self.assertEqual(len(response), 0)
response = self.reqSvc.getRequestByStatus('assigned')
self.assertEqual(len(response), 1)
self.assertEqual(response.values()[0]["SiteWhitelist"], ["T1_US_CBS"])
self.reqSvc.updateRequestStats(
requestName, {
'total_jobs': 100,
'input_lumis': 100,
'input_events': 100,
'input_num_files': 100
})