def handleAnnounce(self, **kwargs):
""" Handler for announcing requests """
requests = self.requestNamesFromCheckboxes(kwargs)
# the distinction good/bad datasets was based upon contacting
# wrong DBS url so it was always giving rubbish,
# perhaps the entire page is just not used at all
datasets = []
goodDatasets = []
badDatasets = []
for requestName in requests:
WMCore.Lexicon.identifier(requestName)
Utilities.changeStatus(requestName,
'announced',
wmstatUrl=self.wmstatWriteURL,
acdcUrl=self.acdcURL)
datasets.extend(Utilities.getOutputForRequest(requestName))
for dataset in datasets:
# was needed for the DBS call to wrong DBS URL
# check code before 2013-04-04
#toks = dataset.split('/')
#data = {'primary_ds_name': toks[0], 'processed_ds_name': toks[1],
# 'data_tier_name': toks[2], 'is_dataset_valid': 1}
goodDatasets.append(dataset)
return self.templatepage("Announce",
requests=requests,
goodDatasets=goodDatasets,
badDatasets=badDatasets)
def putRequest(self, requestName=None, status=None, priority=None):
request = None
if requestName:
request = self.findRequest(requestName)
if request == None:
# Create a new request, with a JSON-encoded schema that is
# sent in the body of the HTTP request
body = cherrypy.request.body.read()
reqInputArgs = Utilities.unidecode(JsonWrapper.loads(body))
reqInputArgs.setdefault(
'CouchURL', Utilities.removePasswordFromUrl(self.couchUrl))
reqInputArgs.setdefault('CouchDBName', self.configDBName)
try:
self.info(
"Creating a request for: '%s'\n\tworkloadDB: '%s'\n\twmstatUrl: "
"'%s' ..." %
(reqInputArgs, self.workloadDBName,
Utilities.removePasswordFromUrl(self.wmstatWriteURL)))
request = Utilities.makeRequest(self, reqInputArgs,
self.couchUrl,
self.workloadDBName,
self.wmstatWriteURL)
except cherrypy.HTTPError as ex:
self.error("Create request failed, reason: %s" % ex)
# Assume that this is a valid HTTPError
raise
except (WMException, Exception) as ex:
# TODO problem not to expose logs to the client
# e.g. on ConfigCacheID not found, the entire CouchDB traceback is sent in ex_message
self.error("Create request failed, reason: %s" % ex)
if hasattr(ex, "name"):
detail = ex.name
else:
detail = "check logs."
msg = "Create request failed, %s" % detail
raise cherrypy.HTTPError(400, msg)
self.info("Request '%s' created." % request['RequestName'])
# see if status & priority need to be upgraded
if status != None:
# forbid assignment here
if status == 'assigned':
raise cherrypy.HTTPError(
403,
"Cannot change status without a team. Please use PUT /reqmgr/reqMgr/assignment/<team>/<requestName>"
)
try:
Utilities.changeStatus(requestName, status,
self.wmstatWriteURL)
except RuntimeError as ex:
# ignore some of these errors: https://svnweb.cern.ch/trac/CMSDMWM/ticket/2002
if status != 'announced' and status != 'closed-out':
self.error("RuntimeError while changeStatus: reason: %s" %
ex)
raise cherrypy.HTTPError(
403, "Failed to change status: %s" % str(ex))
if priority != None:
Utilities.changePriority(requestName, priority,
self.wmstatWriteURL)
return request
def cloneRequest(self, requestName):
"""
Input assumes an existing request, checks that.
The original existing request is not touched.
A new request is generated.
The cloned request has a newly generated RequestName, new timestamp,
RequestDate, however -everything- else is copied from the original request.
Addition: since Edgar changed his mind, he no longer
wants this cloned request be in the 'new' state but put
straight into 'assignment-approved' state.
"""
request = None
if requestName:
self.info("Cloning request: request name: '%s'" % requestName)
requestOrigDict = self.getRequest(requestName)
# add OriginalRequestName to the new request schema
requestOrigDict['OriginalRequestName'] = requestName
if requestOrigDict:
self.info("Request found, cloning ...")
newReqSchema = Utilities.getNewRequestSchema(requestOrigDict)
# since we cloned the request, all the attributes
# manipulation in Utilities.makeRequest() should not be necessary
# the cloned request shall be identical but following arguments
toRemove = ["RequestName", "RequestDate", "timeStamp"]
toRemove.extend(deprecatedRequestArgs)
for remove in toRemove:
try:
del requestOrigDict[remove]
except KeyError:
pass
newReqSchema.update(requestOrigDict) # clone
# problem that priority wasn't preserved went down from here:
# via CheckIn.checkIn() -> MakeRequest.createRequest() -> Request.New.py factory
request = Utilities.buildWorkloadAndCheckIn(
self,
newReqSchema,
self.couchUrl,
self.workloadDBName,
self.wmstatWriteURL,
clone=True)
# change the clone request state as desired
Utilities.changeStatus(newReqSchema["RequestName"],
"assignment-approved",
self.wmstatWriteURL, self.acdcURL)
return request
else:
msg = "Request '%s' not found." % requestName
self.warning(msg)
raise cherrypy.HTTPError(404, msg)
else:
msg = "Received empty request name: '%s', exit." % requestName
self.warn(msg)
raise cherrypy.HTTPError(400, msg)
def cloneRequest(self, requestName):
"""
Input assumes an existing request, checks that.
The original existing request is not touched.
A new request is generated.
The cloned request has a newly generated RequestName, new timestamp,
RequestDate, however -everything- else is copied from the original request.
Addition: since Edgar changed his mind, he no longer
wants this cloned request be in the 'new' state but put
straight into 'assignment-approved' state.
"""
request = None
if requestName:
self.info("Cloning request: request name: '%s'" % requestName)
requestOrigDict = self.getRequest(requestName)
# add OriginalRequestName to the new request schema
requestOrigDict['OriginalRequestName'] = requestName
if requestOrigDict:
self.info("Request found, cloning ...")
newReqSchema = Utilities.getNewRequestSchema(requestOrigDict)
# since we cloned the request, all the attributes
# manipulation in Utilities.makeRequest() should not be necessary
# the cloned request shall be identical but following arguments
toRemove = ["RequestName",
"RequestDate",
"timeStamp"]
toRemove.extend(deprecatedRequestArgs)
for remove in toRemove:
try:
del requestOrigDict[remove]
except KeyError:
pass
newReqSchema.update(requestOrigDict) # clone
# problem that priority wasn't preserved went down from here:
# via CheckIn.checkIn() -> MakeRequest.createRequest() -> Request.New.py factory
request = Utilities.buildWorkloadAndCheckIn(self, newReqSchema,
self.couchUrl, self.workloadDBName,
self.wmstatWriteURL, clone=True)
# change the clone request state as desired
Utilities.changeStatus(newReqSchema["RequestName"],
"assignment-approved",
self.wmstatWriteURL,
self.acdcURL)
return request
else:
msg = "Request '%s' not found." % requestName
self.warning(msg)
raise cherrypy.HTTPError(404, msg)
else:
msg = "Received empty request name: '%s', exit." % requestName
self.warn(msg)
raise cherrypy.HTTPError(400, msg)
def announceRequest(self, requestName, cascade = False):
"""
Announce a request, if the cascade option is given
then it will search for any Resubmission requests
for which the given request is a parent and announce them too.
"""
requestsToAnnounce = [requestName]
if cascade == "True":
requestsToAnnounce.extend(Utilities.retrieveResubmissionChildren(requestName,
self.couchUrl,
self.workloadDBName))
for requestName in requestsToAnnounce:
Utilities.changeStatus(requestName, 'announced',
self.wmstatWriteURL,
self.acdcURL)
return
def announceRequest(self, requestName, cascade=False):
"""
Announce a request, if the cascade option is given
then it will search for any Resubmission requests
for which the given request is a parent and announce them too.
"""
requestsToAnnounce = [requestName]
if cascade == "True":
requestsToAnnounce.extend(
Utilities.retrieveResubmissionChildren(requestName,
self.couchUrl,
self.workloadDBName))
for requestName in requestsToAnnounce:
Utilities.changeStatus(requestName, 'announced',
self.wmstatWriteURL, self.acdcURL)
return
def putRequest(self, requestName=None, status=None, priority=None):
request = None
if requestName:
request = self.findRequest(requestName)
if request == None:
# Create a new request, with a JSON-encoded schema that is
# sent in the body of the HTTP request
body = cherrypy.request.body.read()
reqInputArgs = Utilities.unidecode(JsonWrapper.loads(body))
reqInputArgs.setdefault('CouchURL', Utilities.removePasswordFromUrl(self.couchUrl))
reqInputArgs.setdefault('CouchDBName', self.configDBName)
try:
self.info("Creating a request for: '%s'\n\tworkloadDB: '%s'\n\twmstatUrl: "
"'%s' ..." % (reqInputArgs, self.workloadDBName,
Utilities.removePasswordFromUrl(self.wmstatWriteURL)))
request = Utilities.makeRequest(self, reqInputArgs, self.couchUrl,
self.workloadDBName, self.wmstatWriteURL)
except cherrypy.HTTPError as ex:
self.error("Create request failed, reason: %s" % ex)
# Assume that this is a valid HTTPError
raise
except (WMException, Exception) as ex:
# TODO problem not to expose logs to the client
# e.g. on ConfigCacheID not found, the entire CouchDB traceback is sent in ex_message
self.error("Create request failed, reason: %s" % ex)
if hasattr(ex, "name"):
detail = ex.name
else:
detail = "check logs."
msg = "Create request failed, %s" % detail
raise cherrypy.HTTPError(400, msg)
self.info("Request '%s' created." % request['RequestName'])
# see if status & priority need to be upgraded
if status != None:
# forbid assignment here
if status == 'assigned':
raise cherrypy.HTTPError(403, "Cannot change status without a team. Please use PUT /reqmgr/reqMgr/assignment/<team>/<requestName>")
try:
Utilities.changeStatus(requestName, status, self.wmstatWriteURL)
except RuntimeError as ex:
# ignore some of these errors: https://svnweb.cern.ch/trac/CMSDMWM/ticket/2002
if status != 'announced' and status != 'closed-out':
self.error("RuntimeError while changeStatus: reason: %s" % ex)
raise cherrypy.HTTPError(403, "Failed to change status: %s" % str(ex))
if priority != None:
Utilities.changePriority(requestName, priority, self.wmstatWriteURL)
return request
def closeOutRequest(self, requestName, cascade = False):
"""
Close out a request, if the cascade option is given
then it will search for any Resubmission requests
for which the given request is a parent and close them out
too.
"""
requestsToCloseOut = [requestName]
if cascade == "True":
requestsToCloseOut.extend(Utilities.retrieveResubmissionChildren(requestName,
self.couchUrl,
self.workloadDBName))
for requestName in requestsToCloseOut:
Utilities.changeStatus(requestName, 'closed-out',
self.wmstatWriteURL,
self.acdcURL)
return
def closeOutRequest(self, requestName, cascade=False):
"""
Close out a request, if the cascade option is given
then it will search for any Resubmission requests
for which the given request is a parent and close them out
too.
"""
requestsToCloseOut = [requestName]
if cascade == "True":
requestsToCloseOut.extend(
Utilities.retrieveResubmissionChildren(requestName,
self.couchUrl,
self.workloadDBName))
for requestName in requestsToCloseOut:
Utilities.changeStatus(requestName, 'closed-out',
self.wmstatWriteURL, self.acdcURL)
return
def doAdmin(self, **kwargs):
""" format of kwargs is {'requestname:status' : 'approved', 'requestname:priority' : '2'} """
message = ""
for k, v in kwargs.iteritems():
if k.endswith(':status'):
requestName = k.split(':')[0]
self.validate(requestName)
status = v
priority = kwargs[requestName+':priority']
if priority != '':
Utilities.changePriority(requestName, priority, self.wmstatWriteURL)
message += "Changed priority for %s to %s\n" % (requestName, priority)
if status != "":
Utilities.changeStatus(requestName, status, self.wmstatWriteURL)
message += "Changed status for %s to %s\n" % (requestName, status)
if status == "assigned":
# make a page to choose teams
raise cherrypy.HTTPRedirect('/reqmgr/assign/one/%s' % requestName)
return message + detailsBackLink(requestName)
def doAdmin(self, **kwargs):
""" format of kwargs is {'requestname:status' : 'approved', 'requestname:priority' : '2'} """
message = ""
for k, v in kwargs.iteritems():
if k.endswith(':status'):
requestName = k.split(':')[0]
self.validate(requestName)
status = v
priority = kwargs[requestName+':priority']
if priority != '':
Utilities.changePriority(requestName, priority, self.wmstatWriteURL)
message += "Changed priority for %s to %s.\n" % (requestName, priority)
if status != "":
Utilities.changeStatus(requestName, status, self.wmstatWriteURL, self.acdcURL)
message += "Changed status for %s to %s\n" % (requestName, status)
if status == "assigned":
# make a page to choose teams
raise cherrypy.HTTPRedirect('/reqmgr/assign/one/%s' % requestName)
return message + detailsBackLink(requestName)
try:
request = Utilities.makeRequest(schema, self.couchUrl, self.workloadDBName)
except cherrypy.HTTPError:
# Assume that this is a valid HTTPError
raise
except WMException, ex:
raise cherrypy.HTTPError(400, ex._message)
except Exception, ex:
raise cherrypy.HTTPError(400, ex.message)
# see if status & priority need to be upgraded
if status != None:
# forbid assignment here
if status == 'assigned' and request['RequestStatus'] != 'ops-hold':
raise cherrypy.HTTPError(403, "Cannot change status without a team. Please use PUT /reqmgr/rest/assignment/<team>/<requestName>")
try:
Utilities.changeStatus(requestName, status)
except RuntimeError, e:
# ignore some of these errors: https://svnweb.cern.ch/trac/CMSDMWM/ticket/2002
if status != 'announced' and status != 'closed-out':
raise cherrypy.HTTPError(403, "Failed to change status: %s" % str(e))
if priority != None:
Utilities.changePriority(requestName, priority)
return request
def putAssignment(self, team, requestName):
""" Assigns this request to this team """
# see if it's already assigned
team = urllib.unquote(team)
if not team in ProdManagement.listTeams():
raise cherrypy.HTTPError(404,"Cannot find this team")
requestNamesAndIDs = ListRequests.listRequestsByTeam(team)
def putWorkQueue(self, request, url):
""" Registers the request as "acquired" by the workqueue with the given URL """
Utilities.changeStatus(request, "acquired", self.wmstatWriteURL, self.acdcURL)
return ProdManagement.associateProdMgr(request, urllib.unquote(url))
def putRequest(self, requestName=None, status=None, priority=None, stats=None):
request = None
if requestName:
try:
request = self.getRequest(requestName)
except Exception as ex:
# request presumably doesn't exist
request = None
if request == None:
# Create a new request, with a JSON-encoded schema that is
# sent in the body of the HTTP request
body = cherrypy.request.body.read()
reqInputArgs = Utilities.unidecode(JsonWrapper.loads(body))
# check for forbidden input arguments:
for deprec in deprecatedRequestArgs:
if deprec in reqInputArgs:
msg = ("Creating request failed, unsupported input arg: %s: %s" %
(deprec, reqInputArgs[deprec]))
self.error(msg)
raise cherrypy.HTTPError(400, msg)
reqInputArgs.setdefault('CouchURL', Utilities.removePasswordFromUrl(self.couchUrl))
reqInputArgs.setdefault('CouchWorkloadDBName', self.workloadDBName)
# wrong naming ... but it's all over the place, it's the config cache DB name
reqInputArgs.setdefault('CouchDBName', self.configDBName)
reqInputArgs.setdefault('Requestor', cherrypy.request.user["login"])
try:
self.info("Creating a request for: '%s'\n\tworkloadDB: '%s'\n\twmstatUrl: "
"'%s' ..." % (reqInputArgs, self.workloadDBName,
Utilities.removePasswordFromUrl(self.wmstatWriteURL)))
request = Utilities.makeRequest(self, reqInputArgs, self.couchUrl,
self.workloadDBName, self.wmstatWriteURL)
except cherrypy.HTTPError as ex:
msg = traceback.format_exc()
self.error("Create request failed, reason: %s" % msg)
# Assume that this is a valid HTTPError
raise ex
except (WMException, Exception) as ex:
# TODO problem not to expose logs to the client
# e.g. on ConfigCacheID not found, the entire CouchDB traceback is sent in ex_message
msg = traceback.format_exc()
self.error("Create request failed, reason: %s" % msg)
if hasattr(ex, "message"):
if hasattr(ex.message, '__call__'):
detail = ex.message()
else:
detail = str(ex)
elif hasattr(ex, "name"):
detail = ex.name
else:
detail = "check logs."
msg = "Create request failed, %s" % detail
raise cherrypy.HTTPError(400, msg)
self.info("Request '%s' created." % request['RequestName'])
# see if status & priority need to be upgraded
if status != None:
# forbid assignment here
if status == 'assigned':
raise cherrypy.HTTPError(403, "Cannot change status without a team. Please use PUT /reqmgr/reqMgr/assignment/<team>/<requestName>")
try:
Utilities.changeStatus(requestName, status, self.wmstatWriteURL, self.acdcURL)
except RuntimeError as ex:
# ignore some of these errors: https://svnweb.cern.ch/trac/CMSDMWM/ticket/2002
if status != 'announced' and status != 'closed-out':
self.error("RuntimeError while changeStatus: reason: %s" % ex)
raise cherrypy.HTTPError(403, "Failed to change status: %s" % str(ex))
if priority != None:
Utilities.changePriority(requestName, priority, self.wmstatWriteURL)
if stats != None:
Utilities.updateRequestStats(requestName = requestName,
stats = stats,
couchURL = self.couchUrl,
couchDBName = self.workloadDBName)
return request
try:
request = Utilities.makeRequest(schema, self.couchUrl, self.workloadDBName, self.wmstatWriteURL)
except cherrypy.HTTPError:
# Assume that this is a valid HTTPError
raise
except WMException, ex:
raise cherrypy.HTTPError(400, ex._message)
except Exception, ex:
raise cherrypy.HTTPError(400, ex.message)
# see if status & priority need to be upgraded
if status != None:
# forbid assignment here
if status == 'assigned' and request['RequestStatus'] != 'ops-hold':
raise cherrypy.HTTPError(403, "Cannot change status without a team. Please use PUT /reqmgr/rest/assignment/<team>/<requestName>")
try:
Utilities.changeStatus(requestName, status, self.wmstatWriteURL)
except RuntimeError, e:
# ignore some of these errors: https://svnweb.cern.ch/trac/CMSDMWM/ticket/2002
if status != 'announced' and status != 'closed-out':
raise cherrypy.HTTPError(403, "Failed to change status: %s" % str(e))
if priority != None:
Utilities.changePriority(requestName, priority, self.wmstatWriteURL)
return request
def putAssignment(self, team, requestName):
""" Assigns this request to this team """
# see if it's already assigned
team = urllib.unquote(team)
if not team in ProdManagement.listTeams():
raise cherrypy.HTTPError(404,"Cannot find this team")
requestNamesAndIDs = ListRequests.listRequestsByTeam(team)
def putRequest(self,
requestName=None,
status=None,
priority=None,
stats=None):
request = None
if requestName:
try:
request = self.getRequest(requestName)
except Exception as ex:
# request presumably doesn't exist
request = None
if request == None:
# Create a new request, with a JSON-encoded schema that is
# sent in the body of the HTTP request
body = cherrypy.request.body.read()
reqInputArgs = Utilities.unidecode(JsonWrapper.loads(body))
# check for forbidden input arguments:
for deprec in deprecatedRequestArgs:
if deprec in reqInputArgs:
msg = (
"Creating request failed, unsupported input arg: %s: %s"
% (deprec, reqInputArgs[deprec]))
self.error(msg)
raise cherrypy.HTTPError(400, msg)
reqInputArgs.setdefault(
'CouchURL', Utilities.removePasswordFromUrl(self.couchUrl))
reqInputArgs.setdefault('CouchWorkloadDBName', self.workloadDBName)
# wrong naming ... but it's all over the place, it's the config cache DB name
reqInputArgs.setdefault('CouchDBName', self.configDBName)
reqInputArgs.setdefault('Requestor',
cherrypy.request.user["login"])
try:
self.info(
"Creating a request for: '%s'\n\tworkloadDB: '%s'\n\twmstatUrl: "
"'%s' ..." %
(reqInputArgs, self.workloadDBName,
Utilities.removePasswordFromUrl(self.wmstatWriteURL)))
request = Utilities.makeRequest(self, reqInputArgs,
self.couchUrl,
self.workloadDBName,
self.wmstatWriteURL)
except cherrypy.HTTPError as ex:
msg = traceback.format_exc()
self.error("Create request failed, reason: %s" % msg)
# Assume that this is a valid HTTPError
raise ex
except (WMException, Exception) as ex:
# TODO problem not to expose logs to the client
# e.g. on ConfigCacheID not found, the entire CouchDB traceback is sent in ex_message
msg = traceback.format_exc()
self.error("Create request failed, reason: %s" % msg)
if hasattr(ex, "message"):
if hasattr(ex.message, '__call__'):
detail = ex.message()
else:
detail = str(ex)
elif hasattr(ex, "name"):
detail = ex.name
else:
detail = "check logs."
msg = "Create request failed, %s" % detail
raise cherrypy.HTTPError(400, msg)
self.info("Request '%s' created." % request['RequestName'])
# see if status & priority need to be upgraded
if status != None:
# forbid assignment here
if status == 'assigned':
raise cherrypy.HTTPError(
403,
"Cannot change status without a team. Please use PUT /reqmgr/reqMgr/assignment/<team>/<requestName>"
)
try:
Utilities.changeStatus(requestName, status,
self.wmstatWriteURL, self.acdcURL)
except RuntimeError as ex:
# ignore some of these errors: https://svnweb.cern.ch/trac/CMSDMWM/ticket/2002
if status != 'announced' and status != 'closed-out':
self.error("RuntimeError while changeStatus: reason: %s" %
ex)
raise cherrypy.HTTPError(
403, "Failed to change status: %s" % str(ex))
if priority != None:
Utilities.changePriority(requestName, priority,
self.wmstatWriteURL)
if stats != None:
Utilities.updateRequestStats(requestName=requestName,
stats=stats,
couchURL=self.couchUrl,
couchDBName=self.workloadDBName)
return request
def putWorkQueue(self, request, url):
""" Registers the request as "acquired" by the workqueue with the given URL """
Utilities.changeStatus(request, "acquired", self.wmstatWriteURL,
self.acdcURL)
return ProdManagement.associateProdMgr(request, urllib.unquote(url))
class ReqMgrRESTModel(RESTModel):
""" The REST interface to the ReqMgr database. Documentation may
be found at https://twiki.cern.ch/twiki/bin/viewauth/CMS/ReqMgrSystemDesign """
def __init__(self, config):
RESTModel.__init__(self, config)
self.couchUrl = config.couchUrl
self.workloadDBName = config.workloadDBName
self.configDBName = config.configDBName
self.wmstatWriteURL = "%s/%s" % (self.couchUrl.rstrip('/'),
config.wmstatDBName)
self.acdcURL = "%s/%s" % (self.couchUrl.rstrip('/'), config.acdcDBName)
self.security_params = {'roles': config.security_roles}
# Optional values for individual methods
self.reqPriorityMax = getattr(config, 'maxReqPriority', 100)
self._addMethod('GET',
'request',
self.getRequest,
args=['requestName'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('GET',
'assignment',
self.getAssignment,
args=['teamName', 'request'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('GET',
'user',
self.getUser,
args=['userName'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('GET',
'group',
self.getGroup,
args=['group', 'user'],
secured=True,
expires=0)
self._addMethod('GET',
'version',
self.getVersion,
args=[],
secured=True,
expires=0)
self._addMethod('GET',
'team',
self.getTeam,
args=[],
secured=True,
expires=0)
self._addMethod('GET',
'workQueue',
self.getWorkQueue,
args=['request', 'workQueue'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('GET',
'message',
self.getMessage,
args=['request'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('GET',
'inputdataset',
self.getInputDataset,
args=['prim', 'proc', 'tier'],
secured=True)
self._addMethod('GET',
'outputdataset',
self.getOutputDataset,
args=['prim', 'proc', 'tier'],
secured=True)
self._addMethod('GET',
'campaign',
self.getCampaign,
args=['campaign'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('PUT',
'request',
self.putRequest,
args=['requestName', 'status', 'priority', 'stats'],
secured=True,
validation=[
self.isalnumExceptStats, self.reqPriority,
self.validateStats
])
self._addMethod('PUT',
'clone',
self.cloneRequest,
args=['requestName'],
secured=True,
validation=[self.isalnum])
self._addMethod('PUT',
'assignment',
self.putAssignment,
args=['team', 'requestName'],
secured=True,
security_params=self.security_params,
validation=[self.isalnum])
self._addMethod('PUT',
'user',
self.putUser,
args=['userName', 'email', 'dnName'],
secured=True,
security_params=self.security_params,
validation=[self.validateUser])
self._addMethod('PUT',
'group',
self.putGroup,
args=['group', 'user'],
secured=True,
security_params=self.security_params,
validation=[self.isalnum])
self._addMethod('PUT',
'version',
self.putVersion,
args=['version', 'scramArch'],
secured=True,
security_params=self.security_params,
validation=[self.validateVersion])
self._addMethod('PUT',
'team',
self.putTeam,
args=['team'],
secured=True,
security_params=self.security_params,
validation=[self.isalnum])
self._addMethod('PUT',
'workQueue',
self.putWorkQueue,
args=['request', 'url'],
secured=True,
security_params=self.security_params,
validation=[self.validatePutWorkQueue])
self._addMethod('PUT',
'message',
self.putMessage,
args=['request'],
secured=True,
security_params=self.security_params,
validation=[self.isalnum])
self._addMethod('PUT',
'campaign',
self.putCampaign,
args=['campaign', 'request'],
secured=True,
validation=[self.isalnum])
self._addMethod('POST',
'request',
self.postRequest,
args=[
'requestName', 'events_written', 'events_merged',
'files_written', 'files_merged', 'percent_written',
'percent_success', 'dataset'
],
secured=True,
validation=[self.validateUpdates])
self._addMethod('POST',
'closeout',
self.closeOutRequest,
args=['requestName', 'cascade'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('POST',
'announce',
self.announceRequest,
args=['requestName', 'cascade'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('DELETE',
'request',
self.deleteRequest,
args=['requestName'],
secured=True,
security_params=self.security_params,
validation=[self.isalnum])
self._addMethod('DELETE',
'user',
self.deleteUser,
args=['user'],
secured=True,
security_params=self.security_params,
validation=[self.isalnum])
self._addMethod('DELETE',
'group',
self.deleteGroup,
args=['group', 'user'],
secured=True,
security_params=self.security_params,
validation=[self.isalnum])
self._addMethod('DELETE',
'version',
self.deleteVersion,
args=['version', 'scramArch'],
secured=True,
validation=[self.validateVersion])
self._addMethod('DELETE',
'team',
self.deleteTeam,
args=['team'],
secured=True,
security_params=self.security_params,
validation=[self.isalnum])
self._addMethod('DELETE',
'campaign',
self.deleteCampaign,
args=['campaign'],
secured=True,
security_params=self.security_params,
validation=[self.isalnum])
self._addMethod('GET',
'requestnames',
self.getRequestNames,
args=[],
secured=True,
expires=0)
self._addMethod('GET',
'outputDatasetsByRequestName',
self.getOutputForRequest,
args=['requestName'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('GET',
'outputDatasetsByPrepID',
self.getOutputForPrepID,
args=['prepID'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('GET',
'mostRecentOutputDatasetsByPrepID',
self.getMostRecentOutputForPrepID,
args=['prepID'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('GET',
'configIDs',
self.getConfigIDs,
args=['prim', 'proc', 'tier'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('GET',
'requestsByStatusAndTeam',
self.getRequestsByStatusAndTeam,
args=['teamName', 'status'],
secured=True,
validation=[self.isalnum],
expires=0)
cherrypy.engine.subscribe('start_thread', self.initThread)
def initThread(self, thread_index):
""" The ReqMgr expects the DBI to be contained in the Thread """
myThread = threading.currentThread()
#myThread = cherrypy.thread_data
# Get it from the DBFormatter superclass
myThread.dbi = self.dbi
def isalnum(self, index):
""" Validates that all input is alphanumeric,
with spaces and underscores tolerated"""
for v in index.values():
WMCore.Lexicon.identifier(v)
return index
def isalnumExceptStats(self, index):
""" Validates that all input is alphanumeric,
with spaces and underscores tolerated"""
if index.has_key('stats'):
return index
return self.isalnum(index)
def getDataset(self, prim, proc, tier):
""" If only prim exists, assume it's urlquoted.
If all three exists, assue it's /prim/proc/tier
"""
if not proc and not tier:
dataset = urllib.unquote(prim)
elif prim and proc and tier:
dataset = "/%s/%s/%s" % (prim, proc, tier)
WMCore.Lexicon.dataset(dataset)
return dataset
def intpriority(self, index):
""" Casts priority to an integer """
if index.has_key('priority'):
value = int(index['priority'])
if math.fabs(value) >= sys.maxint:
msg = "Invalid priority! Priority must have abs() less then MAXINT!"
raise cherrypy.HTTPError(400, msg)
index['priority'] = value
return index
def reqPriority(self, index):
"""
_reqPriority_
Sets request priority to an integer.
Also makes sure it's within a certain value.
"""
if not index.has_key('priority'):
return index
index = self.intpriority(index=index)
value = index['priority']
if math.fabs(
value) > self.reqPriorityMax and not Utilities.privileged():
msg = "Invalid requestPriority! Request priority must have abs() less then %i!" % self.reqPriorityMax
raise cherrypy.HTTPError(400, msg)
return index
def validateUser(self, index):
assert index['userName'].isalnum()
assert '@' in index['email']
assert index['email'].replace('@', '').replace('.', '').isalnum()
if 'dnName' in index:
assert index['dnName'].replace(' ', '').isalnum()
return index
def validateVersion(self, index):
""" Make sure it's a legitimate CMSSW version format """
WMCore.Lexicon.cmsswversion(index['version'])
return index
def getRequest(self, requestName=None):
""" If a request name is specified, return the details of the request.
Otherwise, return an overview of all requests """
if requestName == None:
result = GetRequest.getRequests()
else:
result = Utilities.requestDetails(requestName)
try:
teamNames = GetRequest.getAssignmentsByName(requestName)
result['teams'] = teamNames
except:
# Ignore errors, then we just don't have a team name
pass
return result
def getRequestNames(self):
# 2013-02-13 is wrong anyway (e.g. Campaign is not working)
# should be removed
""" return all the request names in RequestManager as list """
#TODO this could me combined with getRequest
return GetRequest.getOverview()
def getOutputForRequest(self, requestName):
"""Return the datasets produced by this request."""
return Utilities.getOutputForRequest(requestName)
def getOutputForPrepID(self, prepID):
"""Return the datasets produced by this prep ID. in a dict of requestName:dataset list"""
requestIDs = GetRequest.getRequestByPrepID(prepID)
result = {}
for requestID in requestIDs:
request = GetRequest.getRequest(requestID)
requestName = request["RequestName"]
helper = Utilities.loadWorkload(request)
result[requestName] = helper.listOutputDatasets()
return result
def getMostRecentOutputForPrepID(self, prepID):
"""Return the datasets produced by the most recently submitted request with this prep ID"""
requestIDs = GetRequest.getRequestByPrepID(prepID)
# most recent will have the largest ID
requestIDs.sort()
requestIDs.reverse()
request = None
# Go through each request in order from largest to smallest
# looking for the first non-failed/non-canceled request
for requestID in requestIDs:
request = GetRequest.getRequest(requestID)
rejectList = ['aborted', 'failed', 'rejected', 'epic-failed']
requestStatus = request.get("RequestStatus", 'aborted').lower()
if requestStatus not in rejectList:
break
if request != None:
helper = Utilities.loadWorkload(request)
return helper.listOutputDatasets()
else:
return []
def getAssignment(self, teamName=None, request=None):
""" If a team name is passed in, get all assignments for that team.
If a request is passed in, return a list of teams the request is
assigned to
"""
# better to use ReqMgr/RequestDB/Interface/ProdSystem/ProdMgrRetrieve?
#requestIDs = ProdMgrRetrieve.findAssignedRequests(teamName)
# Maybe now assigned to team is same as assigned to ProdMgr
result = []
if teamName != None:
requestIDs = ListRequests.listRequestsByTeam(teamName,
"assigned").values()
requests = [GetRequest.getRequest(reqID) for reqID in requestIDs]
# put highest priorities first
requests.sort(key=lambda r: r['RequestPriority'], reverse=True)
# return list of tuples, since we need sorting
result = [[req['RequestName'], req['RequestWorkflow']]
for req in requests]
elif request != None:
result = GetRequest.getAssignmentsByName(request)
return result
def getRequestsByStatusAndTeam(self, teamName, status):
""" Get a list of request names with the given team and status.
"""
requestNames = ListRequests.listRequestsByTeam(teamName, status).keys()
return requestNames
def getUser(self, userName=None, group=None):
""" No args returns a list of all users. Group returns groups this user is in. Username
returs a JSON with information about the user """
if userName != None:
if not Registration.isRegistered(userName):
raise cherrypy.HTTPError(404, "Cannot find user")
result = {}
result['groups'] = GroupInfo.groupsForUser(userName).keys()
result['requests'] = UserRequests.listRequests(userName).keys()
result.update(Registration.userInfo(userName))
return result
elif group != None:
GroupInfo.usersInGroup(group)
else:
return Registration.listUsers()
def getGroup(self, group=None, user=None):
""" No args lists all groups, one args returns JSON with users."""
if group != None:
result = {}
result['users'] = GroupInfo.usersInGroup(group)
return result
elif user != None:
return GroupInfo.groupsForUser(user).keys()
else:
return GroupInfo.listGroups()
def getVersion(self):
""" Returns a list of all CMSSW versions registered with ReqMgr """
archList = SoftwareAdmin.listSoftware()
result = []
for arch in archList.keys():
for version in archList[arch]:
if not version in result:
result.append(version)
return result
def getTeam(self):
""" Returns a list of all teams registered with ReqMgr """
return ProdManagement.listTeams()
def getWorkQueue(self, request=None, workQueue=None):
""" If a request is passed in, return the URl of the workqueue.
If a workqueue is passed in, return all requests acquired by it """
if workQueue != None:
return ProdMgrRetrieve.findAssignedRequests(workQueue)
if request != None:
return ProdManagement.getProdMgr(request)
# return all the workqueue ulr
return GetRequest.getGlobalQueues()
def getMessage(self, request):
""" Returns a list of messages attached to this request """
return ChangeState.getMessages(request)
def getInputDataset(self, prim, proc=None, tier=None):
""" returns a list of requests with this input dataset
Input can either be a single urlquoted dataset, or a
/prim/proc/tier"""
dataset = self.getDataset(prim, proc, tier)
return GetRequest.getRequestsByCriteria("Datasets.GetRequestByInput",
dataset)
def getOutputDataset(self, prim, proc=None, tier=None):
""" returns a list of requests with this output dataset
Input can either be a single urlquoted dataset, or a
/prim/proc/tier"""
dataset = self.getDataset(prim, proc, tier)
return GetRequest.getRequestsByCriteria("Datasets.GetRequestByOutput",
dataset)
def getCampaign(self, campaign=None):
""" returns a list of all campaigns if no argument, and a list of
all requests in a campaign if there is an argument """
if campaign == None:
return Campaign.listCampaigns()
else:
return Campaign.listRequestsByCampaign(campaign)
def putWorkQueue(self, request, url):
""" Registers the request as "acquired" by the workqueue with the given URL """
Utilities.changeStatus(request, "acquired", self.wmstatWriteURL,
self.acdcURL)
return ProdManagement.associateProdMgr(request, urllib.unquote(url))
def validatePutWorkQueue(self, index):
assert index['request'].replace('_',
'').replace('-',
'').replace('.',
'').isalnum()
assert index['url'].startswith('http')
return index
# had no permission control before, security issue fix
@cherrypy.tools.secmodv2(role=Utilities.security_roles(),
group=Utilities.security_groups())
def putRequest(self,
requestName=None,
status=None,
priority=None,
stats=None):
request = None
if requestName:
try:
request = self.getRequest(requestName)
except Exception, ex:
# request presumably doesn't exist
request = None
if request == None:
# Create a new request, with a JSON-encoded schema that is
# sent in the body of the HTTP request
body = cherrypy.request.body.read()
reqInputArgs = Utilities.unidecode(JsonWrapper.loads(body))
# check for forbidden input arguments:
for deprec in deprecatedRequestArgs:
if deprec in reqInputArgs:
msg = (
"Creating request failed, unsupported input arg: %s: %s"
% (deprec, reqInputArgs[deprec]))
self.error(msg)
raise cherrypy.HTTPError(400, msg)
reqInputArgs.setdefault(
'CouchURL', Utilities.removePasswordFromUrl(self.couchUrl))
reqInputArgs.setdefault('CouchWorkloadDBName', self.workloadDBName)
# wrong naming ... but it's all over the place, it's the config cache DB name
reqInputArgs.setdefault('CouchDBName', self.configDBName)
reqInputArgs.setdefault('Requestor',
cherrypy.request.user["login"])
try:
self.info(
"Creating a request for: '%s'\n\tworkloadDB: '%s'\n\twmstatUrl: "
"'%s' ..." %
(reqInputArgs, self.workloadDBName,
Utilities.removePasswordFromUrl(self.wmstatWriteURL)))
request = Utilities.makeRequest(self, reqInputArgs,
self.couchUrl,
self.workloadDBName,
self.wmstatWriteURL)
except cherrypy.HTTPError as ex:
self.error("Create request failed, reason: %s" % ex)
# Assume that this is a valid HTTPError
raise ex
except (WMException, Exception) as ex:
# TODO problem not to expose logs to the client
# e.g. on ConfigCacheID not found, the entire CouchDB traceback is sent in ex_message
self.error("Create request failed, reason: %s" % ex)
if hasattr(ex, "name"):
detail = ex.name
else:
detail = "check logs."
msg = "Create request failed, %s" % detail
raise cherrypy.HTTPError(400, msg)
self.info("Request '%s' created." % request['RequestName'])
# see if status & priority need to be upgraded
if status != None:
# forbid assignment here
if status == 'assigned':
raise cherrypy.HTTPError(
403,
"Cannot change status without a team. Please use PUT /reqmgr/reqMgr/assignment/<team>/<requestName>"
)
try:
Utilities.changeStatus(requestName, status,
self.wmstatWriteURL, self.acdcURL)
except RuntimeError as ex:
# ignore some of these errors: https://svnweb.cern.ch/trac/CMSDMWM/ticket/2002
if status != 'announced' and status != 'closed-out':
self.error("RuntimeError while changeStatus: reason: %s" %
ex)
raise cherrypy.HTTPError(
403, "Failed to change status: %s" % str(ex))
if priority != None:
Utilities.changePriority(requestName, priority,
self.wmstatWriteURL)
if stats != None:
Utilities.updateRequestStats(requestName=requestName,
stats=stats,
couchURL=self.couchUrl,
couchDBName=self.workloadDBName)
return request
