def loadRequestSchema(workload, requestSchema):
"""
_loadRequestSchema_
Does modifications to the workload I don't understand
Takes a WMWorkloadHelper, operates on it directly with the schema
"""
schema = workload.data.request.section_('schema')
for key, value in requestSchema.iteritems():
if type(value) == dict and key == 'LumiList':
value = JsonWrapper.dumps(value)
try:
setattr(schema, key, value)
except Exception as ex:
# Attach TaskChain tasks
if type(value) == dict and requestSchema['RequestType'] == 'TaskChain' and 'Task' in key:
newSec = schema.section_(key)
for k, v in requestSchema[key].iteritems():
if type(value) == dict and key == 'LumiList':
value = JsonWrapper.dumps(value)
try:
setattr(newSec, k, v)
except Exception as ex:
# this logging need to change to cherry py logging
logging.error("Invalid Value: %s" % str(ex))
else:
# this logging need to change to cherry py logging
logging.error("Invalid Value: %s" % str(ex))
schema.timeStamp = int(time.time())
schema = workload.data.request.schema
# might belong in another method to apply existing schema
workload.data.owner.Group = schema.Group
workload.data.owner.Requestor = schema.Requestor
def test_cjson(self):
"""
Test cjson implementation.
"""
try:
import cjson
except:
raise nose.SkipTest
json_wrap._module = "cjson"
result = json_wrap.dumps(self.record)
expect = json.dumps(self.record)
self.assertEqual(expect, result)
data = result
result = json_wrap.loads(data)
expect = json.loads(data)
self.assertEqual(expect, result)
try:
json_wrap.loads("bogusbogus")
except cjson.DecodeError as ex:
self.assertEqual(ex.args,
("cannot parse JSON description: bogusbogus", ))
def setUp(self):
self.setConfig(config)
self.setCouchDBs([(config.views.data.couch_reqmgr_db, "ReqMgr"), (config.views.data.couch_reqmgr_aux_db, None)])
self.setSchemaModules([])
RESTBaseUnitTestWithDBBackend.setUp(self)
self.setFakeDN()
# print "%s" % self.test_authz_key.data
self.default_status_header = getAuthHeader(self.test_authz_key.data, DEFAULT_STATUS_PERMISSION)
requestPath = os.path.join(getWMBASE(), "test", "data", "ReqMgr", "requests", "DMWM")
rerecoFile = open(os.path.join(requestPath, "ReReco.json"), "r")
rerecoArgs = JsonWrapper.load(rerecoFile)
self.rerecoCreateArgs = rerecoArgs["createRequest"]
self.rerecoCreateArgs["PrepID"] = "test_prepid"
self.rerecoAssignArgs = rerecoArgs["assignRequest"]
# overwrite rereco args
self.rerecoAssignArgs["AcquisitionEra"] = "test_aqc"
lheFile = open(os.path.join(requestPath, "TaskChain_Data.json"), "r")
lheArgs = JsonWrapper.load(lheFile)
self.lheStep0CreateArgs = lheArgs["createRequest"]
self.lheStep0AssignArgs = lheArgs["assignRequest"]
self.lheStep0AssignArgs["AcquisitionEra"] = "test_aqc"
cmsswDoc = {"_id": "software"}
cmsswDoc[self.rerecoCreateArgs["ScramArch"]] = []
cmsswDoc[self.rerecoCreateArgs["ScramArch"]].append(self.rerecoCreateArgs["CMSSWVersion"])
insertDataToCouch(os.getenv("COUCHURL"), config.views.data.couch_reqmgr_aux_db, cmsswDoc)
def setUp(self):
self.setConfig(config)
self.setCouchDBs([(config.views.data.couch_reqmgr_db, "ReqMgr"),
(config.views.data.couch_reqmgr_aux_db, None)])
self.setSchemaModules([])
RESTBaseUnitTestWithDBBackend.setUp(self)
self.setFakeDN()
#print "%s" % self.test_authz_key.data
self.default_status_header = getAuthHeader(self.test_authz_key.data, DEFAULT_STATUS_PERMISSION)
requestPath = os.path.join(getWMBASE(), "test", "data", "ReqMgr", "requests")
rerecoFile = open(os.path.join(requestPath, "ReReco.json"), 'r')
rerecoArgs = JsonWrapper.load(rerecoFile)
self.rerecoCreateArgs = rerecoArgs["createRequest"]
self.rerecoAssignArgs = rerecoArgs["assignRequest"]
# overwrite rereco args
self.rerecoAssignArgs["AcquisitionEra"] = "test_aqc"
lheFile = open(os.path.join(requestPath, "LHEStep0.json"), 'r')
lheArgs = JsonWrapper.load(lheFile)
self.lheStep0CreateArgs = lheArgs["createRequest"]
self.lheStep0AssignArgs = lheArgs["assignRequest"]
self.lheStep0AssignArgs["AcquisitionEra"] = "test_aqc"
cmsswDoc = {"_id": "software"}
cmsswDoc[self.rerecoCreateArgs["ScramArch"]] = []
cmsswDoc[self.rerecoCreateArgs["ScramArch"]].append(self.rerecoCreateArgs["CMSSWVersion"])
insertDataToCouch(os.getenv("COUCHURL"), config.views.data.couch_reqmgr_aux_db, cmsswDoc)
def purgeService(self, obj):
"""
Purge job (even bulk) from wms
"""
# Implement as getOutput where the "No output files ..."
# is not an error condition but the expected status
if type(obj) == Job and self.valid(obj.runningJob):
# the object passed is a valid Job
command = "glite-wms-job-output --json --noint --dir /tmp/ " + obj.runningJob["schedulerId"]
out, ret = self.ExecuteCommand(self.proxyString + command)
if ret != 0:
self.logging.error("Purge of %s Failed with exit code =%d" % (str(job.runningJob["schedulerId"]), ret))
self.logging.error("output was \n%s" % out)
elif ret == 0:
loadOut = json.loads(out)
if unicode("result") in loadOut and loadOut[unicode("result")] == unicode("success"):
self.logging.info("Purge of %s successfully done" % str(job.runningJob["schedulerId"]))
else:
self.logging.error("Purge of %s Executed NOT VALIDATED" % str(job.runningJob["schedulerId"]))
self.logging.error(
"Purge of %s Executed with output\n%s" % (str(job.runningJob["schedulerId"]), out)
)
elif type(obj) == Task:
# the object passed is a Task
for job in obj.jobs:
if not self.valid(job.runningJob):
continue
command = "glite-wms-job-output --json --noint --dir /tmp/ " + job.runningJob["schedulerId"]
out, ret = self.ExecuteCommand(self.proxyString + command)
if ret != 0:
self.logging.error(
"Purge of %s Failed with exit code =%d" % (str(job.runningJob["schedulerId"]), ret)
)
self.logging.error("output was \n%s" % out)
elif ret == 0:
loadOut = json.loads(out)
if unicode("result") in loadOut and loadOut[unicode("result")] == unicode("success"):
self.logging.info("Purge of %s successfully done" % str(job.runningJob["schedulerId"]))
else:
self.logging.error("Purge of %s Executed NOT VALIDATED" % str(job.runningJob["schedulerId"]))
self.logging.error(
"Purge of %s Executed with output\n%s" % (str(job.runningJob["schedulerId"]), out)
)
def updateInboxElements(self, *elementIds, **updatedParams):
"""Update given inbox element's (identified by id) with new parameters"""
uri = "/" + self.inbox.name + "/_design/WorkQueue/_update/in-place/"
optionsArg = {}
if "options" in updatedParams:
optionsArg.update(updatedParams.pop("options"))
data = {"updates": json.dumps(updatedParams), "options": json.dumps(optionsArg)}
for ele in elementIds:
thisuri = uri + ele + "?" + urllib.urlencode(data)
self.inbox.makeRequest(uri=thisuri, type="PUT")
return
def set_opts(self, curl, url, params, headers,
ckey=None, cert=None, capath=None, verbose=None, verb='GET', doseq=True, cainfo=None):
"""Set options for given curl object, params should be a dictionary"""
if params == None:
params = {}
if not isinstance(params, dict):
raise TypeError("pycurl parameters should be passed as dictionary")
curl.setopt(pycurl.NOSIGNAL, self.nosignal)
curl.setopt(pycurl.TIMEOUT, self.timeout)
curl.setopt(pycurl.CONNECTTIMEOUT, self.connecttimeout)
curl.setopt(pycurl.FOLLOWLOCATION, self.followlocation)
curl.setopt(pycurl.MAXREDIRS, self.maxredirs)
encoded_data = urllib.urlencode(params, doseq=doseq)
if verb == 'GET':
url = url + '?' + encoded_data
elif verb == 'HEAD':
url = url + '?' + encoded_data
curl.setopt(pycurl.CUSTOMREQUEST, verb)
curl.setopt(pycurl.HEADER, 1)
curl.setopt(pycurl.NOBODY, True)
elif verb == 'POST':
curl.setopt(pycurl.POST, 1)
curl.setopt(pycurl.POSTFIELDS, json.dumps(params))
elif verb == 'DELETE' or verb == 'PUT':
curl.setopt(pycurl.CUSTOMREQUEST, verb)
curl.setopt(pycurl.HTTPHEADER, ['Transfer-Encoding: chunked'])
curl.setopt(pycurl.POSTFIELDS, json.dumps(params))
else:
raise Exception('Unsupported HTTP method "%s"' % verb)
curl.setopt(pycurl.URL, url)
if headers:
curl.setopt(pycurl.HTTPHEADER, \
["%s: %s" % (k, v) for k, v in headers.items()])
bbuf = StringIO.StringIO()
hbuf = StringIO.StringIO()
curl.setopt(pycurl.WRITEFUNCTION, bbuf.write)
curl.setopt(pycurl.HEADERFUNCTION, hbuf.write)
if capath:
curl.setopt(pycurl.CAPATH, capath)
curl.setopt(pycurl.SSL_VERIFYPEER, True)
if cainfo:
curl.setopt(pycurl.CAINFO, cainfo)
else:
curl.setopt(pycurl.SSL_VERIFYPEER, False)
if ckey:
curl.setopt(pycurl.SSLKEY, ckey)
if cert:
curl.setopt(pycurl.SSLCERT, cert)
if verbose:
curl.setopt(pycurl.VERBOSE, 1)
curl.setopt(pycurl.DEBUGFUNCTION, self.debug)
return bbuf, hbuf
def updateInboxElements(self, *elementIds, **updatedParams):
"""Update given inbox element's (identified by id) with new parameters"""
uri = "/" + self.inbox.name + "/_design/WorkQueue/_update/in-place/"
optionsArg = {}
if "options" in updatedParams:
optionsArg.update(updatedParams.pop("options"))
data = {
"updates": json.dumps(updatedParams),
"options": json.dumps(optionsArg)
}
for ele in elementIds:
thisuri = uri + ele + "?" + urllib.urlencode(data)
self.inbox.makeRequest(uri=thisuri, type='PUT')
return
def availableWork(self, conditions, teams=None, wfs=None):
"""Get work which is available to be run"""
elements = []
for site in conditions.keys():
if not conditions[site] > 0:
del conditions[site]
if not conditions:
return elements, conditions
options = {}
options['include_docs'] = True
options['descending'] = True
options['resources'] = conditions
if teams:
options['teams'] = teams
if wfs:
result = []
for i in xrange(0, len(wfs), 20):
options['wfs'] = wfs[i:i + 20]
data = self.db.loadList('WorkQueue', 'workRestrictions',
'availableByPriority', options)
result.extend(json.loads(data))
# sort final list
result.sort(key=lambda x: x[
'WMCore.WorkQueue.DataStructs.WorkQueueElement.WorkQueueElement'
]['Priority'])
else:
result = self.db.loadList('WorkQueue', 'workRestrictions',
'availableByPriority', options)
result = json.loads(result)
for i in result:
element = CouchWorkQueueElement.fromDocument(self.db, i)
elements.append(element)
# Remove 1st random site that can run work
names = conditions.keys()
random.shuffle(names)
for site in names:
if element.passesSiteRestriction(site):
slots_left = conditions[site] - element['Jobs']
if slots_left > 0:
conditions[site] = slots_left
else:
conditions.pop(site, None)
break
if not conditions:
break
return elements, conditions
def test_json(self):
"""
Test default json implementation.
"""
result = json_wrap.dumps(self.record)
expect = json.dumps(self.record)
self.assertEqual(expect, result)
data = result
result = json_wrap.loads(data)
expect = json.loads(data)
self.assertEqual(expect, result)
with self.assertRaises(ValueError):
json_wrap.loads("bogusbogus")
def check_resident(self, site, stage_data):
"""
Checks that the requested data is resident on the site
"""
goodToGo = True
# Format site name
locSite = site.replace('_Buffer', '_MSS')
if not locSite.endswith('_MSS'):
locSite += '_MSS'
qParams = {'node': locSite}
# Get block info from PhEDEx
phedex = Requests(url='https://cmsweb.cern.ch', idict={'accept_type':'application/json'})
for data in stage_data:
if data.find('#') < 0:
qParams['dataset'] = data
else:
qParams['block'] = data
self.logger.debug('Checking data residency for %s at %s' % (data, locSite))
try:
pdata = phedex.get('/phedex/datasvc/json/prod/blockreplicas', qParams)[0]
except httplib.HTTPException, he:
self.handleHTTPExcept(he, 'HTTPException for block: %s node: %s' % (data, locSite))
# Parse block info and check > 0 block exist
try:
if len(JsonWrapper.loads(pdata)['phedex']['block']) == 0:
goodToGo = False
self.logger.error('Block %s not resident at site %s' % (data, locSite))
except:
self.logger.debug('error loading json')
goodToGo = False
def makeSchema(self, **schema):
schema.setdefault('CouchURL', Utilities.removePasswordFromUrl(self.couchUrl))
schema.setdefault('CouchDBName', self.configDBName)
decodedSchema = {}
for key in schema.keys():
try:
decodedSchema[key] = JsonWrapper.loads(schema[key])
except:
# We don't know what kind of exception we'll get, so ignore them all
# If it does except, it probably wasn't in JSON to begin with.
# Anything else should be caught by the parsers and the validation
decodedSchema[key] = schema[key]
try:
self.info("Creating a request for: '%s'\n\tworkloadDB: '%s'\n\twmstatUrl: "
"'%s' ..." % (decodedSchema, self.workloadDBName,
Utilities.removePasswordFromUrl(self.wmstatWriteURL)))
request = Utilities.makeRequest(self, decodedSchema, self.couchUrl, self.workloadDBName, self.wmstatWriteURL)
# catching here KeyError is just terrible
except (RuntimeError, KeyError, Exception) as ex:
# TODO problem not to expose logs to the client
# e.g. on ConfigCacheID not found, the entire CouchDB traceback is sent in ex_message
self.error("Create request failed, reason: %s" % ex)
if hasattr(ex, "name"):
detail = ex.name
else:
detail = "check logs."
msg = "Create request failed, %s" % detail
raise cherrypy.HTTPError(400, msg)
baseURL = cherrypy.request.base
raise cherrypy.HTTPRedirect('%s/reqmgr/view/details/%s' % (baseURL, request['RequestName']))
def json(self, data):
if isinstance(data, GeneratorType):
out = ''.join([r for r in self.genstreamer(data)])
return out
thunker = JSONThunker()
data = thunker.thunk(data)
return JsonWrapper.dumps(data)
def splitting(self, requestName):
"""
_splitting_
Retrieve the current values for splitting parameters for all tasks in
the spec. Format them in the manner that the splitting page expects
and pass them to the template.
"""
self.validate(requestName)
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
splittingDict = helper.listJobSplittingParametersByTask(performance = False)
taskNames = splittingDict.keys()
taskNames.sort()
splitInfo = []
for taskName in taskNames:
jsonSplittingParams = JsonWrapper.dumps(splittingDict[taskName])
splitInfo.append({"splitAlgo": splittingDict[taskName]["algorithm"],
"splitParams": jsonSplittingParams,
"taskType": splittingDict[taskName]["type"],
"taskName": taskName})
return self.templatepage("Splitting", requestName = requestName,
taskInfo = splitInfo, taskNames = taskNames)
def _getResult(self, callname, clearCache = True,
args = None, verb = "GET"):
"""
_getResult_
retrieve JSON/XML formatted information given the service name and the
argument dictionaries
TODO: Probably want to move this up into Service
"""
result = ''
file = callname.replace("/", "_")
if clearCache:
self.clearCache(file, args, verb)
f = self.refreshCache(file, callname, args, verb = verb)
result = f.read()
f.close()
# if self.responseType == "json":
# decoder = json.JSONDecoder()
# return decoder.decode(result)
if result:
result = JsonWrapper.loads(result)
return result
def _makeHash(self, inputdata, hash):
"""
Turn the input data into json and hash the string. This is simple and
means that the input data must be json-serialisable, which is good.
"""
json_hash = json.dumps(inputdata)
return json_hash.__hash__()
def _validateMultiRequests(self, param, safe, valFunc):
data = cherrypy.request.body.read()
if data:
request_names, request_args = self._getMultiRequestArgs(JsonWrapper.loads(data))
else:
# actually this is error case
#cherrypy.log(str(param.kwargs))
request_names, request_args = self._getMultiRequestArgs(param.kwargs)
for prop in request_args:
if prop == "RequestStatus":
del param.kwargs["new_status"]
else:
del param.kwargs[prop]
del param.kwargs["ids"]
#remove this
#tmp = []
#for prop in param.kwargs:
# tmp.append(prop)
#for prop in tmp:
# del param.kwargs[prop]
safe.kwargs['workload_pair_list'] = []
for request_name in request_names:
request_args["RequestName"] = request_name
workload, r_args = valFunc(request_args, self.config, self.reqmgr_db_service, param)
safe.kwargs['workload_pair_list'].append((workload, r_args))
safe.kwargs["multi_update_flag"] = True
def availableWork(self, conditions, teams = None, wfs = None):
"""Get work which is available to be run"""
elements = []
for site in conditions.keys():
if not conditions[site] > 0:
del conditions[site]
if not conditions:
return elements, conditions
options = {}
options['include_docs'] = True
options['descending'] = True
options['resources'] = conditions
if teams:
options['teams'] = teams
if wfs:
options['wfs'] = wfs
result = self.db.loadList('WorkQueue', 'workRestrictions', 'availableByPriority', options)
result = json.loads(result)
for i in result:
element = CouchWorkQueueElement.fromDocument(self.db, i)
elements.append(element)
# Remove 1st random site that can run work
names = conditions.keys()
random.shuffle(names)
for site in names:
if element.passesSiteRestriction(site):
slots_left = conditions[site] - element['Jobs']
if slots_left > 0:
conditions[site] = slots_left
else:
conditions.pop(site, None)
break
return elements, conditions
def setUp(self):
self.setConfig(config)
self.setCouchDBs([(config.views.data.couch_reqmgr_db, "ReqMgr"),
(config.views.data.couch_reqmgr_aux_db, None)])
self.setSchemaModules([])
RESTBaseUnitTestWithDBBackend.setUp(self)
self.setFakeDN()
requestPath = os.path.join(getWMBASE(), "test", "data", "ReqMgr",
"requests", "DMWM")
rerecoFile = open(os.path.join(requestPath, "ReReco.json"), 'r')
rerecoArgs = JsonWrapper.load(rerecoFile)
self.rerecoCreateArgs = rerecoArgs["createRequest"]
self.rerecoAssignArgs = rerecoArgs["assignRequest"]
cmsswDoc = {"_id": "software"}
cmsswDoc[self.rerecoCreateArgs["ScramArch"]] = []
cmsswDoc[self.rerecoCreateArgs["ScramArch"]].append(
self.rerecoCreateArgs["CMSSWVersion"])
insertDataToCouch(os.getenv("COUCHURL"),
config.views.data.couch_reqmgr_aux_db, cmsswDoc)
self.reqSvc = ReqMgr(self.jsonSender["host"])
self.reqSvc._noStale = True
self.reqSvc['requests'].additionalHeaders = self.create_header
def _validateRequestBase(self, param, safe, valFunc, requestName=None):
data = cherrypy.request.body.read()
if data:
request_args = JsonWrapper.loads(data)
if requestName:
request_args["RequestName"] = requestName
if isinstance(request_args, dict):
request_args = [request_args]
else:
# actually this is error case
# cherrypy.log(str(param.kwargs))
request_args = {}
for prop in param.kwargs:
request_args[prop] = param.kwargs[prop]
for prop in request_args:
del param.kwargs[prop]
if requestName:
request_args["RequestName"] = requestName
request_args = [request_args]
safe.kwargs['workload_pair_list'] = []
if isinstance(request_args, dict):
request_args = [request_args]
for args in request_args:
workload, r_args = valFunc(args, self.config, self.reqmgr_db_service, param)
safe.kwargs['workload_pair_list'].append((workload, r_args))
def _validateMultiRequests(self, param, safe, valFunc):
data = cherrypy.request.body.read()
if data:
request_names, request_args = self._getMultiRequestArgs(JsonWrapper.loads(data))
else:
# actually this is error case
# cherrypy.log(str(param.kwargs))
request_names, request_args = self._getMultiRequestArgs(param.kwargs)
for prop in request_args:
if prop == "RequestStatus":
del param.kwargs["new_status"]
else:
del param.kwargs[prop]
del param.kwargs["ids"]
# remove this
# tmp = []
# for prop in param.kwargs:
# tmp.append(prop)
# for prop in tmp:
# del param.kwargs[prop]
safe.kwargs['workload_pair_list'] = []
for request_name in request_names:
request_args["RequestName"] = request_name
workload, r_args = valFunc(request_args, self.config, self.reqmgr_db_service, param)
safe.kwargs['workload_pair_list'].append((workload, r_args))
safe.kwargs["multi_update_flag"] = True
def _getResult(self, callname, clearCache=True, args=None, verb="GET"):
"""
_getResult_
retrieve JSON/XML formatted information given the service name and the
argument dictionaries
TODO: Probably want to move this up into Service
"""
result = ''
file = callname.replace("/", "_")
if clearCache:
self.clearCache(file, args, verb)
f = self.refreshCache(file, callname, args, verb=verb)
result = f.read()
f.close()
# if self.responseType == "json":
# decoder = json.JSONDecoder()
# return decoder.decode(result)
if result:
result = JsonWrapper.loads(result)
return result
def _makeHash(self, inputdata, hash):
"""
Turn the input data into json and hash the string. This is simple and
means that the input data must be json-serialisable, which is good.
"""
json_hash = json.dumps(inputdata)
return json_hash.__hash__()
def json(self, data):
if isinstance(data, GeneratorType):
out = ''.join([r for r in self.genstreamer(data)])
return out
thunker = JSONThunker()
data = thunker.thunk(data)
return JsonWrapper.dumps(data)
def splitting(self, requestName):
"""
_splitting_
Retrieve the current values for splitting parameters for all tasks in
the spec. Format them in the manner that the splitting page expects
and pass them to the template.
"""
self.validate(requestName)
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
splittingDict = helper.listJobSplittingParametersByTask()
taskNames = splittingDict.keys()
taskNames.sort()
splitInfo = []
for taskName in taskNames:
jsonSplittingParams = JsonWrapper.dumps(splittingDict[taskName])
splitInfo.append({"splitAlgo": splittingDict[taskName]["algorithm"],
"splitParams": jsonSplittingParams,
"taskType": splittingDict[taskName]["type"],
"taskName": taskName})
return self.templatepage("Splitting", requestName = requestName,
taskInfo = splitInfo, taskNames = taskNames)
def _validateRequestBase(self, param, safe, valFunc, requestName=None):
data = cherrypy.request.body.read()
if data:
request_args = JsonWrapper.loads(data)
if requestName:
request_args["RequestName"] = requestName
if isinstance(request_args, dict):
request_args = [request_args]
else:
# actually this is error case
#cherrypy.log(str(param.kwargs))
request_args = {}
for prop in param.kwargs:
request_args[prop] = param.kwargs[prop]
for prop in request_args:
del param.kwargs[prop]
if requestName:
request_args["RequestName"] = requestName
request_args = [request_args]
safe.kwargs['workload_pair_list'] = []
if isinstance(request_args, dict):
request_args = [request_args]
for args in request_args:
workload, r_args = valFunc(args, self.config,
self.reqmgr_db_service, param)
safe.kwargs['workload_pair_list'].append((workload, r_args))
def putRequest(self, requestName=None, status=None, priority=None):
request = None
if requestName:
request = self.findRequest(requestName)
if request == None:
# Create a new request, with a JSON-encoded schema that is
# sent in the body of the HTTP request
body = cherrypy.request.body.read()
reqInputArgs = Utilities.unidecode(JsonWrapper.loads(body))
reqInputArgs.setdefault(
'CouchURL', Utilities.removePasswordFromUrl(self.couchUrl))
reqInputArgs.setdefault('CouchDBName', self.configDBName)
try:
self.info(
"Creating a request for: '%s'\n\tworkloadDB: '%s'\n\twmstatUrl: "
"'%s' ..." %
(reqInputArgs, self.workloadDBName,
Utilities.removePasswordFromUrl(self.wmstatWriteURL)))
request = Utilities.makeRequest(self, reqInputArgs,
self.couchUrl,
self.workloadDBName,
self.wmstatWriteURL)
except cherrypy.HTTPError as ex:
self.error("Create request failed, reason: %s" % ex)
# Assume that this is a valid HTTPError
raise
except (WMException, Exception) as ex:
# TODO problem not to expose logs to the client
# e.g. on ConfigCacheID not found, the entire CouchDB traceback is sent in ex_message
self.error("Create request failed, reason: %s" % ex)
if hasattr(ex, "name"):
detail = ex.name
else:
detail = "check logs."
msg = "Create request failed, %s" % detail
raise cherrypy.HTTPError(400, msg)
self.info("Request '%s' created." % request['RequestName'])
# see if status & priority need to be upgraded
if status != None:
# forbid assignment here
if status == 'assigned':
raise cherrypy.HTTPError(
403,
"Cannot change status without a team. Please use PUT /reqmgr/reqMgr/assignment/<team>/<requestName>"
)
try:
Utilities.changeStatus(requestName, status,
self.wmstatWriteURL)
except RuntimeError as ex:
# ignore some of these errors: https://svnweb.cern.ch/trac/CMSDMWM/ticket/2002
if status != 'announced' and status != 'closed-out':
self.error("RuntimeError while changeStatus: reason: %s" %
ex)
raise cherrypy.HTTPError(
403, "Failed to change status: %s" % str(ex))
if priority != None:
Utilities.changePriority(requestName, priority,
self.wmstatWriteURL)
return request
def updateInboxElements(self, *elementIds, **updatedParams):
"""Update given inbox element's (identified by id) with new parameters"""
uri = "/" + self.inbox.name + "/_design/WorkQueue/_update/in-place/"
data = {"updates" : json.dumps(updatedParams)}
for ele in elementIds:
thisuri = uri + ele + "?" + urllib.urlencode(data)
self.inbox.makeRequest(uri = thisuri, type = 'PUT')
return
def makeLumiList(lumiDict):
try:
if isinstance(lumiDict, basestring):
lumiDict = JsonWrapper.loads(lumiDict)
ll = LumiList(compactList=lumiDict)
return ll.getCompactList()
except:
raise WMSpecFactoryException("Could not parse LumiList, %s: %s" % (type(lumiDict), lumiDict))
def wrapper (self, data, expires, contentType = "application/json"):
data = func (self, data)
try:
# jsondata = encoder.iterencode(data)
jsondata = JsonWrapper.dumps(data)
_setCherryPyHeaders(jsondata, contentType, expires)
return jsondata
except:
raise
def wrapper (self, data, expires, contentType = "application/json"):
data = func (self, data)
try:
# jsondata = encoder.iterencode(data)
jsondata = JsonWrapper.dumps(data)
_setCherryPyHeaders(jsondata, contentType, expires)
return jsondata
except:
raise
def sites(siteDbUrl):
""" download a list of all the sites from siteDB """
data = JsonWrapper.loads(urllib.urlopen(siteDbUrl).read().replace("'", '"'))
# kill duplicates, then put in alphabetical order
siteset = set([d['name'] for d in data.values()])
# warning: alliteration
sitelist = list(siteset)
sitelist.sort()
return sitelist
def availableWork(self, conditions, teams = None, wfs = None):
"""Get work which is available to be run"""
elements = []
for site in conditions.keys():
if not conditions[site] > 0:
del conditions[site]
if not conditions:
return elements, conditions
options = {}
options['include_docs'] = True
options['descending'] = True
options['resources'] = conditions
if teams:
options['teams'] = teams
if wfs:
result = []
for i in xrange(0, len(wfs), 20):
options['wfs'] = wfs[i:i+20]
data = self.db.loadList('WorkQueue', 'workRestrictions', 'availableByPriority', options)
result.extend(json.loads(data))
# sort final list
result.sort(key = lambda x: x['WMCore.WorkQueue.DataStructs.WorkQueueElement.WorkQueueElement']['Priority'])
else:
result = self.db.loadList('WorkQueue', 'workRestrictions', 'availableByPriority', options)
result = json.loads(result)
for i in result:
element = CouchWorkQueueElement.fromDocument(self.db, i)
elements.append(element)
# Remove 1st random site that can run work
names = conditions.keys()
random.shuffle(names)
for site in names:
if element.passesSiteRestriction(site):
slots_left = conditions[site] - element['Jobs']
if slots_left > 0:
conditions[site] = slots_left
else:
conditions.pop(site, None)
break
if not conditions:
break
return elements, conditions
def test_json(self):
"""
Test default json implementation.
"""
json_wrap._module = "json"
result = json_wrap.dumps(self.record)
expect = json.dumps(self.record)
self.assertEqual(expect, result)
data = result
result = json_wrap.loads(data)
expect = json.loads(data)
self.assertEqual(expect, result)
try:
json_wrap.loads("bogusbogus")
except ValueError as ex:
self.assertEqual(ex.args, ("No JSON object could be decoded: bogusbogus",))
def updateElements(self, *elementIds, **updatedParams):
"""Update given element's (identified by id) with new parameters"""
if not elementIds:
return
uri = "/" + self.db.name + "/_design/WorkQueue/_update/in-place/"
data = {"updates": json.dumps(updatedParams)}
for ele in elementIds:
thisuri = uri + ele + "?" + urllib.urlencode(data)
self.db.makeRequest(uri=thisuri, type='PUT')
return
def sites(siteDbUrl):
""" download a list of all the sites from siteDB """
data = JsonWrapper.loads(
urllib.urlopen(siteDbUrl).read().replace("'", '"'))
# kill duplicates, then put in alphabetical order
siteset = set([d['name'] for d in data.values()])
# warning: alliteration
sitelist = list(siteset)
sitelist.sort()
return sitelist
def setUp(self):
"""
setUP global values
"""
RESTBaseUnitTest.setUp(self)
reqMgrHost = self.config.getServerUrl()
self.requestSchema = getRequestSchema()
print reqMgrHost
self.jsonSender = JSONRequests(reqMgrHost)
#self.requestTypes = ['ReReco', 'StoreResults', 'CmsGen', 'Reco']
#self.requestTypes = ['ReReco', 'MonteCarlo']
self.requestTypes = ['ReReco']
if 'me' in self.jsonSender.get('user')[0]:
self.jsonSender.delete('user/me')
self.assertFalse('me' in self.jsonSender.get('user')[0])
self.assertEqual(self.jsonSender.put('user/[email protected]')[1], 200)
self.assertTrue('me' in self.jsonSender.get('user')[0])
if 'PeopleLikeMe' in self.jsonSender.get('group')[0]:
self.jsonSender.delete('group/PeopleLikeMe')
self.assertFalse('PeopleLikeMe' in self.jsonSender.get('group')[0])
self.assertEqual(self.jsonSender.put('group/PeopleLikeMe')[1], 200)
self.assertTrue( 'PeopleLikeMe' in self.jsonSender.get('group')[0])
self.jsonSender.put('group/PeopleLikeMe/me')
users = json.loads(self.jsonSender.get('group/PeopleLikeMe')[0])['users']
self.assertTrue('me' in users)
groups = json.loads(self.jsonSender.get('user/me')[0])['groups']
self.assertTrue('PeopleLikeMe' in groups)
groups2 = self.jsonSender.get('group?user=me')[0]
self.assertTrue('PeopleLikeMe' in groups2)
if 'White Sox' in self.jsonSender.get('team')[0]:
self.jsonSender.delete(urllib.quote('team/White Sox'))
self.assertFalse('White Sox' in self.jsonSender.get('team')[0])
self.assertEqual(self.jsonSender.put(urllib.quote('team/White Sox'))[1], 200)
self.assertTrue('White Sox' in self.jsonSender.get('team')[0])
# some foreign key stuff to dealwith
#self.assertFalse('CMSSW_X_Y_Z' in self.jsonSender.get('version')[0])
self.assertTrue(self.jsonSender.put('version/CMSSW_3_5_8')[1] == 200)
self.assertTrue('CMSSW_3_5_8' in self.jsonSender.get('version')[0])
def sites():
# download all the sites from siteDB
url = 'https://cmsweb.cern.ch/sitedb/json/index/CEtoCMSName?name'
data = JsonWrapper.loads(urllib.urlopen(url).read().replace("'", '"'))
# kill duplicates, then put in alphabetical order
siteset = set([d['name'] for d in data.values()])
# warning: alliteration
sitelist = list(siteset)
sitelist.sort()
return sitelist
def updateElements(self, *elementIds, **updatedParams):
"""Update given element's (identified by id) with new parameters"""
import urllib
uri = "/" + self.db.name + "/_design/WorkQueue/_update/in-place/"
data = {"updates": json.dumps(updatedParams)}
for ele in elementIds:
thisuri = uri + ele + "?" + urllib.urlencode(data)
answer = self.db.makeRequest(uri=thisuri, type="PUT")
return
def sites():
# download all the sites from siteDB
url = 'https://cmsweb.cern.ch/sitedb/json/index/CEtoCMSName?name'
data = JsonWrapper.loads(urllib.urlopen(url).read().replace("'", '"'))
# kill duplicates, then put in alphabetical order
siteset = set([d['name'] for d in data.values()])
# warning: alliteration
sitelist = list(siteset)
sitelist.sort()
return sitelist
def test_json(self):
"""
Test default json implementation.
"""
json_wrap._module = "json"
result = json_wrap.dumps(self.record)
expect = json.dumps(self.record)
self.assertEqual(expect, result)
data = result
result = json_wrap.loads(data)
expect = json.loads(data)
self.assertEqual(expect, result)
try:
json_wrap.loads("bogusbogus")
except ValueError, ex:
self.assertEqual(ex.args, ("No JSON object could be decoded: bogusbogus",))
def wrapper (self, data, expires, contentType = "application/json+thunk"):
data = func (self, data)
try:
thunker = JSONThunker()
data = thunker.thunk(data)
jsondata = JsonWrapper.dumps(data)
_setCherryPyHeaders(jsondata, contentType, expires)
return jsondata
except:
raise
def wrapper (self, data, expires, contentType = "application/json+thunk"):
data = func (self, data)
try:
thunker = JSONThunker()
data = thunker.thunk(data)
jsondata = JsonWrapper.dumps(data)
_setCherryPyHeaders(jsondata, contentType, expires)
return jsondata
except:
raise
def setUp(self):
filePath = os.path.join(
getTestBase(),
"WMCore_t/WorkQueue_t/DataStructs_t/wq_available_elements.json")
with open(filePath, "r") as f:
gqData = JsonWrapper.load(f)
self.gqElements = []
for ele in gqData:
self.gqElements.append(WorkQueueElement(**ele))
def setUp(self):
filePath = os.path.join(getTestBase(),
"WMCore_t/WorkQueue_t/DataStructs_t/wq_available_elements.json")
with open(filePath, "r") as f:
gqData = JsonWrapper.load(f)
self.gqElements = []
for ele in gqData:
self.gqElements.append(WorkQueueElement(**ele))
def validateStats(self, index):
""" Check the values for the updates """
if not index.has_key('stats'):
return index
index['stats'] = Utilities.unidecode(JsonWrapper.loads(index['stats']))
for k in ['input_lummis', 'input_num_files',
'input_events', 'total_jobs']:
if k in index['stats']:
index['stats'][k] = int(index['stats'][k])
return index
def validateStats(self, index):
""" Check the values for the updates """
if 'stats' not in index:
return index
index['stats'] = Utilities.unidecode(JsonWrapper.loads(index['stats']))
for k in ['input_lumis', 'input_num_files',
'input_events', 'total_jobs']:
if k in index['stats']:
index['stats'][k] = int(index['stats'][k])
return index
def getElements(self, status = None, elementIDs = None, returnIdOnly = False,
db = None, loadSpec = False, WorkflowName = None, **elementFilters):
"""Return elements that match requirements
status, elementIDs & filters are 'AND'ed together to filter elements.
returnIdOnly causes the element not to be loaded and only the id returned
db is used to specify which database to return from
loadSpec causes the workflow for each spec to be loaded.
WorkflowName may be used in the place of RequestName
"""
key = []
if not db:
db = self.db
if elementFilters.get('RequestName') and not WorkflowName:
WorkflowName = elementFilters.pop('RequestName')
if elementIDs:
if elementFilters or status or returnIdOnly:
raise ValueError, "Can't specify extra filters (or return id's) when using element id's with getElements()"
elements = [CouchWorkQueueElement(db, i).load() for i in elementIDs]
else:
options = {'include_docs' : True, 'filter' : elementFilters, 'idOnly' : returnIdOnly, 'reduce' : False}
# filter on workflow or status if possible
filter = 'elementsByWorkflow'
if WorkflowName:
key.append(WorkflowName)
elif status:
filter = 'elementsByStatus'
key.append(status)
elif elementFilters.get('SubscriptionId'):
key.append(elementFilters['SubscriptionId'])
filter = 'elementsBySubscription'
# add given params to filters
if status:
options['filter']['Status'] = status
if WorkflowName:
options['filter']['RequestName'] = WorkflowName
view = db.loadList('WorkQueue', 'filter', filter, options, key)
view = json.loads(view)
if returnIdOnly:
return view
elements = [CouchWorkQueueElement.fromDocument(db, row) for row in view]
if loadSpec:
specs = {} # cache as may have multiple elements for same spec
for ele in elements:
if ele['RequestName'] not in specs:
wmspec = self.getWMSpec(ele['RequestName'])
specs[ele['RequestName']] = wmspec
ele['WMSpec'] = specs[ele['RequestName']]
del specs
return elements
def set_opts(self, curl, url, params, headers,
ckey=None, cert=None, capath=None, verbose=None, verb='GET', doseq=True, cainfo=None):
"""Set options for given curl object
param needs to be a dictionary in case of GET, while PUT and POST
assume it is a string already encoded/quoted with urllib.encode and
urllib.quote.
"""
curl.setopt(pycurl.NOSIGNAL, self.nosignal)
curl.setopt(pycurl.TIMEOUT, self.timeout)
curl.setopt(pycurl.CONNECTTIMEOUT, self.connecttimeout)
curl.setopt(pycurl.FOLLOWLOCATION, self.followlocation)
curl.setopt(pycurl.MAXREDIRS, self.maxredirs)
if params and verb != 'GET':
if isinstance(params, dict):
params = json.dumps(params)
if verb == 'GET':
encoded_data = urllib.urlencode(params, doseq=doseq)
url = url + '?' + encoded_data
elif verb == 'POST':
curl.setopt(pycurl.POST, 1)
if params:
curl.setopt(pycurl.POSTFIELDS, params)
elif verb == 'DELETE' or verb == 'PUT':
curl.setopt(pycurl.CUSTOMREQUEST, verb)
curl.setopt(pycurl.HTTPHEADER, ['Transfer-Encoding: chunked'])
curl.setopt(pycurl.POSTFIELDS, params)
else:
raise Exception('Unsupported HTTP method "%s"' % verb)
curl.setopt(pycurl.URL, url)
curl.setopt(pycurl.HTTPHEADER, \
["%s: %s" % (k, v) for k, v in headers.items()])
bbuf = StringIO.StringIO()
hbuf = StringIO.StringIO()
curl.setopt(pycurl.WRITEFUNCTION, bbuf.write)
curl.setopt(pycurl.HEADERFUNCTION, hbuf.write)
if capath:
curl.setopt(pycurl.CAPATH, capath)
curl.setopt(pycurl.SSL_VERIFYPEER, True)
if cainfo:
curl.setopt(pycurl.CAINFO, cainfo)
else:
curl.setopt(pycurl.SSL_VERIFYPEER, False)
if ckey:
curl.setopt(pycurl.SSLKEY, ckey)
if cert:
curl.setopt(pycurl.SSLCERT, cert)
if verbose:
curl.setopt(pycurl.VERBOSE, 1)
curl.setopt(pycurl.DEBUGFUNCTION, self.debug)
return bbuf, hbuf
def process_files(self, stage_data = [], request_id=''):
"""
Contact PhEDEx data service to get a list of files for a given request.
TODO: use Service.PhEDEx
"""
db = self.localcouch.connectDatabase('%s_stagequeue' % self.site)
# TODO: make the phedex URL a configurable!
phedex = Requests(url='https://cmsweb.cern.ch', idict={'accept_type':'application/json'})
totalFiles = 0
totalBytes = 0
# Need to clean up the input a bit
for d in stage_data:
qParams = {'node': self.node}
# Need to pass in blocks
if d.find('#') < 0:
qParams['dataset'] = d
else:
qParams['block'] = d
try:
pdata = phedex.get('/phedex/datasvc/json/prod/filereplicas', qParams)[0]
except httplib.HTTPException, he:
self.handleHTTPExcept(he, 'HTTPException for block: %s node: %s' % (d, self.node))
# Parse block info
try:
p = JsonWrapper.loads(pdata)
if len(p['phedex']['block']) == 0:
self.logger.error('Block %s not resident at site %s' % (d, self.node))
else:
self.logger.debug('Creating stage-in requests for %s' % self.node)
for blk in p['phedex']['block']:
self.logger.debug('Creating stage-in requests for block %s' % blk['name'])
for fle in blk['file']:
self.logger.debug('Creating stage-in requests for file %s' % fle['name'])
checksum = fle['checksum']
f = {'_id': fle['name'],
'bytes': int(fle['bytes']),
'checksum': {checksum.split(':')[0]: checksum.split(':')[1]},
'state': 'new',
'retry_count': [],
'request_id': request_id
}
try:
db.queue(f, timestamp = True, viewlist=['stagequeue/file_state'])
totalFiles += 1
totalBytes += f['bytes']
except httplib.HTTPException, he:
self.handleHTTPExcept(he, 'Could not commit data')
except:
self.logger.debug('error loading json')
def test_string_compare(self):
"""
Test that cjson and json libraries do the same thing.
"""
try:
import cjson
except:
raise nose.SkipTest
json_wrap._module = "cjson"
json_wrap._module = "cjson"
cj_result = json_wrap.dumps(self.record)
json_wrap._module = "json"
dj_result = json_wrap.dumps(self.record)
self.assertEqual(dj_result, cj_result)
data = dj_result
json_wrap._module = "cjson"
cj_result = json_wrap.loads(data)
json_wrap._module = "json"
dj_result = json_wrap.loads(data)
self.assertEqual(dj_result, cj_result)
def parse_body(self, data, decode=False):
"""
Parse body part of URL request (by default use json).
This method can be overwritten.
"""
if decode:
try:
res = json.loads(data)
except ValueError as exc:
msg = 'Unable to load JSON data\n%s' % data
raise ValueError(exc)
return res
else:
return data
def jsonHandler(self, args, kwargs):
"""
TODO: Ugly temporary hack to convert unicode to string for kwargs
Only tested on python26 built-in json
"""
#if get verb doesn't have request.boby
#TODO: maybe this should filtered on upper level
if request.body != None:
from WMCore.Wrappers import JsonWrapper
params = request.body.read()
if params:
kw = JsonWrapper.loads(params)
kwargs.update(kw)
return args, kwargs
def abortRequest(requestName):
conn = httplib.HTTPConnection("vocms144.cern.ch:8687")
conn.request("GET", "/reqmgr/reqMgr/request?requestName=%s" % requestName)
resp = conn.getresponse()
status = json_wrap.loads(resp.read())["WMCore.RequestManager.DataStructs.Request.Request"]["RequestStatus"]
print "Status: %s" % status
if status == "acquired":
os.system("curl -X PUT -d \"requestName=%s&status=running\" \"http://vocms144:8687/reqmgr/reqMgr/request\"" % requestName)
os.system("curl -X PUT -d \"requestName=%s&status=aborted\" \"http://vocms144:8687/reqmgr/reqMgr/request\"" % requestName)
else:
os.system("curl -X PUT -d \"requestName=%s&status=rejected\" \"http://vocms144:8687/reqmgr/reqMgr/request\"" % requestName)
return
def loadRequestSchema(workload, requestSchema):
"""
_loadRequestSchema_
Does modifications to the workload I don't understand
Takes a WMWorkloadHelper, operates on it directly with the schema
"""
schema = workload.data.request.section_('schema')
for key, value in requestSchema.iteritems():
if type(value) == dict and key == 'LumiList':
value = JsonWrapper.dumps(value)
try:
setattr(schema, key, value)
except Exception as ex:
# Attach TaskChain tasks
if type(value) == dict and requestSchema[
'RequestType'] == 'TaskChain' and 'Task' in key:
newSec = schema.section_(key)
for k, v in requestSchema[key].iteritems():
if type(value) == dict and key == 'LumiList':
value = JsonWrapper.dumps(value)
try:
setattr(newSec, k, v)
except Exception as ex:
# this logging need to change to cherry py logging
logging.error("Invalid Value: %s" % str(ex))
else:
# this logging need to change to cherry py logging
logging.error("Invalid Value: %s" % str(ex))
schema.timeStamp = int(time.time())
schema = workload.data.request.schema
# might belong in another method to apply existing schema
workload.data.owner.Group = schema.Group
workload.data.owner.Requestor = schema.Requestor
def handleAssignmentPage(self, **kwargs):
""" handler for the main page """
#Accept Json encoded strings
decodedArgs = {}
for key in kwargs.keys():
try:
decodedArgs[key] = JsonWrapper.loads(kwargs[key])
except:
#Probably wasn't JSON
decodedArgs[key] = kwargs[key]
kwargs = decodedArgs
# handle the checkboxes
teams = []
requestNames = []
for key, value in kwargs.iteritems():
if isinstance(value, types.StringTypes):
kwargs[key] = value.strip()
if key.startswith("Team"):
teams.append(key[4:])
if key.startswith("checkbox"):
requestName = key[8:]
self.validate(requestName)
requestNames.append(key[8:])
for requestName in requestNames:
if kwargs['action'] == 'Reject':
ChangeState.changeRequestStatus(requestName,
'rejected',
wmstatUrl=self.wmstatWriteURL)
else:
assignments = GetRequest.getAssignmentsByName(requestName)
if teams == [] and assignments == []:
raise cherrypy.HTTPError(
400, "Must assign to one or more teams")
kwargs["Teams"] = teams
self.assignWorkload(requestName, kwargs)
for team in teams:
if not team in assignments:
ChangeState.assignRequest(
requestName, team, wmstatUrl=self.wmstatWriteURL)
priority = kwargs.get(requestName + ':priority', '')
if priority != '':
Utilities.changePriority(requestName, priority,
self.wmstatWriteURL)
participle = kwargs['action'] + 'ed'
return self.templatepage("Acknowledge",
participle=participle,
requests=requestNames)
def parse_body(self, data, decode=False):
"""
Parse body part of URL request (by default use json).
This method can be overwritten.
"""
if decode:
try:
res = json.loads(data)
except ValueError as exc:
msg = 'Unable to load JSON data, %s, data type=%s, pass as is' \
% (str(exc), type(data))
logging.warning(msg)
return data
return data
else:
return data
def setUp(self):
self.setConfig(config)
self.setCouchDBs([(config.views.data.couch_reqmgr_db, "ReqMgr"),
(config.views.data.couch_reqmgr_aux_db, None)])
self.setSchemaModules([])
RESTBaseUnitTestWithDBBackend.setUp(self)
# put into ReqMgr auxiliary database under "software" document scram/cmsms
# which we'll need a little for request injection
#Warning: this assumes the same structure in jenkins wmcore_root/test
requestPath = os.path.join(getWMBASE(), "test", "data", "ReqMgr", "requests")
mcFile = open(os.path.join(requestPath, "MonteCarlo.json"), 'r')
self.mcArgs = JsonWrapper.load(mcFile)["createRequest"]
cmsswDoc = {"_id": "software"}
cmsswDoc[self.mcArgs["ScramArch"]] = []
cmsswDoc[self.mcArgs["ScramArch"]].append(self.mcArgs["CMSSWVersion"])
insertDataToCouch(os.getenv("COUCHURL"), config.views.data.couch_reqmgr_aux_db, cmsswDoc)
def makeSchema(self, **schema):
schema.setdefault('CouchURL',
Utilities.removePasswordFromUrl(self.couchUrl))
# wrong naming ... but it's all over the place, it's the config cache DB name
schema.setdefault('CouchDBName', self.configDBName)
schema.setdefault('CouchWorkloadDBName', self.workloadDBName)
decodedSchema = {}
for key in schema.keys():
try:
decodedSchema[key] = JsonWrapper.loads(schema[key])
except:
# We don't know what kind of exception we'll get, so ignore them all
# If it does except, it probably wasn't in JSON to begin with.
# Anything else should be caught by the parsers and the validation
decodedSchema[key] = schema[key]
try:
self.info(
"Creating a request for: '%s'\n\tworkloadDB: '%s'\n\twmstatUrl: "
"'%s' ..." %
(decodedSchema, self.workloadDBName,
Utilities.removePasswordFromUrl(self.wmstatWriteURL)))
request = Utilities.makeRequest(self, decodedSchema, self.couchUrl,
self.workloadDBName,
self.wmstatWriteURL)
# catching here KeyError is just terrible
except (RuntimeError, KeyError, Exception) as ex:
# TODO problem not to expose logs to the client
# e.g. on ConfigCacheID not found, the entire CouchDB traceback is sent in ex_message
self.error("Create request failed, reason: %s" % str(ex))
if hasattr(ex, "message"):
if hasattr(ex.message, '__call__'):
detail = ex.message()
else:
detail = str(ex)
elif hasattr(ex, "name"):
detail = ex.name
else:
detail = "check logs."
msg = "Create request failed, %s" % detail
raise cherrypy.HTTPError(400, msg)
baseURL = cherrypy.request.base
raise cherrypy.HTTPRedirect('%s/reqmgr/view/details/%s' %
(baseURL, request['RequestName']))