def test_cjson(self):
"""
Test cjson implementation.
"""
try:
import cjson
except:
raise nose.SkipTest
json_wrap._module = "cjson"
result = json_wrap.dumps(self.record)
expect = json.dumps(self.record)
self.assertEqual(expect, result)
data = result
result = json_wrap.loads(data)
expect = json.loads(data)
self.assertEqual(expect, result)
try:
json_wrap.loads("bogusbogus")
except cjson.DecodeError as ex:
self.assertEqual(ex.args,
("cannot parse JSON description: bogusbogus", ))
def purgeService(self, obj):
"""
Purge job (even bulk) from wms
"""
# Implement as getOutput where the "No output files ..."
# is not an error condition but the expected status
if type(obj) == Job and self.valid(obj.runningJob):
# the object passed is a valid Job
command = "glite-wms-job-output --json --noint --dir /tmp/ " + obj.runningJob["schedulerId"]
out, ret = self.ExecuteCommand(self.proxyString + command)
if ret != 0:
self.logging.error("Purge of %s Failed with exit code =%d" % (str(job.runningJob["schedulerId"]), ret))
self.logging.error("output was \n%s" % out)
elif ret == 0:
loadOut = json.loads(out)
if unicode("result") in loadOut and loadOut[unicode("result")] == unicode("success"):
self.logging.info("Purge of %s successfully done" % str(job.runningJob["schedulerId"]))
else:
self.logging.error("Purge of %s Executed NOT VALIDATED" % str(job.runningJob["schedulerId"]))
self.logging.error(
"Purge of %s Executed with output\n%s" % (str(job.runningJob["schedulerId"]), out)
)
elif type(obj) == Task:
# the object passed is a Task
for job in obj.jobs:
if not self.valid(job.runningJob):
continue
command = "glite-wms-job-output --json --noint --dir /tmp/ " + job.runningJob["schedulerId"]
out, ret = self.ExecuteCommand(self.proxyString + command)
if ret != 0:
self.logging.error(
"Purge of %s Failed with exit code =%d" % (str(job.runningJob["schedulerId"]), ret)
)
self.logging.error("output was \n%s" % out)
elif ret == 0:
loadOut = json.loads(out)
if unicode("result") in loadOut and loadOut[unicode("result")] == unicode("success"):
self.logging.info("Purge of %s successfully done" % str(job.runningJob["schedulerId"]))
else:
self.logging.error("Purge of %s Executed NOT VALIDATED" % str(job.runningJob["schedulerId"]))
self.logging.error(
"Purge of %s Executed with output\n%s" % (str(job.runningJob["schedulerId"]), out)
)
def availableWork(self, conditions, teams=None, wfs=None):
"""Get work which is available to be run"""
elements = []
for site in conditions.keys():
if not conditions[site] > 0:
del conditions[site]
if not conditions:
return elements, conditions
options = {}
options['include_docs'] = True
options['descending'] = True
options['resources'] = conditions
if teams:
options['teams'] = teams
if wfs:
result = []
for i in xrange(0, len(wfs), 20):
options['wfs'] = wfs[i:i + 20]
data = self.db.loadList('WorkQueue', 'workRestrictions',
'availableByPriority', options)
result.extend(json.loads(data))
# sort final list
result.sort(key=lambda x: x[
'WMCore.WorkQueue.DataStructs.WorkQueueElement.WorkQueueElement'
]['Priority'])
else:
result = self.db.loadList('WorkQueue', 'workRestrictions',
'availableByPriority', options)
result = json.loads(result)
for i in result:
element = CouchWorkQueueElement.fromDocument(self.db, i)
elements.append(element)
# Remove 1st random site that can run work
names = conditions.keys()
random.shuffle(names)
for site in names:
if element.passesSiteRestriction(site):
slots_left = conditions[site] - element['Jobs']
if slots_left > 0:
conditions[site] = slots_left
else:
conditions.pop(site, None)
break
if not conditions:
break
return elements, conditions
def test_json(self):
"""
Test default json implementation.
"""
result = json_wrap.dumps(self.record)
expect = json.dumps(self.record)
self.assertEqual(expect, result)
data = result
result = json_wrap.loads(data)
expect = json.loads(data)
self.assertEqual(expect, result)
with self.assertRaises(ValueError):
json_wrap.loads("bogusbogus")
def makeSchema(self, **schema):
schema.setdefault('CouchURL', Utilities.removePasswordFromUrl(self.couchUrl))
schema.setdefault('CouchDBName', self.configDBName)
decodedSchema = {}
for key in schema.keys():
try:
decodedSchema[key] = JsonWrapper.loads(schema[key])
except:
# We don't know what kind of exception we'll get, so ignore them all
# If it does except, it probably wasn't in JSON to begin with.
# Anything else should be caught by the parsers and the validation
decodedSchema[key] = schema[key]
try:
self.info("Creating a request for: '%s'\n\tworkloadDB: '%s'\n\twmstatUrl: "
"'%s' ..." % (decodedSchema, self.workloadDBName,
Utilities.removePasswordFromUrl(self.wmstatWriteURL)))
request = Utilities.makeRequest(self, decodedSchema, self.couchUrl, self.workloadDBName, self.wmstatWriteURL)
# catching here KeyError is just terrible
except (RuntimeError, KeyError, Exception) as ex:
# TODO problem not to expose logs to the client
# e.g. on ConfigCacheID not found, the entire CouchDB traceback is sent in ex_message
self.error("Create request failed, reason: %s" % ex)
if hasattr(ex, "name"):
detail = ex.name
else:
detail = "check logs."
msg = "Create request failed, %s" % detail
raise cherrypy.HTTPError(400, msg)
baseURL = cherrypy.request.base
raise cherrypy.HTTPRedirect('%s/reqmgr/view/details/%s' % (baseURL, request['RequestName']))
def _getResult(self, callname, clearCache=True, args=None, verb="GET"):
"""
_getResult_
retrieve JSON/XML formatted information given the service name and the
argument dictionaries
TODO: Probably want to move this up into Service
"""
result = ''
file = callname.replace("/", "_")
if clearCache:
self.clearCache(file, args, verb)
f = self.refreshCache(file, callname, args, verb=verb)
result = f.read()
f.close()
# if self.responseType == "json":
# decoder = json.JSONDecoder()
# return decoder.decode(result)
if result:
result = JsonWrapper.loads(result)
return result
def availableWork(self, conditions, teams = None, wfs = None):
"""Get work which is available to be run"""
elements = []
for site in conditions.keys():
if not conditions[site] > 0:
del conditions[site]
if not conditions:
return elements, conditions
options = {}
options['include_docs'] = True
options['descending'] = True
options['resources'] = conditions
if teams:
options['teams'] = teams
if wfs:
options['wfs'] = wfs
result = self.db.loadList('WorkQueue', 'workRestrictions', 'availableByPriority', options)
result = json.loads(result)
for i in result:
element = CouchWorkQueueElement.fromDocument(self.db, i)
elements.append(element)
# Remove 1st random site that can run work
names = conditions.keys()
random.shuffle(names)
for site in names:
if element.passesSiteRestriction(site):
slots_left = conditions[site] - element['Jobs']
if slots_left > 0:
conditions[site] = slots_left
else:
conditions.pop(site, None)
break
return elements, conditions
def _validateMultiRequests(self, param, safe, valFunc):
data = cherrypy.request.body.read()
if data:
request_names, request_args = self._getMultiRequestArgs(JsonWrapper.loads(data))
else:
# actually this is error case
# cherrypy.log(str(param.kwargs))
request_names, request_args = self._getMultiRequestArgs(param.kwargs)
for prop in request_args:
if prop == "RequestStatus":
del param.kwargs["new_status"]
else:
del param.kwargs[prop]
del param.kwargs["ids"]
# remove this
# tmp = []
# for prop in param.kwargs:
# tmp.append(prop)
# for prop in tmp:
# del param.kwargs[prop]
safe.kwargs['workload_pair_list'] = []
for request_name in request_names:
request_args["RequestName"] = request_name
workload, r_args = valFunc(request_args, self.config, self.reqmgr_db_service, param)
safe.kwargs['workload_pair_list'].append((workload, r_args))
safe.kwargs["multi_update_flag"] = True
def _validateRequestBase(self, param, safe, valFunc, requestName=None):
data = cherrypy.request.body.read()
if data:
request_args = JsonWrapper.loads(data)
if requestName:
request_args["RequestName"] = requestName
if isinstance(request_args, dict):
request_args = [request_args]
else:
# actually this is error case
# cherrypy.log(str(param.kwargs))
request_args = {}
for prop in param.kwargs:
request_args[prop] = param.kwargs[prop]
for prop in request_args:
del param.kwargs[prop]
if requestName:
request_args["RequestName"] = requestName
request_args = [request_args]
safe.kwargs['workload_pair_list'] = []
if isinstance(request_args, dict):
request_args = [request_args]
for args in request_args:
workload, r_args = valFunc(args, self.config, self.reqmgr_db_service, param)
safe.kwargs['workload_pair_list'].append((workload, r_args))
def _validateMultiRequests(self, param, safe, valFunc):
data = cherrypy.request.body.read()
if data:
request_names, request_args = self._getMultiRequestArgs(JsonWrapper.loads(data))
else:
# actually this is error case
#cherrypy.log(str(param.kwargs))
request_names, request_args = self._getMultiRequestArgs(param.kwargs)
for prop in request_args:
if prop == "RequestStatus":
del param.kwargs["new_status"]
else:
del param.kwargs[prop]
del param.kwargs["ids"]
#remove this
#tmp = []
#for prop in param.kwargs:
# tmp.append(prop)
#for prop in tmp:
# del param.kwargs[prop]
safe.kwargs['workload_pair_list'] = []
for request_name in request_names:
request_args["RequestName"] = request_name
workload, r_args = valFunc(request_args, self.config, self.reqmgr_db_service, param)
safe.kwargs['workload_pair_list'].append((workload, r_args))
safe.kwargs["multi_update_flag"] = True
def _validateRequestBase(self, param, safe, valFunc, requestName=None):
data = cherrypy.request.body.read()
if data:
request_args = JsonWrapper.loads(data)
if requestName:
request_args["RequestName"] = requestName
if isinstance(request_args, dict):
request_args = [request_args]
else:
# actually this is error case
#cherrypy.log(str(param.kwargs))
request_args = {}
for prop in param.kwargs:
request_args[prop] = param.kwargs[prop]
for prop in request_args:
del param.kwargs[prop]
if requestName:
request_args["RequestName"] = requestName
request_args = [request_args]
safe.kwargs['workload_pair_list'] = []
if isinstance(request_args, dict):
request_args = [request_args]
for args in request_args:
workload, r_args = valFunc(args, self.config,
self.reqmgr_db_service, param)
safe.kwargs['workload_pair_list'].append((workload, r_args))
def _getResult(self, callname, clearCache = True,
args = None, verb = "GET"):
"""
_getResult_
retrieve JSON/XML formatted information given the service name and the
argument dictionaries
TODO: Probably want to move this up into Service
"""
result = ''
file = callname.replace("/", "_")
if clearCache:
self.clearCache(file, args, verb)
f = self.refreshCache(file, callname, args, verb = verb)
result = f.read()
f.close()
# if self.responseType == "json":
# decoder = json.JSONDecoder()
# return decoder.decode(result)
if result:
result = JsonWrapper.loads(result)
return result
def check_resident(self, site, stage_data):
"""
Checks that the requested data is resident on the site
"""
goodToGo = True
# Format site name
locSite = site.replace('_Buffer', '_MSS')
if not locSite.endswith('_MSS'):
locSite += '_MSS'
qParams = {'node': locSite}
# Get block info from PhEDEx
phedex = Requests(url='https://cmsweb.cern.ch', idict={'accept_type':'application/json'})
for data in stage_data:
if data.find('#') < 0:
qParams['dataset'] = data
else:
qParams['block'] = data
self.logger.debug('Checking data residency for %s at %s' % (data, locSite))
try:
pdata = phedex.get('/phedex/datasvc/json/prod/blockreplicas', qParams)[0]
except httplib.HTTPException, he:
self.handleHTTPExcept(he, 'HTTPException for block: %s node: %s' % (data, locSite))
# Parse block info and check > 0 block exist
try:
if len(JsonWrapper.loads(pdata)['phedex']['block']) == 0:
goodToGo = False
self.logger.error('Block %s not resident at site %s' % (data, locSite))
except:
self.logger.debug('error loading json')
goodToGo = False
def putRequest(self, requestName=None, status=None, priority=None):
request = None
if requestName:
request = self.findRequest(requestName)
if request == None:
# Create a new request, with a JSON-encoded schema that is
# sent in the body of the HTTP request
body = cherrypy.request.body.read()
reqInputArgs = Utilities.unidecode(JsonWrapper.loads(body))
reqInputArgs.setdefault(
'CouchURL', Utilities.removePasswordFromUrl(self.couchUrl))
reqInputArgs.setdefault('CouchDBName', self.configDBName)
try:
self.info(
"Creating a request for: '%s'\n\tworkloadDB: '%s'\n\twmstatUrl: "
"'%s' ..." %
(reqInputArgs, self.workloadDBName,
Utilities.removePasswordFromUrl(self.wmstatWriteURL)))
request = Utilities.makeRequest(self, reqInputArgs,
self.couchUrl,
self.workloadDBName,
self.wmstatWriteURL)
except cherrypy.HTTPError as ex:
self.error("Create request failed, reason: %s" % ex)
# Assume that this is a valid HTTPError
raise
except (WMException, Exception) as ex:
# TODO problem not to expose logs to the client
# e.g. on ConfigCacheID not found, the entire CouchDB traceback is sent in ex_message
self.error("Create request failed, reason: %s" % ex)
if hasattr(ex, "name"):
detail = ex.name
else:
detail = "check logs."
msg = "Create request failed, %s" % detail
raise cherrypy.HTTPError(400, msg)
self.info("Request '%s' created." % request['RequestName'])
# see if status & priority need to be upgraded
if status != None:
# forbid assignment here
if status == 'assigned':
raise cherrypy.HTTPError(
403,
"Cannot change status without a team. Please use PUT /reqmgr/reqMgr/assignment/<team>/<requestName>"
)
try:
Utilities.changeStatus(requestName, status,
self.wmstatWriteURL)
except RuntimeError as ex:
# ignore some of these errors: https://svnweb.cern.ch/trac/CMSDMWM/ticket/2002
if status != 'announced' and status != 'closed-out':
self.error("RuntimeError while changeStatus: reason: %s" %
ex)
raise cherrypy.HTTPError(
403, "Failed to change status: %s" % str(ex))
if priority != None:
Utilities.changePriority(requestName, priority,
self.wmstatWriteURL)
return request
def makeLumiList(lumiDict):
try:
if isinstance(lumiDict, basestring):
lumiDict = JsonWrapper.loads(lumiDict)
ll = LumiList(compactList=lumiDict)
return ll.getCompactList()
except:
raise WMSpecFactoryException("Could not parse LumiList, %s: %s" % (type(lumiDict), lumiDict))
def sites(siteDbUrl):
""" download a list of all the sites from siteDB """
data = JsonWrapper.loads(urllib.urlopen(siteDbUrl).read().replace("'", '"'))
# kill duplicates, then put in alphabetical order
siteset = set([d['name'] for d in data.values()])
# warning: alliteration
sitelist = list(siteset)
sitelist.sort()
return sitelist
def availableWork(self, conditions, teams = None, wfs = None):
"""Get work which is available to be run"""
elements = []
for site in conditions.keys():
if not conditions[site] > 0:
del conditions[site]
if not conditions:
return elements, conditions
options = {}
options['include_docs'] = True
options['descending'] = True
options['resources'] = conditions
if teams:
options['teams'] = teams
if wfs:
result = []
for i in xrange(0, len(wfs), 20):
options['wfs'] = wfs[i:i+20]
data = self.db.loadList('WorkQueue', 'workRestrictions', 'availableByPriority', options)
result.extend(json.loads(data))
# sort final list
result.sort(key = lambda x: x['WMCore.WorkQueue.DataStructs.WorkQueueElement.WorkQueueElement']['Priority'])
else:
result = self.db.loadList('WorkQueue', 'workRestrictions', 'availableByPriority', options)
result = json.loads(result)
for i in result:
element = CouchWorkQueueElement.fromDocument(self.db, i)
elements.append(element)
# Remove 1st random site that can run work
names = conditions.keys()
random.shuffle(names)
for site in names:
if element.passesSiteRestriction(site):
slots_left = conditions[site] - element['Jobs']
if slots_left > 0:
conditions[site] = slots_left
else:
conditions.pop(site, None)
break
if not conditions:
break
return elements, conditions
def sites():
# download all the sites from siteDB
url = 'https://cmsweb.cern.ch/sitedb/json/index/CEtoCMSName?name'
data = JsonWrapper.loads(urllib.urlopen(url).read().replace("'", '"'))
# kill duplicates, then put in alphabetical order
siteset = set([d['name'] for d in data.values()])
# warning: alliteration
sitelist = list(siteset)
sitelist.sort()
return sitelist
def test_json(self):
"""
Test default json implementation.
"""
json_wrap._module = "json"
result = json_wrap.dumps(self.record)
expect = json.dumps(self.record)
self.assertEqual(expect, result)
data = result
result = json_wrap.loads(data)
expect = json.loads(data)
self.assertEqual(expect, result)
try:
json_wrap.loads("bogusbogus")
except ValueError, ex:
self.assertEqual(ex.args, ("No JSON object could be decoded: bogusbogus",))
def setUp(self):
"""
setUP global values
"""
RESTBaseUnitTest.setUp(self)
reqMgrHost = self.config.getServerUrl()
self.requestSchema = getRequestSchema()
print reqMgrHost
self.jsonSender = JSONRequests(reqMgrHost)
#self.requestTypes = ['ReReco', 'StoreResults', 'CmsGen', 'Reco']
#self.requestTypes = ['ReReco', 'MonteCarlo']
self.requestTypes = ['ReReco']
if 'me' in self.jsonSender.get('user')[0]:
self.jsonSender.delete('user/me')
self.assertFalse('me' in self.jsonSender.get('user')[0])
self.assertEqual(self.jsonSender.put('user/[email protected]')[1], 200)
self.assertTrue('me' in self.jsonSender.get('user')[0])
if 'PeopleLikeMe' in self.jsonSender.get('group')[0]:
self.jsonSender.delete('group/PeopleLikeMe')
self.assertFalse('PeopleLikeMe' in self.jsonSender.get('group')[0])
self.assertEqual(self.jsonSender.put('group/PeopleLikeMe')[1], 200)
self.assertTrue( 'PeopleLikeMe' in self.jsonSender.get('group')[0])
self.jsonSender.put('group/PeopleLikeMe/me')
users = json.loads(self.jsonSender.get('group/PeopleLikeMe')[0])['users']
self.assertTrue('me' in users)
groups = json.loads(self.jsonSender.get('user/me')[0])['groups']
self.assertTrue('PeopleLikeMe' in groups)
groups2 = self.jsonSender.get('group?user=me')[0]
self.assertTrue('PeopleLikeMe' in groups2)
if 'White Sox' in self.jsonSender.get('team')[0]:
self.jsonSender.delete(urllib.quote('team/White Sox'))
self.assertFalse('White Sox' in self.jsonSender.get('team')[0])
self.assertEqual(self.jsonSender.put(urllib.quote('team/White Sox'))[1], 200)
self.assertTrue('White Sox' in self.jsonSender.get('team')[0])
# some foreign key stuff to dealwith
#self.assertFalse('CMSSW_X_Y_Z' in self.jsonSender.get('version')[0])
self.assertTrue(self.jsonSender.put('version/CMSSW_3_5_8')[1] == 200)
self.assertTrue('CMSSW_3_5_8' in self.jsonSender.get('version')[0])
def sites():
# download all the sites from siteDB
url = 'https://cmsweb.cern.ch/sitedb/json/index/CEtoCMSName?name'
data = JsonWrapper.loads(urllib.urlopen(url).read().replace("'", '"'))
# kill duplicates, then put in alphabetical order
siteset = set([d['name'] for d in data.values()])
# warning: alliteration
sitelist = list(siteset)
sitelist.sort()
return sitelist
def test_json(self):
"""
Test default json implementation.
"""
json_wrap._module = "json"
result = json_wrap.dumps(self.record)
expect = json.dumps(self.record)
self.assertEqual(expect, result)
data = result
result = json_wrap.loads(data)
expect = json.loads(data)
self.assertEqual(expect, result)
try:
json_wrap.loads("bogusbogus")
except ValueError as ex:
self.assertEqual(ex.args, ("No JSON object could be decoded: bogusbogus",))
def sites(siteDbUrl):
""" download a list of all the sites from siteDB """
data = JsonWrapper.loads(
urllib.urlopen(siteDbUrl).read().replace("'", '"'))
# kill duplicates, then put in alphabetical order
siteset = set([d['name'] for d in data.values()])
# warning: alliteration
sitelist = list(siteset)
sitelist.sort()
return sitelist
def validateStats(self, index):
""" Check the values for the updates """
if 'stats' not in index:
return index
index['stats'] = Utilities.unidecode(JsonWrapper.loads(index['stats']))
for k in ['input_lumis', 'input_num_files',
'input_events', 'total_jobs']:
if k in index['stats']:
index['stats'][k] = int(index['stats'][k])
return index
def validateStats(self, index):
""" Check the values for the updates """
if not index.has_key('stats'):
return index
index['stats'] = Utilities.unidecode(JsonWrapper.loads(index['stats']))
for k in ['input_lummis', 'input_num_files',
'input_events', 'total_jobs']:
if k in index['stats']:
index['stats'][k] = int(index['stats'][k])
return index
def process_files(self, stage_data = [], request_id=''):
"""
Contact PhEDEx data service to get a list of files for a given request.
TODO: use Service.PhEDEx
"""
db = self.localcouch.connectDatabase('%s_stagequeue' % self.site)
# TODO: make the phedex URL a configurable!
phedex = Requests(url='https://cmsweb.cern.ch', idict={'accept_type':'application/json'})
totalFiles = 0
totalBytes = 0
# Need to clean up the input a bit
for d in stage_data:
qParams = {'node': self.node}
# Need to pass in blocks
if d.find('#') < 0:
qParams['dataset'] = d
else:
qParams['block'] = d
try:
pdata = phedex.get('/phedex/datasvc/json/prod/filereplicas', qParams)[0]
except httplib.HTTPException, he:
self.handleHTTPExcept(he, 'HTTPException for block: %s node: %s' % (d, self.node))
# Parse block info
try:
p = JsonWrapper.loads(pdata)
if len(p['phedex']['block']) == 0:
self.logger.error('Block %s not resident at site %s' % (d, self.node))
else:
self.logger.debug('Creating stage-in requests for %s' % self.node)
for blk in p['phedex']['block']:
self.logger.debug('Creating stage-in requests for block %s' % blk['name'])
for fle in blk['file']:
self.logger.debug('Creating stage-in requests for file %s' % fle['name'])
checksum = fle['checksum']
f = {'_id': fle['name'],
'bytes': int(fle['bytes']),
'checksum': {checksum.split(':')[0]: checksum.split(':')[1]},
'state': 'new',
'retry_count': [],
'request_id': request_id
}
try:
db.queue(f, timestamp = True, viewlist=['stagequeue/file_state'])
totalFiles += 1
totalBytes += f['bytes']
except httplib.HTTPException, he:
self.handleHTTPExcept(he, 'Could not commit data')
except:
self.logger.debug('error loading json')
def getElements(self, status = None, elementIDs = None, returnIdOnly = False,
db = None, loadSpec = False, WorkflowName = None, **elementFilters):
"""Return elements that match requirements
status, elementIDs & filters are 'AND'ed together to filter elements.
returnIdOnly causes the element not to be loaded and only the id returned
db is used to specify which database to return from
loadSpec causes the workflow for each spec to be loaded.
WorkflowName may be used in the place of RequestName
"""
key = []
if not db:
db = self.db
if elementFilters.get('RequestName') and not WorkflowName:
WorkflowName = elementFilters.pop('RequestName')
if elementIDs:
if elementFilters or status or returnIdOnly:
raise ValueError, "Can't specify extra filters (or return id's) when using element id's with getElements()"
elements = [CouchWorkQueueElement(db, i).load() for i in elementIDs]
else:
options = {'include_docs' : True, 'filter' : elementFilters, 'idOnly' : returnIdOnly, 'reduce' : False}
# filter on workflow or status if possible
filter = 'elementsByWorkflow'
if WorkflowName:
key.append(WorkflowName)
elif status:
filter = 'elementsByStatus'
key.append(status)
elif elementFilters.get('SubscriptionId'):
key.append(elementFilters['SubscriptionId'])
filter = 'elementsBySubscription'
# add given params to filters
if status:
options['filter']['Status'] = status
if WorkflowName:
options['filter']['RequestName'] = WorkflowName
view = db.loadList('WorkQueue', 'filter', filter, options, key)
view = json.loads(view)
if returnIdOnly:
return view
elements = [CouchWorkQueueElement.fromDocument(db, row) for row in view]
if loadSpec:
specs = {} # cache as may have multiple elements for same spec
for ele in elements:
if ele['RequestName'] not in specs:
wmspec = self.getWMSpec(ele['RequestName'])
specs[ele['RequestName']] = wmspec
ele['WMSpec'] = specs[ele['RequestName']]
del specs
return elements
def handleAssignmentPage(self, **kwargs):
""" handler for the main page """
#Accept Json encoded strings
decodedArgs = {}
for key in kwargs.keys():
try:
decodedArgs[key] = JsonWrapper.loads(kwargs[key])
except:
#Probably wasn't JSON
decodedArgs[key] = kwargs[key]
kwargs = decodedArgs
# handle the checkboxes
teams = []
requestNames = []
for key, value in kwargs.iteritems():
if isinstance(value, types.StringTypes):
kwargs[key] = value.strip()
if key.startswith("Team"):
teams.append(key[4:])
if key.startswith("checkbox"):
requestName = key[8:]
self.validate(requestName)
requestNames.append(key[8:])
for requestName in requestNames:
if kwargs['action'] == 'Reject':
ChangeState.changeRequestStatus(requestName, 'rejected', wmstatUrl = self.wmstatWriteURL)
else:
assignments = GetRequest.getAssignmentsByName(requestName)
if teams == [] and assignments == []:
raise cherrypy.HTTPError(400, "Must assign to one or more teams")
self.assignWorkload(requestName, kwargs)
for team in teams:
if not team in assignments:
ChangeState.assignRequest(requestName, team, wmstatUrl = self.wmstatWriteURL)
priority = kwargs.get(requestName+':priority', '')
if priority != '':
Utilities.changePriority(requestName, priority, self.wmstatWriteURL)
participle=kwargs['action']+'ed'
if self.opshold and kwargs['action'] == 'Assign':
participle='put into "ops-hold" state (see <a href="%s">OpsClipboard</a>)' % self.clipboardUrl
# this, previously used, call made all requests injected into OpsClipboard to
# have campaign_id null since the call doesn't propagate request's
# CampaignName at all, AcquisitionEra remains default null and probably
# a bunch of other things is wrong too
#requests = [GetRequest.getRequestByName(requestName) for requestName in requestNames]
requests = [Utilities.requestDetails(requestName) for requestName in requestNames]
OpsClipboard.inject(self.couchUrl, self.clipboardDB, *requests)
for request in requestNames:
ChangeState.changeRequestStatus(requestName, 'ops-hold', wmstatUrl = self.wmstatWriteURL)
return self.templatepage("Acknowledge", participle=participle, requests=requestNames)
def test_string_compare(self):
"""
Test that cjson and json libraries do the same thing.
"""
try:
import cjson
except:
raise nose.SkipTest
json_wrap._module = "cjson"
json_wrap._module = "cjson"
cj_result = json_wrap.dumps(self.record)
json_wrap._module = "json"
dj_result = json_wrap.dumps(self.record)
self.assertEqual(dj_result, cj_result)
data = dj_result
json_wrap._module = "cjson"
cj_result = json_wrap.loads(data)
json_wrap._module = "json"
dj_result = json_wrap.loads(data)
self.assertEqual(dj_result, cj_result)
def parse_body(self, data, decode=False):
"""
Parse body part of URL request (by default use json).
This method can be overwritten.
"""
if decode:
try:
res = json.loads(data)
except ValueError as exc:
msg = 'Unable to load JSON data\n%s' % data
raise ValueError(exc)
return res
else:
return data
def test_string_compare(self):
"""
Test that cjson and json libraries do the same thing.
"""
try:
import cjson
except:
raise nose.SkipTest
json_wrap._module = "cjson"
json_wrap._module = "cjson"
cj_result = json_wrap.dumps(self.record)
json_wrap._module = "json"
dj_result = json_wrap.dumps(self.record)
self.assertEqual(dj_result, cj_result)
data = dj_result
json_wrap._module = "cjson"
cj_result = json_wrap.loads(data)
json_wrap._module = "json"
dj_result = json_wrap.loads(data)
self.assertEqual(dj_result, cj_result)
def jsonHandler(self, args, kwargs):
"""
TODO: Ugly temporary hack to convert unicode to string for kwargs
Only tested on python26 built-in json
"""
#if get verb doesn't have request.boby
#TODO: maybe this should filtered on upper level
if request.body != None:
from WMCore.Wrappers import JsonWrapper
params = request.body.read()
if params:
kw = JsonWrapper.loads(params)
kwargs.update(kw)
return args, kwargs
def parse_body(self, data, decode=False):
"""
Parse body part of URL request (by default use json).
This method can be overwritten.
"""
if decode:
try:
res = json.loads(data)
except ValueError as exc:
msg = 'Unable to load JSON data\n%s' % data
raise ValueError(exc)
return res
else:
return data
def test_cjson(self):
"""
Test cjson implementation.
"""
try:
import cjson
except:
raise nose.SkipTest
json_wrap._module = "cjson"
result = json_wrap.dumps(self.record)
expect = json.dumps(self.record)
self.assertEqual(expect, result)
data = result
result = json_wrap.loads(data)
expect = json.loads(data)
self.assertEqual(expect, result)
try:
json_wrap.loads("bogusbogus")
except cjson.DecodeError as ex:
self.assertEqual(ex.args, ("cannot parse JSON description: bogusbogus",))
def abortRequest(requestName):
conn = httplib.HTTPConnection("vocms144.cern.ch:8687")
conn.request("GET", "/reqmgr/reqMgr/request?requestName=%s" % requestName)
resp = conn.getresponse()
status = json_wrap.loads(resp.read())["WMCore.RequestManager.DataStructs.Request.Request"]["RequestStatus"]
print "Status: %s" % status
if status == "acquired":
os.system("curl -X PUT -d \"requestName=%s&status=running\" \"http://vocms144:8687/reqmgr/reqMgr/request\"" % requestName)
os.system("curl -X PUT -d \"requestName=%s&status=aborted\" \"http://vocms144:8687/reqmgr/reqMgr/request\"" % requestName)
else:
os.system("curl -X PUT -d \"requestName=%s&status=rejected\" \"http://vocms144:8687/reqmgr/reqMgr/request\"" % requestName)
return
def abortRequest(requestName):
conn = httplib.HTTPConnection("vocms144.cern.ch:8687")
conn.request("GET", "/reqmgr/reqMgr/request?requestName=%s" % requestName)
resp = conn.getresponse()
status = json_wrap.loads(resp.read())["WMCore.RequestManager.DataStructs.Request.Request"]["RequestStatus"]
print "Status: %s" % status
if status == "acquired":
os.system("curl -X PUT -d \"requestName=%s&status=running\" \"http://vocms144:8687/reqmgr/reqMgr/request\"" % requestName)
os.system("curl -X PUT -d \"requestName=%s&status=aborted\" \"http://vocms144:8687/reqmgr/reqMgr/request\"" % requestName)
else:
os.system("curl -X PUT -d \"requestName=%s&status=rejected\" \"http://vocms144:8687/reqmgr/reqMgr/request\"" % requestName)
return
def handleAssignmentPage(self, **kwargs):
""" handler for the main page """
#Accept Json encoded strings
decodedArgs = {}
for key in kwargs.keys():
try:
decodedArgs[key] = JsonWrapper.loads(kwargs[key])
except:
#Probably wasn't JSON
decodedArgs[key] = kwargs[key]
kwargs = decodedArgs
# handle the checkboxes
teams = []
requestNames = []
for key, value in kwargs.iteritems():
if isinstance(value, types.StringTypes):
kwargs[key] = value.strip()
if key.startswith("Team"):
teams.append(key[4:])
if key.startswith("checkbox"):
requestName = key[8:]
self.validate(requestName)
requestNames.append(key[8:])
for requestName in requestNames:
if kwargs['action'] == 'Reject':
ChangeState.changeRequestStatus(requestName,
'rejected',
wmstatUrl=self.wmstatWriteURL)
else:
assignments = GetRequest.getAssignmentsByName(requestName)
if teams == [] and assignments == []:
raise cherrypy.HTTPError(
400, "Must assign to one or more teams")
kwargs["Teams"] = teams
self.assignWorkload(requestName, kwargs)
for team in teams:
if not team in assignments:
ChangeState.assignRequest(
requestName, team, wmstatUrl=self.wmstatWriteURL)
priority = kwargs.get(requestName + ':priority', '')
if priority != '':
Utilities.changePriority(requestName, priority,
self.wmstatWriteURL)
participle = kwargs['action'] + 'ed'
return self.templatepage("Acknowledge",
participle=participle,
requests=requestNames)
def parse_body(self, data, decode=False):
"""
Parse body part of URL request (by default use json).
This method can be overwritten.
"""
if decode:
try:
res = json.loads(data)
except ValueError as exc:
msg = 'Unable to load JSON data, %s, data type=%s, pass as is' \
% (str(exc), type(data))
logging.warning(msg)
return data
return data
else:
return data
def putRequest(self, requestName=None, status=None, priority=None):
""" Checks the request n the body with one arg, and changes the status with kwargs """
request = None
if requestName:
request = self.findRequest(requestName)
if request == None:
""" Creates a new request, with a JSON-encoded schema that is sent in the
body of the request """
body = cherrypy.request.body.read()
schema = Utilities.unidecode(JsonWrapper.loads(body))
schema.setdefault('CouchURL', Utilities.removePasswordFromUrl(self.couchUrl))
schema.setdefault('CouchDBName', self.configDBName)
try:
request = Utilities.makeRequest(schema, self.couchUrl, self.workloadDBName)
except RuntimeError, ex:
raise cherrypy.HTTPError(400, ex.message)
def parse_body(self, data, decode=False):
"""
Parse body part of URL request (by default use json).
This method can be overwritten.
"""
if decode:
try:
res = json.loads(data)
except ValueError as exc:
msg = 'Unable to load JSON data, %s, data type=%s, pass as is' \
% (str(exc), type(data))
logging.warning(msg)
return data
return res
else:
return data
def putRequest(self, requestName=None, status=None, priority=None):
request = None
if requestName:
request = self.findRequest(requestName)
if request == None:
# Create a new request, with a JSON-encoded schema that is
# sent in the body of the HTTP request
body = cherrypy.request.body.read()
reqInputArgs = Utilities.unidecode(JsonWrapper.loads(body))
reqInputArgs.setdefault('CouchURL', Utilities.removePasswordFromUrl(self.couchUrl))
reqInputArgs.setdefault('CouchDBName', self.configDBName)
try:
self.info("Creating a request for: '%s'\n\tworkloadDB: '%s'\n\twmstatUrl: "
"'%s' ..." % (reqInputArgs, self.workloadDBName,
Utilities.removePasswordFromUrl(self.wmstatWriteURL)))
request = Utilities.makeRequest(self, reqInputArgs, self.couchUrl,
self.workloadDBName, self.wmstatWriteURL)
except cherrypy.HTTPError as ex:
self.error("Create request failed, reason: %s" % ex)
# Assume that this is a valid HTTPError
raise
except (WMException, Exception) as ex:
# TODO problem not to expose logs to the client
# e.g. on ConfigCacheID not found, the entire CouchDB traceback is sent in ex_message
self.error("Create request failed, reason: %s" % ex)
if hasattr(ex, "name"):
detail = ex.name
else:
detail = "check logs."
msg = "Create request failed, %s" % detail
raise cherrypy.HTTPError(400, msg)
self.info("Request '%s' created." % request['RequestName'])
# see if status & priority need to be upgraded
if status != None:
# forbid assignment here
if status == 'assigned':
raise cherrypy.HTTPError(403, "Cannot change status without a team. Please use PUT /reqmgr/reqMgr/assignment/<team>/<requestName>")
try:
Utilities.changeStatus(requestName, status, self.wmstatWriteURL)
except RuntimeError as ex:
# ignore some of these errors: https://svnweb.cern.ch/trac/CMSDMWM/ticket/2002
if status != 'announced' and status != 'closed-out':
self.error("RuntimeError while changeStatus: reason: %s" % ex)
raise cherrypy.HTTPError(403, "Failed to change status: %s" % str(ex))
if priority != None:
Utilities.changePriority(requestName, priority, self.wmstatWriteURL)
return request
def makeSchema(self, **schema):
schema.setdefault('CouchURL', Utilities.removePasswordFromUrl(self.couchUrl))
schema.setdefault('CouchDBName', self.configDBName)
decodedSchema = {}
for key in schema.keys():
try:
decodedSchema[key] = JsonWrapper.loads(schema[key])
except:
# We don't know what kind of exception we'll get, so ignore them all
# If it does except, it probably wasn't in JSON to begin with.
# Anything else should be caught by the parsers and the validation
decodedSchema[key] = schema[key]
try:
request = Utilities.makeRequest(decodedSchema, self.couchUrl, self.workloadDBName)
except RuntimeError, e:
raise cherrypy.HTTPError(400, "Error creating request: %s" % e)
def makeSchema(self, **schema):
schema.setdefault('CouchURL',
Utilities.removePasswordFromUrl(self.couchUrl))
# wrong naming ... but it's all over the place, it's the config cache DB name
schema.setdefault('CouchDBName', self.configDBName)
schema.setdefault('CouchWorkloadDBName', self.workloadDBName)
decodedSchema = {}
for key in schema.keys():
try:
decodedSchema[key] = JsonWrapper.loads(schema[key])
except:
# We don't know what kind of exception we'll get, so ignore them all
# If it does except, it probably wasn't in JSON to begin with.
# Anything else should be caught by the parsers and the validation
decodedSchema[key] = schema[key]
try:
self.info(
"Creating a request for: '%s'\n\tworkloadDB: '%s'\n\twmstatUrl: "
"'%s' ..." %
(decodedSchema, self.workloadDBName,
Utilities.removePasswordFromUrl(self.wmstatWriteURL)))
request = Utilities.makeRequest(self, decodedSchema, self.couchUrl,
self.workloadDBName,
self.wmstatWriteURL)
# catching here KeyError is just terrible
except (RuntimeError, KeyError, Exception) as ex:
# TODO problem not to expose logs to the client
# e.g. on ConfigCacheID not found, the entire CouchDB traceback is sent in ex_message
self.error("Create request failed, reason: %s" % str(ex))
if hasattr(ex, "message"):
if hasattr(ex.message, '__call__'):
detail = ex.message()
else:
detail = str(ex)
elif hasattr(ex, "name"):
detail = ex.name
else:
detail = "check logs."
msg = "Create request failed, %s" % detail
raise cherrypy.HTTPError(400, msg)
baseURL = cherrypy.request.base
raise cherrypy.HTTPRedirect('%s/reqmgr/view/details/%s' %
(baseURL, request['RequestName']))
def multirequest(self,
url,
parray,
headers=None,
ckey=None,
cert=None,
verbose=None):
"""Fetch data for given set of parameters"""
multi = pycurl.CurlMulti()
for params in parray:
curl = pycurl.Curl()
bbuf, hbuf = \
self.set_opts(curl, url, params, headers, ckey, cert, verbose)
multi.add_handle(curl)
while True:
ret, num_handles = multi.perform()
if ret != pycurl.E_CALL_MULTI_PERFORM:
break
while num_handles:
ret = multi.select(1.0)
if ret == -1:
continue
while 1:
ret, num_handles = multi.perform()
if ret != pycurl.E_CALL_MULTI_PERFORM:
break
_numq, response, _err = multi.info_read()
for _cobj in response:
data = json.loads(bbuf.getvalue())
if isinstance(data, dict):
data.update(params)
yield data
if isinstance(data, list):
for item in data:
if isinstance(item, dict):
item.update(params)
yield item
else:
err = 'Unsupported data format: data=%s, type=%s'\
% (item, type(item))
raise Exception(err)
bbuf.flush()
hbuf.flush()
def handleAssignmentPage(self, **kwargs):
""" handler for the main page """
# Accept Json encoded strings
decodedArgs = {}
for key in kwargs.keys():
try:
decodedArgs[key] = JsonWrapper.loads(kwargs[key])
except Exception:
# Probably wasn't JSON
decodedArgs[key] = kwargs[key]
kwargs = decodedArgs
# handle the checkboxes
teams = []
requestNames = []
for key, value in kwargs.iteritems():
if isinstance(value, basestring):
kwargs[key] = value.strip()
if key.startswith("Team"):
teams.append(key[4:])
if key.startswith("checkbox"):
requestName = key[8:]
self.validate(requestName)
requestNames.append(key[8:])
for requestName in requestNames:
if kwargs["action"] == "Reject":
ChangeState.changeRequestStatus(requestName, "rejected", wmstatUrl=self.wmstatWriteURL)
else:
assignments = GetRequest.getAssignmentsByName(requestName)
if teams == [] and assignments == []:
raise cherrypy.HTTPError(400, "Must assign to one or more teams")
kwargs["Teams"] = teams
self.assignWorkload(requestName, kwargs)
for team in teams:
if not team in assignments:
ChangeState.assignRequest(requestName, team, wmstatUrl=self.wmstatWriteURL)
priority = kwargs.get(requestName + ":priority", "")
if priority != "":
Utilities.changePriority(requestName, priority, self.wmstatWriteURL)
participle = kwargs["action"] + "ed"
return self.templatepage("Acknowledge", participle=participle, requests=requestNames)
def post(self):
"""
Create / inject a new request. Request input schema is specified in
the body of the request as JSON encoded data.
ReqMgr related request arguments validation to happen in
DataStructs.Request.validate(), the rest in spec.
ReqMgr related arguments manipulation to happen in the .request_initialize(),
before the spec is instantiated.
TODO:
this method will have some parts factored out so that e.g. clone call
can share functionality.
NOTES:
1) do not strip spaces, #4705 will fails upon injection with spaces ;
currently the chain relies on a number of things coming in #4705
2) reqInputArgs = Utilities.unidecode(JsonWrapper.loads(body))
(from ReqMgrRESTModel.putRequest)
"""
json_input_request_args = cherrypy.request.body.read()
request_input_dict = JsonWrapper.loads(json_input_request_args)
cherrypy.log("INFO: Create request, input args: %s ..." %
request_input_dict)
request = RequestData() # this returns a new request dictionary
request.update(request_input_dict)
try:
request.validate_automatic_args_empty()
# fill in automatic request arguments and further request args meddling
self.request_initialize(request)
self.request_validate(request)
except RequestDataError, ex:
cherrypy.log(ex.message)
raise cherrypy.HTTPError(400, ex.message)
def post(self, name):
"""
Parse job splitting parameters sent from the splitting parameter update
page. Pull down the request and modify the new spec applying the
updated splitting parameters.
"""
data = cherrypy.request.body.read()
splittingInfo = JsonWrapper.loads(data)
for taskInfo in splittingInfo:
splittingTask = taskInfo["taskName"]
splittingAlgo = taskInfo["splitAlgo"]
submittedParams = taskInfo["splitParams"]
splitParams = {}
for param in submittedParams:
validFlag, castType = _validate_split_param(splittingAlgo, param)
if validFlag:
_assign_if_key_exsit(param, submittedParams, splitParams, castType)
else:
#TO Maybe raise the error messge
pass
#TODO: this is only gets updated through script. Maybe we should disallow it.
_assign_if_key_exsit("include_parents", submittedParams, splitParams, bool)
helper = WMWorkloadHelper()
try:
helper.loadSpecFromCouch(self.reqdb_url, name)
except Exception:
raise cherrypy.HTTPError(404, "Cannot find workload: %s" % name)
helper.setJobSplittingParameters(splittingTask, splittingAlgo, splitParams)
# Not sure why it needs to updated per each task but if following lines are outside the loop
# it doesn't work
url = "%s/%s" % (self.reqdb_url, name)
result = helper.saveCouchUrl(url)
return result
def post(self):
"""
Create / inject a new request. Request input schema is specified in
the body of the request as JSON encoded data.
ReqMgr related request arguments validation to happen in
DataStructs.Request.validate(), the rest in spec.
ReqMgr related arguments manipulation to happen in the .request_initialize(),
before the spec is instantiated.
TODO:
this method will have some parts factored out so that e.g. clone call
can share functionality.
NOTES:
1) do not strip spaces, #4705 will fails upon injection with spaces ;
currently the chain relies on a number of things coming in #4705
2) reqInputArgs = Utilities.unidecode(JsonWrapper.loads(body))
(from ReqMgrRESTModel.putRequest)
"""
json_input_request_args = cherrypy.request.body.read()
request_input_dict = JsonWrapper.loads(json_input_request_args)
cherrypy.log("INFO: Create request, input args: %s ..." % request_input_dict)
request = RequestData() # this returns a new request dictionary
request.update(request_input_dict)
try:
request.validate_automatic_args_empty()
# fill in automatic request arguments and further request args meddling
self.request_initialize(request)
self.request_validate(request)
except RequestDataError, ex:
cherrypy.log(ex.message)
raise cherrypy.HTTPError(400, ex.message)
def _getResult(self, callname, clearCache=False, args=None, verb="POST"):
"""
_getResult_
retrieve JSON/XML formatted information given the service name and the
argument dictionaries
TODO: Probably want to move this up into Service
"""
result = ''
# make base file name from call name.
file = callname.replace("/", "_")
if clearCache:
self.clearCache(file, args, verb=verb)
f = self.refreshCache(file, callname, args, verb=verb)
result = f.read()
f.close()
if self.responseType == "json":
return JsonWrapper.loads(result)
return result
def getElements(self,
status=None,
elementIDs=None,
returnIdOnly=False,
db=None,
loadSpec=False,
WorkflowName=None,
**elementFilters):
"""Return elements that match requirements
status, elementIDs & filters are 'AND'ed together to filter elements.
returnIdOnly causes the element not to be loaded and only the id returned
db is used to specify which database to return from
loadSpec causes the workflow for each spec to be loaded.
WorkflowName may be used in the place of RequestName
"""
key = []
if not db:
db = self.db
if elementFilters.get('RequestName') and not WorkflowName:
WorkflowName = elementFilters.pop('RequestName')
if elementIDs:
if elementFilters or status or returnIdOnly:
raise ValueError, "Can't specify extra filters (or return id's) when using element id's with getElements()"
elements = [
CouchWorkQueueElement(db, i).load() for i in elementIDs
]
else:
options = {
'include_docs': True,
'filter': elementFilters,
'idOnly': returnIdOnly,
'reduce': False
}
# filter on workflow or status if possible
filter = 'elementsByWorkflow'
if WorkflowName:
key.append(WorkflowName)
elif status:
filter = 'elementsByStatus'
key.append(status)
elif elementFilters.get('SubscriptionId'):
key.append(elementFilters['SubscriptionId'])
filter = 'elementsBySubscription'
# add given params to filters
if status:
options['filter']['Status'] = status
if WorkflowName:
options['filter']['RequestName'] = WorkflowName
view = db.loadList('WorkQueue', 'filter', filter, options, key)
view = json.loads(view)
if returnIdOnly:
return view
elements = [
CouchWorkQueueElement.fromDocument(db, row) for row in view
]
if loadSpec:
specs = {} # cache as may have multiple elements for same spec
for ele in elements:
if ele['RequestName'] not in specs:
wmspec = self.getWMSpec(ele['RequestName'])
specs[ele['RequestName']] = wmspec
ele['WMSpec'] = specs[ele['RequestName']]
del specs
return elements
def putRequest(self,
requestName=None,
status=None,
priority=None,
stats=None):
request = None
if requestName:
try:
request = self.getRequest(requestName)
except Exception as ex:
# request presumably doesn't exist
request = None
if request == None:
# Create a new request, with a JSON-encoded schema that is
# sent in the body of the HTTP request
body = cherrypy.request.body.read()
reqInputArgs = Utilities.unidecode(JsonWrapper.loads(body))
# check for forbidden input arguments:
for deprec in deprecatedRequestArgs:
if deprec in reqInputArgs:
msg = (
"Creating request failed, unsupported input arg: %s: %s"
% (deprec, reqInputArgs[deprec]))
self.error(msg)
raise cherrypy.HTTPError(400, msg)
reqInputArgs.setdefault(
'CouchURL', Utilities.removePasswordFromUrl(self.couchUrl))
reqInputArgs.setdefault('CouchWorkloadDBName', self.workloadDBName)
# wrong naming ... but it's all over the place, it's the config cache DB name
reqInputArgs.setdefault('CouchDBName', self.configDBName)
reqInputArgs.setdefault('Requestor',
cherrypy.request.user["login"])
try:
self.info(
"Creating a request for: '%s'\n\tworkloadDB: '%s'\n\twmstatUrl: "
"'%s' ..." %
(reqInputArgs, self.workloadDBName,
Utilities.removePasswordFromUrl(self.wmstatWriteURL)))
request = Utilities.makeRequest(self, reqInputArgs,
self.couchUrl,
self.workloadDBName,
self.wmstatWriteURL)
except cherrypy.HTTPError as ex:
msg = traceback.format_exc()
self.error("Create request failed, reason: %s" % msg)
# Assume that this is a valid HTTPError
raise ex
except (WMException, Exception) as ex:
# TODO problem not to expose logs to the client
# e.g. on ConfigCacheID not found, the entire CouchDB traceback is sent in ex_message
msg = traceback.format_exc()
self.error("Create request failed, reason: %s" % msg)
if hasattr(ex, "message"):
if hasattr(ex.message, '__call__'):
detail = ex.message()
else:
detail = str(ex)
elif hasattr(ex, "name"):
detail = ex.name
else:
detail = "check logs."
msg = "Create request failed, %s" % detail
raise cherrypy.HTTPError(400, msg)
self.info("Request '%s' created." % request['RequestName'])
# see if status & priority need to be upgraded
if status != None:
# forbid assignment here
if status == 'assigned':
raise cherrypy.HTTPError(
403,
"Cannot change status without a team. Please use PUT /reqmgr/reqMgr/assignment/<team>/<requestName>"
)
try:
Utilities.changeStatus(requestName, status,
self.wmstatWriteURL, self.acdcURL)
except RuntimeError as ex:
# ignore some of these errors: https://svnweb.cern.ch/trac/CMSDMWM/ticket/2002
if status != 'announced' and status != 'closed-out':
self.error("RuntimeError while changeStatus: reason: %s" %
ex)
raise cherrypy.HTTPError(
403, "Failed to change status: %s" % str(ex))
if priority != None:
Utilities.changePriority(requestName, priority,
self.wmstatWriteURL)
if stats != None:
Utilities.updateRequestStats(requestName=requestName,
stats=stats,
couchURL=self.couchUrl,
couchDBName=self.workloadDBName)
return request
def query(self, obj, service='', objType='node'):
"""
query status and eventually other scheduler related information
"""
# jobId for remapping
jobIds = {}
# parent Ids for status query
parentIds = []
# counter for job position in list
count = 0
# the object passed is a Task:
if type(obj) == Task:
if objType == 'node':
# loop!
for job in obj.jobs:
if job.runningJob is None \
or job.runningJob.active != True \
or job.runningJob['schedulerId'] is None \
or job.runningJob['closed'] == "Y" \
or job.runningJob['status'] in self.invalidList :
count += 1
continue
# append in joblist
jobIds[str(job.runningJob['schedulerId'])] = count
count += 1
if jobIds:
formattedJobIds = ','.join(jobIds)
command = self.commandQueryPath \
+ 'GLiteStatusQuery.py --jobId=%s' % formattedJobIds
outJson, ret = self.ExecuteCommand(self.proxyString +
self.hackEnv + command)
# Check error
if ret != 0:
# obj.errors doesn't exist for Task object...
obj.warnings.append("Errors: " + str(outJson.strip()))
raise SchedulerError(
'error executing GLiteStatusQuery', \
str(outJson.strip()))
else:
# parse JSON output
try:
out = json.loads(outJson)
# DEBUG # print json.dumps( out, indent=4 )
except ValueError:
raise SchedulerError('error parsing JSON', outJson)
elif objType == 'parent':
# loop!
for job in obj.jobs:
# consider just valid jobs
if self.valid(job.runningJob):
# append in joblist
jobIds[str(job.runningJob['schedulerId'])] = count
# update unique parent ids list
if job.runningJob['schedulerParentId'] \
not in parentIds:
parentIds.append(
str(job.runningJob['schedulerParentId']))
count += 1
if jobIds:
formattedParentIds = ','.join(parentIds)
formattedJobIds = ','.join(jobIds)
command = self.commandQueryPath \
+ 'GLiteStatusQuery.py --parentId=%s --jobId=%s' \
% (formattedParentIds, formattedJobIds)
outJson, ret = self.ExecuteCommand(self.proxyString +
self.hackEnv + command)
# Check error
if ret != 0:
# obj.errors doesn't exist for Task object...
obj.warnings.append("Errors: " + str(outJson.strip()))
raise SchedulerError(
'error executing GLiteStatusQuery', \
str(outJson.strip()))
else:
# parse JSON output
try:
out = json.loads(outJson)
# DEBUG # print json.dumps( out, indent=4 )
except ValueError:
raise SchedulerError('error parsing JSON', outJson)
if jobIds:
# Refill objects...
for jobId in jobIds.values():
jobIdToUpdate = obj.jobs[jobId].runningJob['schedulerId']
updatedJobInfo = out[jobIdToUpdate]
for currentKey in updatedJobInfo.keys():
if updatedJobInfo[currentKey] is None:
pass
else:
obj.jobs[jobId].runningJob[currentKey] = \
updatedJobInfo[currentKey]
# unknown object type
else:
raise SchedulerError('wrong argument type', str(type(obj)))
"shadow": True,
"percentage": False,
"labels": False, # don't print labels when having legend
"legend": "topright",
"sort": "value"
}
plotDefinition["title"] = "Element status - hard-coded"
hardCodedPlotData = urllib.quote(json.dumps(plotDefinition, ensure_ascii=True))
# retrieve data from the system
# URL http://localhost:8888/workqueue/elementsinfo
data, _, _, _ = makeRequest(DATA_SOURCE_URL,
verb="GET",
accept="text/json+das",
contentType="application/json")
data = JsonWrapper.loads(data)
systemSeries = series[:]
states = {}
for s in STATES:
states[s] = 0 # status labels tuple
for elements in data["results"]:
states[elements["status"]] = states[elements["status"]] + 1
for s in systemSeries:
s["value"] = states[s["label"]]
plotDefinition["title"] = "Element status - WorkQueue"
plotDefinition["series"] = systemSeries
systemData = urllib.quote(json.dumps(plotDefinition, ensure_ascii=True))
# write result file / generate result html snippet
class ReqMgrRESTModel(RESTModel):
""" The REST interface to the ReqMgr database. Documentation may
be found at https://twiki.cern.ch/twiki/bin/viewauth/CMS/ReqMgrSystemDesign """
def __init__(self, config):
RESTModel.__init__(self, config)
self.couchUrl = config.couchUrl
self.workloadDBName = config.workloadDBName
self.configDBName = config.configDBName
self.wmstatWriteURL = "%s/%s" % (self.couchUrl.rstrip('/'),
config.wmstatDBName)
self.acdcURL = "%s/%s" % (self.couchUrl.rstrip('/'), config.acdcDBName)
self.security_params = {'roles': config.security_roles}
# Optional values for individual methods
self.reqPriorityMax = getattr(config, 'maxReqPriority', 100)
self._addMethod('GET',
'request',
self.getRequest,
args=['requestName'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('GET',
'assignment',
self.getAssignment,
args=['teamName', 'request'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('GET',
'user',
self.getUser,
args=['userName'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('GET',
'group',
self.getGroup,
args=['group', 'user'],
secured=True,
expires=0)
self._addMethod('GET',
'version',
self.getVersion,
args=[],
secured=True,
expires=0)
self._addMethod('GET',
'team',
self.getTeam,
args=[],
secured=True,
expires=0)
self._addMethod('GET',
'workQueue',
self.getWorkQueue,
args=['request', 'workQueue'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('GET',
'message',
self.getMessage,
args=['request'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('GET',
'inputdataset',
self.getInputDataset,
args=['prim', 'proc', 'tier'],
secured=True)
self._addMethod('GET',
'outputdataset',
self.getOutputDataset,
args=['prim', 'proc', 'tier'],
secured=True)
self._addMethod('GET',
'campaign',
self.getCampaign,
args=['campaign'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('PUT',
'request',
self.putRequest,
args=['requestName', 'status', 'priority', 'stats'],
secured=True,
validation=[
self.isalnumExceptStats, self.reqPriority,
self.validateStats
])
self._addMethod('PUT',
'clone',
self.cloneRequest,
args=['requestName'],
secured=True,
validation=[self.isalnum])
self._addMethod('PUT',
'assignment',
self.putAssignment,
args=['team', 'requestName'],
secured=True,
security_params=self.security_params,
validation=[self.isalnum])
self._addMethod('PUT',
'user',
self.putUser,
args=['userName', 'email', 'dnName'],
secured=True,
security_params=self.security_params,
validation=[self.validateUser])
self._addMethod('PUT',
'group',
self.putGroup,
args=['group', 'user'],
secured=True,
security_params=self.security_params,
validation=[self.isalnum])
self._addMethod('PUT',
'version',
self.putVersion,
args=['version', 'scramArch'],
secured=True,
security_params=self.security_params,
validation=[self.validateVersion])
self._addMethod('PUT',
'team',
self.putTeam,
args=['team'],
secured=True,
security_params=self.security_params,
validation=[self.isalnum])
self._addMethod('PUT',
'workQueue',
self.putWorkQueue,
args=['request', 'url'],
secured=True,
security_params=self.security_params,
validation=[self.validatePutWorkQueue])
self._addMethod('PUT',
'message',
self.putMessage,
args=['request'],
secured=True,
security_params=self.security_params,
validation=[self.isalnum])
self._addMethod('PUT',
'campaign',
self.putCampaign,
args=['campaign', 'request'],
secured=True,
validation=[self.isalnum])
self._addMethod('POST',
'request',
self.postRequest,
args=[
'requestName', 'events_written', 'events_merged',
'files_written', 'files_merged', 'percent_written',
'percent_success', 'dataset'
],
secured=True,
validation=[self.validateUpdates])
self._addMethod('POST',
'closeout',
self.closeOutRequest,
args=['requestName', 'cascade'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('POST',
'announce',
self.announceRequest,
args=['requestName', 'cascade'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('DELETE',
'request',
self.deleteRequest,
args=['requestName'],
secured=True,
security_params=self.security_params,
validation=[self.isalnum])
self._addMethod('DELETE',
'user',
self.deleteUser,
args=['user'],
secured=True,
security_params=self.security_params,
validation=[self.isalnum])
self._addMethod('DELETE',
'group',
self.deleteGroup,
args=['group', 'user'],
secured=True,
security_params=self.security_params,
validation=[self.isalnum])
self._addMethod('DELETE',
'version',
self.deleteVersion,
args=['version', 'scramArch'],
secured=True,
validation=[self.validateVersion])
self._addMethod('DELETE',
'team',
self.deleteTeam,
args=['team'],
secured=True,
security_params=self.security_params,
validation=[self.isalnum])
self._addMethod('DELETE',
'campaign',
self.deleteCampaign,
args=['campaign'],
secured=True,
security_params=self.security_params,
validation=[self.isalnum])
self._addMethod('GET',
'requestnames',
self.getRequestNames,
args=[],
secured=True,
expires=0)
self._addMethod('GET',
'outputDatasetsByRequestName',
self.getOutputForRequest,
args=['requestName'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('GET',
'outputDatasetsByPrepID',
self.getOutputForPrepID,
args=['prepID'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('GET',
'mostRecentOutputDatasetsByPrepID',
self.getMostRecentOutputForPrepID,
args=['prepID'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('GET',
'configIDs',
self.getConfigIDs,
args=['prim', 'proc', 'tier'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('GET',
'requestsByStatusAndTeam',
self.getRequestsByStatusAndTeam,
args=['teamName', 'status'],
secured=True,
validation=[self.isalnum],
expires=0)
cherrypy.engine.subscribe('start_thread', self.initThread)
def initThread(self, thread_index):
""" The ReqMgr expects the DBI to be contained in the Thread """
myThread = threading.currentThread()
#myThread = cherrypy.thread_data
# Get it from the DBFormatter superclass
myThread.dbi = self.dbi
def isalnum(self, index):
""" Validates that all input is alphanumeric,
with spaces and underscores tolerated"""
for v in index.values():
WMCore.Lexicon.identifier(v)
return index
def isalnumExceptStats(self, index):
""" Validates that all input is alphanumeric,
with spaces and underscores tolerated"""
if index.has_key('stats'):
return index
return self.isalnum(index)
def getDataset(self, prim, proc, tier):
""" If only prim exists, assume it's urlquoted.
If all three exists, assue it's /prim/proc/tier
"""
if not proc and not tier:
dataset = urllib.unquote(prim)
elif prim and proc and tier:
dataset = "/%s/%s/%s" % (prim, proc, tier)
WMCore.Lexicon.dataset(dataset)
return dataset
def intpriority(self, index):
""" Casts priority to an integer """
if index.has_key('priority'):
value = int(index['priority'])
if math.fabs(value) >= sys.maxint:
msg = "Invalid priority! Priority must have abs() less then MAXINT!"
raise cherrypy.HTTPError(400, msg)
index['priority'] = value
return index
def reqPriority(self, index):
"""
_reqPriority_
Sets request priority to an integer.
Also makes sure it's within a certain value.
"""
if not index.has_key('priority'):
return index
index = self.intpriority(index=index)
value = index['priority']
if math.fabs(
value) > self.reqPriorityMax and not Utilities.privileged():
msg = "Invalid requestPriority! Request priority must have abs() less then %i!" % self.reqPriorityMax
raise cherrypy.HTTPError(400, msg)
return index
def validateUser(self, index):
assert index['userName'].isalnum()
assert '@' in index['email']
assert index['email'].replace('@', '').replace('.', '').isalnum()
if 'dnName' in index:
assert index['dnName'].replace(' ', '').isalnum()
return index
def validateVersion(self, index):
""" Make sure it's a legitimate CMSSW version format """
WMCore.Lexicon.cmsswversion(index['version'])
return index
def getRequest(self, requestName=None):
""" If a request name is specified, return the details of the request.
Otherwise, return an overview of all requests """
if requestName == None:
result = GetRequest.getRequests()
else:
result = Utilities.requestDetails(requestName)
try:
teamNames = GetRequest.getAssignmentsByName(requestName)
result['teams'] = teamNames
except:
# Ignore errors, then we just don't have a team name
pass
return result
def getRequestNames(self):
# 2013-02-13 is wrong anyway (e.g. Campaign is not working)
# should be removed
""" return all the request names in RequestManager as list """
#TODO this could me combined with getRequest
return GetRequest.getOverview()
def getOutputForRequest(self, requestName):
"""Return the datasets produced by this request."""
return Utilities.getOutputForRequest(requestName)
def getOutputForPrepID(self, prepID):
"""Return the datasets produced by this prep ID. in a dict of requestName:dataset list"""
requestIDs = GetRequest.getRequestByPrepID(prepID)
result = {}
for requestID in requestIDs:
request = GetRequest.getRequest(requestID)
requestName = request["RequestName"]
helper = Utilities.loadWorkload(request)
result[requestName] = helper.listOutputDatasets()
return result
def getMostRecentOutputForPrepID(self, prepID):
"""Return the datasets produced by the most recently submitted request with this prep ID"""
requestIDs = GetRequest.getRequestByPrepID(prepID)
# most recent will have the largest ID
requestIDs.sort()
requestIDs.reverse()
request = None
# Go through each request in order from largest to smallest
# looking for the first non-failed/non-canceled request
for requestID in requestIDs:
request = GetRequest.getRequest(requestID)
rejectList = ['aborted', 'failed', 'rejected', 'epic-failed']
requestStatus = request.get("RequestStatus", 'aborted').lower()
if requestStatus not in rejectList:
break
if request != None:
helper = Utilities.loadWorkload(request)
return helper.listOutputDatasets()
else:
return []
def getAssignment(self, teamName=None, request=None):
""" If a team name is passed in, get all assignments for that team.
If a request is passed in, return a list of teams the request is
assigned to
"""
# better to use ReqMgr/RequestDB/Interface/ProdSystem/ProdMgrRetrieve?
#requestIDs = ProdMgrRetrieve.findAssignedRequests(teamName)
# Maybe now assigned to team is same as assigned to ProdMgr
result = []
if teamName != None:
requestIDs = ListRequests.listRequestsByTeam(teamName,
"assigned").values()
requests = [GetRequest.getRequest(reqID) for reqID in requestIDs]
# put highest priorities first
requests.sort(key=lambda r: r['RequestPriority'], reverse=True)
# return list of tuples, since we need sorting
result = [[req['RequestName'], req['RequestWorkflow']]
for req in requests]
elif request != None:
result = GetRequest.getAssignmentsByName(request)
return result
def getRequestsByStatusAndTeam(self, teamName, status):
""" Get a list of request names with the given team and status.
"""
requestNames = ListRequests.listRequestsByTeam(teamName, status).keys()
return requestNames
def getUser(self, userName=None, group=None):
""" No args returns a list of all users. Group returns groups this user is in. Username
returs a JSON with information about the user """
if userName != None:
if not Registration.isRegistered(userName):
raise cherrypy.HTTPError(404, "Cannot find user")
result = {}
result['groups'] = GroupInfo.groupsForUser(userName).keys()
result['requests'] = UserRequests.listRequests(userName).keys()
result.update(Registration.userInfo(userName))
return result
elif group != None:
GroupInfo.usersInGroup(group)
else:
return Registration.listUsers()
def getGroup(self, group=None, user=None):
""" No args lists all groups, one args returns JSON with users."""
if group != None:
result = {}
result['users'] = GroupInfo.usersInGroup(group)
return result
elif user != None:
return GroupInfo.groupsForUser(user).keys()
else:
return GroupInfo.listGroups()
def getVersion(self):
""" Returns a list of all CMSSW versions registered with ReqMgr """
archList = SoftwareAdmin.listSoftware()
result = []
for arch in archList.keys():
for version in archList[arch]:
if not version in result:
result.append(version)
return result
def getTeam(self):
""" Returns a list of all teams registered with ReqMgr """
return ProdManagement.listTeams()
def getWorkQueue(self, request=None, workQueue=None):
""" If a request is passed in, return the URl of the workqueue.
If a workqueue is passed in, return all requests acquired by it """
if workQueue != None:
return ProdMgrRetrieve.findAssignedRequests(workQueue)
if request != None:
return ProdManagement.getProdMgr(request)
# return all the workqueue ulr
return GetRequest.getGlobalQueues()
def getMessage(self, request):
""" Returns a list of messages attached to this request """
return ChangeState.getMessages(request)
def getInputDataset(self, prim, proc=None, tier=None):
""" returns a list of requests with this input dataset
Input can either be a single urlquoted dataset, or a
/prim/proc/tier"""
dataset = self.getDataset(prim, proc, tier)
return GetRequest.getRequestsByCriteria("Datasets.GetRequestByInput",
dataset)
def getOutputDataset(self, prim, proc=None, tier=None):
""" returns a list of requests with this output dataset
Input can either be a single urlquoted dataset, or a
/prim/proc/tier"""
dataset = self.getDataset(prim, proc, tier)
return GetRequest.getRequestsByCriteria("Datasets.GetRequestByOutput",
dataset)
def getCampaign(self, campaign=None):
""" returns a list of all campaigns if no argument, and a list of
all requests in a campaign if there is an argument """
if campaign == None:
return Campaign.listCampaigns()
else:
return Campaign.listRequestsByCampaign(campaign)
def putWorkQueue(self, request, url):
""" Registers the request as "acquired" by the workqueue with the given URL """
Utilities.changeStatus(request, "acquired", self.wmstatWriteURL,
self.acdcURL)
return ProdManagement.associateProdMgr(request, urllib.unquote(url))
def validatePutWorkQueue(self, index):
assert index['request'].replace('_',
'').replace('-',
'').replace('.',
'').isalnum()
assert index['url'].startswith('http')
return index
# had no permission control before, security issue fix
@cherrypy.tools.secmodv2(role=Utilities.security_roles(),
group=Utilities.security_groups())
def putRequest(self,
requestName=None,
status=None,
priority=None,
stats=None):
request = None
if requestName:
try:
request = self.getRequest(requestName)
except Exception, ex:
# request presumably doesn't exist
request = None
if request == None:
# Create a new request, with a JSON-encoded schema that is
# sent in the body of the HTTP request
body = cherrypy.request.body.read()
reqInputArgs = Utilities.unidecode(JsonWrapper.loads(body))
# check for forbidden input arguments:
for deprec in deprecatedRequestArgs:
if deprec in reqInputArgs:
msg = (
"Creating request failed, unsupported input arg: %s: %s"
% (deprec, reqInputArgs[deprec]))
self.error(msg)
raise cherrypy.HTTPError(400, msg)
reqInputArgs.setdefault(
'CouchURL', Utilities.removePasswordFromUrl(self.couchUrl))
reqInputArgs.setdefault('CouchWorkloadDBName', self.workloadDBName)
# wrong naming ... but it's all over the place, it's the config cache DB name
reqInputArgs.setdefault('CouchDBName', self.configDBName)
reqInputArgs.setdefault('Requestor',
cherrypy.request.user["login"])
try:
self.info(
"Creating a request for: '%s'\n\tworkloadDB: '%s'\n\twmstatUrl: "
"'%s' ..." %
(reqInputArgs, self.workloadDBName,
Utilities.removePasswordFromUrl(self.wmstatWriteURL)))
request = Utilities.makeRequest(self, reqInputArgs,
self.couchUrl,
self.workloadDBName,
self.wmstatWriteURL)
except cherrypy.HTTPError as ex:
self.error("Create request failed, reason: %s" % ex)
# Assume that this is a valid HTTPError
raise ex
except (WMException, Exception) as ex:
# TODO problem not to expose logs to the client
# e.g. on ConfigCacheID not found, the entire CouchDB traceback is sent in ex_message
self.error("Create request failed, reason: %s" % ex)
if hasattr(ex, "name"):
detail = ex.name
else:
detail = "check logs."
msg = "Create request failed, %s" % detail
raise cherrypy.HTTPError(400, msg)
self.info("Request '%s' created." % request['RequestName'])
# see if status & priority need to be upgraded
if status != None:
# forbid assignment here
if status == 'assigned':
raise cherrypy.HTTPError(
403,
"Cannot change status without a team. Please use PUT /reqmgr/reqMgr/assignment/<team>/<requestName>"
)
try:
Utilities.changeStatus(requestName, status,
self.wmstatWriteURL, self.acdcURL)
except RuntimeError as ex:
# ignore some of these errors: https://svnweb.cern.ch/trac/CMSDMWM/ticket/2002
if status != 'announced' and status != 'closed-out':
self.error("RuntimeError while changeStatus: reason: %s" %
ex)
raise cherrypy.HTTPError(
403, "Failed to change status: %s" % str(ex))
if priority != None:
Utilities.changePriority(requestName, priority,
self.wmstatWriteURL)
if stats != None:
Utilities.updateRequestStats(requestName=requestName,
stats=stats,
couchURL=self.couchUrl,
couchDBName=self.workloadDBName)
return request
def putMessage(self, request):
""" Attaches a message to this request """
message = JsonWrapper.loads(cherrypy.request.body.read())
result = ChangeState.putMessage(request, message)
return result
def _getRequestNamesFromBody(self, param, safe, valFunc):
request_names = JsonWrapper.loads(cherrypy.request.body.read())
safe.kwargs['workload_pair_list'] = request_names
safe.kwargs["multi_names_flag"] = True
def purgeService(self, obj):
"""
Purge job (even bulk) from wms
"""
# Implement as getOutput where the "No output files ..."
# is not an error condition but the expected status
if type(obj) == Job and self.valid(obj.runningJob):
# the object passed is a valid Job
command = "glite-wms-job-output --json --noint --dir /tmp/ " \
+ obj.runningJob['schedulerId']
out, ret = self.ExecuteCommand(self.proxyString + command)
if ret != 0:
self.logging.error("Purge of %s Failed with exit code =%d" %
(str(job.runningJob['schedulerId']), ret))
self.logging.error("output was \n%s" % out)
elif ret == 0:
loadOut = json.loads(out)
if unicode('result') in loadOut and loadOut[unicode(
'result')] == unicode('success'):
self.logging.info("Purge of %s successfully done" %
str(job.runningJob['schedulerId']))
else:
self.logging.error("Purge of %s Executed NOT VALIDATED" %
str(job.runningJob['schedulerId']))
self.logging.error(
"Purge of %s Executed with output\n%s" %
(str(job.runningJob['schedulerId']), out))
elif type(obj) == Task:
# the object passed is a Task
for job in obj.jobs:
if not self.valid(job.runningJob):
continue
command = "glite-wms-job-output --json --noint --dir /tmp/ " \
+ job.runningJob['schedulerId']
out, ret = self.ExecuteCommand(self.proxyString + command)
if ret != 0:
self.logging.error(
"Purge of %s Failed with exit code =%d" %
(str(job.runningJob['schedulerId']), ret))
self.logging.error("output was \n%s" % out)
elif ret == 0:
loadOut = json.loads(out)
if unicode('result') in loadOut and loadOut[unicode(
'result')] == unicode('success'):
self.logging.info("Purge of %s successfully done" %
str(job.runningJob['schedulerId']))
else:
self.logging.error(
"Purge of %s Executed NOT VALIDATED" %
str(job.runningJob['schedulerId']))
self.logging.error(
"Purge of %s Executed with output\n%s" %
(str(job.runningJob['schedulerId']), out))
def availableWork(self, thresholds, siteJobCounts, teams=None, wfs=None):
"""
Get work which is available to be run
Assume thresholds is a dictionary; keys are the site name, values are
the maximum number of running jobs at that site.
Assumes site_job_counts is a dictionary-of-dictionaries; keys are the site
name and task priorities. The value is the number of jobs running at that
priority.
"""
self.logger.info("Getting available work from %s/%s" %
(sanitizeURL(self.server.url)['url'], self.db.name))
elements = []
# We used to pre-filter sites, looking to see if there are idle job slots
# We don't do this anymore, as we may over-allocate
# jobs to sites if the new jobs have a higher priority.
# If there are no sites, punt early.
if not thresholds:
self.logger.error("No thresholds is set: Please check")
return elements, thresholds, siteJobCounts
options = {}
options['include_docs'] = True
options['descending'] = True
options['resources'] = thresholds
if teams:
options['teams'] = teams
self.logger.info("setting teams %s" % teams)
if wfs:
result = []
for i in xrange(0, len(wfs), 20):
options['wfs'] = wfs[i:i + 20]
data = self.db.loadList('WorkQueue', 'workRestrictions',
'availableByPriority', options)
result.extend(json.loads(data))
# sort final list
result.sort(key=lambda x: x[
'WMCore.WorkQueue.DataStructs.WorkQueueElement.WorkQueueElement'
]['Priority'])
else:
result = self.db.loadList('WorkQueue', 'workRestrictions',
'availableByPriority', options)
result = json.loads(result)
if len(result) == 0:
self.logger.info(
"""No available work in WQ or didn't pass workqueue restriction
- check Pileup, site white list, etc""")
self.logger.debug("Available Work:\n %s \n for resources\n %s" %
(result, thresholds))
# Iterate through the results; apply whitelist / blacklist / data
# locality restrictions. Only assign jobs if they are high enough
# priority.
for i in result:
element = CouchWorkQueueElement.fromDocument(self.db, i)
prio = element['Priority']
possibleSite = None
sites = thresholds.keys()
random.shuffle(sites)
for site in sites:
if element.passesSiteRestriction(site):
# Count the number of jobs currently running of greater priority
prio = element['Priority']
curJobCount = sum(
map(lambda x: x[1] if x[0] >= prio else 0,
siteJobCounts.get(site, {}).items()))
self.logger.debug("Job Count: %s, site: %s threshods: %s" %
(curJobCount, site, thresholds[site]))
if curJobCount < thresholds[site]:
possibleSite = site
break
if possibleSite:
elements.append(element)
if site not in siteJobCounts:
siteJobCounts[site] = {}
siteJobCounts[site][prio] = siteJobCounts[site].setdefault(
prio, 0) + element['Jobs']
else:
self.logger.info("No possible site for %s" % element)
# sort elements to get them in priority first and timestamp order
elements.sort(key=lambda element: element['CreationTime'])
elements.sort(key=lambda x: x['Priority'], reverse=True)
return elements, thresholds, siteJobCounts