def loadRequestSchema(workload, requestSchema): """ _loadRequestSchema_ Does modifications to the workload I don't understand Takes a WMWorkloadHelper, operates on it directly with the schema """ schema = workload.data.request.section_('schema') for key, value in requestSchema.iteritems(): if type(value) == dict and key == 'LumiList': value = JsonWrapper.dumps(value) try: setattr(schema, key, value) except Exception as ex: # Attach TaskChain tasks if type(value) == dict and requestSchema['RequestType'] == 'TaskChain' and 'Task' in key: newSec = schema.section_(key) for k, v in requestSchema[key].iteritems(): if type(value) == dict and key == 'LumiList': value = JsonWrapper.dumps(value) try: setattr(newSec, k, v) except Exception as ex: # this logging need to change to cherry py logging logging.error("Invalid Value: %s" % str(ex)) else: # this logging need to change to cherry py logging logging.error("Invalid Value: %s" % str(ex)) schema.timeStamp = int(time.time()) schema = workload.data.request.schema # might belong in another method to apply existing schema workload.data.owner.Group = schema.Group workload.data.owner.Requestor = schema.Requestor
def test_cjson(self): """ Test cjson implementation. """ try: import cjson except: raise nose.SkipTest json_wrap._module = "cjson" result = json_wrap.dumps(self.record) expect = json.dumps(self.record) self.assertEqual(expect, result) data = result result = json_wrap.loads(data) expect = json.loads(data) self.assertEqual(expect, result) try: json_wrap.loads("bogusbogus") except cjson.DecodeError as ex: self.assertEqual(ex.args, ("cannot parse JSON description: bogusbogus", ))
def setUp(self): self.setConfig(config) self.setCouchDBs([(config.views.data.couch_reqmgr_db, "ReqMgr"), (config.views.data.couch_reqmgr_aux_db, None)]) self.setSchemaModules([]) RESTBaseUnitTestWithDBBackend.setUp(self) self.setFakeDN() # print "%s" % self.test_authz_key.data self.default_status_header = getAuthHeader(self.test_authz_key.data, DEFAULT_STATUS_PERMISSION) requestPath = os.path.join(getWMBASE(), "test", "data", "ReqMgr", "requests", "DMWM") rerecoFile = open(os.path.join(requestPath, "ReReco.json"), "r") rerecoArgs = JsonWrapper.load(rerecoFile) self.rerecoCreateArgs = rerecoArgs["createRequest"] self.rerecoCreateArgs["PrepID"] = "test_prepid" self.rerecoAssignArgs = rerecoArgs["assignRequest"] # overwrite rereco args self.rerecoAssignArgs["AcquisitionEra"] = "test_aqc" lheFile = open(os.path.join(requestPath, "TaskChain_Data.json"), "r") lheArgs = JsonWrapper.load(lheFile) self.lheStep0CreateArgs = lheArgs["createRequest"] self.lheStep0AssignArgs = lheArgs["assignRequest"] self.lheStep0AssignArgs["AcquisitionEra"] = "test_aqc" cmsswDoc = {"_id": "software"} cmsswDoc[self.rerecoCreateArgs["ScramArch"]] = [] cmsswDoc[self.rerecoCreateArgs["ScramArch"]].append(self.rerecoCreateArgs["CMSSWVersion"]) insertDataToCouch(os.getenv("COUCHURL"), config.views.data.couch_reqmgr_aux_db, cmsswDoc)
def setUp(self): self.setConfig(config) self.setCouchDBs([(config.views.data.couch_reqmgr_db, "ReqMgr"), (config.views.data.couch_reqmgr_aux_db, None)]) self.setSchemaModules([]) RESTBaseUnitTestWithDBBackend.setUp(self) self.setFakeDN() #print "%s" % self.test_authz_key.data self.default_status_header = getAuthHeader(self.test_authz_key.data, DEFAULT_STATUS_PERMISSION) requestPath = os.path.join(getWMBASE(), "test", "data", "ReqMgr", "requests") rerecoFile = open(os.path.join(requestPath, "ReReco.json"), 'r') rerecoArgs = JsonWrapper.load(rerecoFile) self.rerecoCreateArgs = rerecoArgs["createRequest"] self.rerecoAssignArgs = rerecoArgs["assignRequest"] # overwrite rereco args self.rerecoAssignArgs["AcquisitionEra"] = "test_aqc" lheFile = open(os.path.join(requestPath, "LHEStep0.json"), 'r') lheArgs = JsonWrapper.load(lheFile) self.lheStep0CreateArgs = lheArgs["createRequest"] self.lheStep0AssignArgs = lheArgs["assignRequest"] self.lheStep0AssignArgs["AcquisitionEra"] = "test_aqc" cmsswDoc = {"_id": "software"} cmsswDoc[self.rerecoCreateArgs["ScramArch"]] = [] cmsswDoc[self.rerecoCreateArgs["ScramArch"]].append(self.rerecoCreateArgs["CMSSWVersion"]) insertDataToCouch(os.getenv("COUCHURL"), config.views.data.couch_reqmgr_aux_db, cmsswDoc)
def purgeService(self, obj): """ Purge job (even bulk) from wms """ # Implement as getOutput where the "No output files ..." # is not an error condition but the expected status if type(obj) == Job and self.valid(obj.runningJob): # the object passed is a valid Job command = "glite-wms-job-output --json --noint --dir /tmp/ " + obj.runningJob["schedulerId"] out, ret = self.ExecuteCommand(self.proxyString + command) if ret != 0: self.logging.error("Purge of %s Failed with exit code =%d" % (str(job.runningJob["schedulerId"]), ret)) self.logging.error("output was \n%s" % out) elif ret == 0: loadOut = json.loads(out) if unicode("result") in loadOut and loadOut[unicode("result")] == unicode("success"): self.logging.info("Purge of %s successfully done" % str(job.runningJob["schedulerId"])) else: self.logging.error("Purge of %s Executed NOT VALIDATED" % str(job.runningJob["schedulerId"])) self.logging.error( "Purge of %s Executed with output\n%s" % (str(job.runningJob["schedulerId"]), out) ) elif type(obj) == Task: # the object passed is a Task for job in obj.jobs: if not self.valid(job.runningJob): continue command = "glite-wms-job-output --json --noint --dir /tmp/ " + job.runningJob["schedulerId"] out, ret = self.ExecuteCommand(self.proxyString + command) if ret != 0: self.logging.error( "Purge of %s Failed with exit code =%d" % (str(job.runningJob["schedulerId"]), ret) ) self.logging.error("output was \n%s" % out) elif ret == 0: loadOut = json.loads(out) if unicode("result") in loadOut and loadOut[unicode("result")] == unicode("success"): self.logging.info("Purge of %s successfully done" % str(job.runningJob["schedulerId"])) else: self.logging.error("Purge of %s Executed NOT VALIDATED" % str(job.runningJob["schedulerId"])) self.logging.error( "Purge of %s Executed with output\n%s" % (str(job.runningJob["schedulerId"]), out) )
def updateInboxElements(self, *elementIds, **updatedParams): """Update given inbox element's (identified by id) with new parameters""" uri = "/" + self.inbox.name + "/_design/WorkQueue/_update/in-place/" optionsArg = {} if "options" in updatedParams: optionsArg.update(updatedParams.pop("options")) data = {"updates": json.dumps(updatedParams), "options": json.dumps(optionsArg)} for ele in elementIds: thisuri = uri + ele + "?" + urllib.urlencode(data) self.inbox.makeRequest(uri=thisuri, type="PUT") return
def set_opts(self, curl, url, params, headers, ckey=None, cert=None, capath=None, verbose=None, verb='GET', doseq=True, cainfo=None): """Set options for given curl object, params should be a dictionary""" if params == None: params = {} if not isinstance(params, dict): raise TypeError("pycurl parameters should be passed as dictionary") curl.setopt(pycurl.NOSIGNAL, self.nosignal) curl.setopt(pycurl.TIMEOUT, self.timeout) curl.setopt(pycurl.CONNECTTIMEOUT, self.connecttimeout) curl.setopt(pycurl.FOLLOWLOCATION, self.followlocation) curl.setopt(pycurl.MAXREDIRS, self.maxredirs) encoded_data = urllib.urlencode(params, doseq=doseq) if verb == 'GET': url = url + '?' + encoded_data elif verb == 'HEAD': url = url + '?' + encoded_data curl.setopt(pycurl.CUSTOMREQUEST, verb) curl.setopt(pycurl.HEADER, 1) curl.setopt(pycurl.NOBODY, True) elif verb == 'POST': curl.setopt(pycurl.POST, 1) curl.setopt(pycurl.POSTFIELDS, json.dumps(params)) elif verb == 'DELETE' or verb == 'PUT': curl.setopt(pycurl.CUSTOMREQUEST, verb) curl.setopt(pycurl.HTTPHEADER, ['Transfer-Encoding: chunked']) curl.setopt(pycurl.POSTFIELDS, json.dumps(params)) else: raise Exception('Unsupported HTTP method "%s"' % verb) curl.setopt(pycurl.URL, url) if headers: curl.setopt(pycurl.HTTPHEADER, \ ["%s: %s" % (k, v) for k, v in headers.items()]) bbuf = StringIO.StringIO() hbuf = StringIO.StringIO() curl.setopt(pycurl.WRITEFUNCTION, bbuf.write) curl.setopt(pycurl.HEADERFUNCTION, hbuf.write) if capath: curl.setopt(pycurl.CAPATH, capath) curl.setopt(pycurl.SSL_VERIFYPEER, True) if cainfo: curl.setopt(pycurl.CAINFO, cainfo) else: curl.setopt(pycurl.SSL_VERIFYPEER, False) if ckey: curl.setopt(pycurl.SSLKEY, ckey) if cert: curl.setopt(pycurl.SSLCERT, cert) if verbose: curl.setopt(pycurl.VERBOSE, 1) curl.setopt(pycurl.DEBUGFUNCTION, self.debug) return bbuf, hbuf
def updateInboxElements(self, *elementIds, **updatedParams): """Update given inbox element's (identified by id) with new parameters""" uri = "/" + self.inbox.name + "/_design/WorkQueue/_update/in-place/" optionsArg = {} if "options" in updatedParams: optionsArg.update(updatedParams.pop("options")) data = { "updates": json.dumps(updatedParams), "options": json.dumps(optionsArg) } for ele in elementIds: thisuri = uri + ele + "?" + urllib.urlencode(data) self.inbox.makeRequest(uri=thisuri, type='PUT') return
def availableWork(self, conditions, teams=None, wfs=None): """Get work which is available to be run""" elements = [] for site in conditions.keys(): if not conditions[site] > 0: del conditions[site] if not conditions: return elements, conditions options = {} options['include_docs'] = True options['descending'] = True options['resources'] = conditions if teams: options['teams'] = teams if wfs: result = [] for i in xrange(0, len(wfs), 20): options['wfs'] = wfs[i:i + 20] data = self.db.loadList('WorkQueue', 'workRestrictions', 'availableByPriority', options) result.extend(json.loads(data)) # sort final list result.sort(key=lambda x: x[ 'WMCore.WorkQueue.DataStructs.WorkQueueElement.WorkQueueElement' ]['Priority']) else: result = self.db.loadList('WorkQueue', 'workRestrictions', 'availableByPriority', options) result = json.loads(result) for i in result: element = CouchWorkQueueElement.fromDocument(self.db, i) elements.append(element) # Remove 1st random site that can run work names = conditions.keys() random.shuffle(names) for site in names: if element.passesSiteRestriction(site): slots_left = conditions[site] - element['Jobs'] if slots_left > 0: conditions[site] = slots_left else: conditions.pop(site, None) break if not conditions: break return elements, conditions
def test_json(self): """ Test default json implementation. """ result = json_wrap.dumps(self.record) expect = json.dumps(self.record) self.assertEqual(expect, result) data = result result = json_wrap.loads(data) expect = json.loads(data) self.assertEqual(expect, result) with self.assertRaises(ValueError): json_wrap.loads("bogusbogus")
def check_resident(self, site, stage_data): """ Checks that the requested data is resident on the site """ goodToGo = True # Format site name locSite = site.replace('_Buffer', '_MSS') if not locSite.endswith('_MSS'): locSite += '_MSS' qParams = {'node': locSite} # Get block info from PhEDEx phedex = Requests(url='https://cmsweb.cern.ch', idict={'accept_type':'application/json'}) for data in stage_data: if data.find('#') < 0: qParams['dataset'] = data else: qParams['block'] = data self.logger.debug('Checking data residency for %s at %s' % (data, locSite)) try: pdata = phedex.get('/phedex/datasvc/json/prod/blockreplicas', qParams)[0] except httplib.HTTPException, he: self.handleHTTPExcept(he, 'HTTPException for block: %s node: %s' % (data, locSite)) # Parse block info and check > 0 block exist try: if len(JsonWrapper.loads(pdata)['phedex']['block']) == 0: goodToGo = False self.logger.error('Block %s not resident at site %s' % (data, locSite)) except: self.logger.debug('error loading json') goodToGo = False
def makeSchema(self, **schema): schema.setdefault('CouchURL', Utilities.removePasswordFromUrl(self.couchUrl)) schema.setdefault('CouchDBName', self.configDBName) decodedSchema = {} for key in schema.keys(): try: decodedSchema[key] = JsonWrapper.loads(schema[key]) except: # We don't know what kind of exception we'll get, so ignore them all # If it does except, it probably wasn't in JSON to begin with. # Anything else should be caught by the parsers and the validation decodedSchema[key] = schema[key] try: self.info("Creating a request for: '%s'\n\tworkloadDB: '%s'\n\twmstatUrl: " "'%s' ..." % (decodedSchema, self.workloadDBName, Utilities.removePasswordFromUrl(self.wmstatWriteURL))) request = Utilities.makeRequest(self, decodedSchema, self.couchUrl, self.workloadDBName, self.wmstatWriteURL) # catching here KeyError is just terrible except (RuntimeError, KeyError, Exception) as ex: # TODO problem not to expose logs to the client # e.g. on ConfigCacheID not found, the entire CouchDB traceback is sent in ex_message self.error("Create request failed, reason: %s" % ex) if hasattr(ex, "name"): detail = ex.name else: detail = "check logs." msg = "Create request failed, %s" % detail raise cherrypy.HTTPError(400, msg) baseURL = cherrypy.request.base raise cherrypy.HTTPRedirect('%s/reqmgr/view/details/%s' % (baseURL, request['RequestName']))
def json(self, data): if isinstance(data, GeneratorType): out = ''.join([r for r in self.genstreamer(data)]) return out thunker = JSONThunker() data = thunker.thunk(data) return JsonWrapper.dumps(data)
def splitting(self, requestName): """ _splitting_ Retrieve the current values for splitting parameters for all tasks in the spec. Format them in the manner that the splitting page expects and pass them to the template. """ self.validate(requestName) request = GetRequest.getRequestByName(requestName) helper = Utilities.loadWorkload(request) splittingDict = helper.listJobSplittingParametersByTask(performance = False) taskNames = splittingDict.keys() taskNames.sort() splitInfo = [] for taskName in taskNames: jsonSplittingParams = JsonWrapper.dumps(splittingDict[taskName]) splitInfo.append({"splitAlgo": splittingDict[taskName]["algorithm"], "splitParams": jsonSplittingParams, "taskType": splittingDict[taskName]["type"], "taskName": taskName}) return self.templatepage("Splitting", requestName = requestName, taskInfo = splitInfo, taskNames = taskNames)
def _getResult(self, callname, clearCache = True, args = None, verb = "GET"): """ _getResult_ retrieve JSON/XML formatted information given the service name and the argument dictionaries TODO: Probably want to move this up into Service """ result = '' file = callname.replace("/", "_") if clearCache: self.clearCache(file, args, verb) f = self.refreshCache(file, callname, args, verb = verb) result = f.read() f.close() # if self.responseType == "json": # decoder = json.JSONDecoder() # return decoder.decode(result) if result: result = JsonWrapper.loads(result) return result
def _makeHash(self, inputdata, hash): """ Turn the input data into json and hash the string. This is simple and means that the input data must be json-serialisable, which is good. """ json_hash = json.dumps(inputdata) return json_hash.__hash__()
def _validateMultiRequests(self, param, safe, valFunc): data = cherrypy.request.body.read() if data: request_names, request_args = self._getMultiRequestArgs(JsonWrapper.loads(data)) else: # actually this is error case #cherrypy.log(str(param.kwargs)) request_names, request_args = self._getMultiRequestArgs(param.kwargs) for prop in request_args: if prop == "RequestStatus": del param.kwargs["new_status"] else: del param.kwargs[prop] del param.kwargs["ids"] #remove this #tmp = [] #for prop in param.kwargs: # tmp.append(prop) #for prop in tmp: # del param.kwargs[prop] safe.kwargs['workload_pair_list'] = [] for request_name in request_names: request_args["RequestName"] = request_name workload, r_args = valFunc(request_args, self.config, self.reqmgr_db_service, param) safe.kwargs['workload_pair_list'].append((workload, r_args)) safe.kwargs["multi_update_flag"] = True
def availableWork(self, conditions, teams = None, wfs = None): """Get work which is available to be run""" elements = [] for site in conditions.keys(): if not conditions[site] > 0: del conditions[site] if not conditions: return elements, conditions options = {} options['include_docs'] = True options['descending'] = True options['resources'] = conditions if teams: options['teams'] = teams if wfs: options['wfs'] = wfs result = self.db.loadList('WorkQueue', 'workRestrictions', 'availableByPriority', options) result = json.loads(result) for i in result: element = CouchWorkQueueElement.fromDocument(self.db, i) elements.append(element) # Remove 1st random site that can run work names = conditions.keys() random.shuffle(names) for site in names: if element.passesSiteRestriction(site): slots_left = conditions[site] - element['Jobs'] if slots_left > 0: conditions[site] = slots_left else: conditions.pop(site, None) break return elements, conditions
def setUp(self): self.setConfig(config) self.setCouchDBs([(config.views.data.couch_reqmgr_db, "ReqMgr"), (config.views.data.couch_reqmgr_aux_db, None)]) self.setSchemaModules([]) RESTBaseUnitTestWithDBBackend.setUp(self) self.setFakeDN() requestPath = os.path.join(getWMBASE(), "test", "data", "ReqMgr", "requests", "DMWM") rerecoFile = open(os.path.join(requestPath, "ReReco.json"), 'r') rerecoArgs = JsonWrapper.load(rerecoFile) self.rerecoCreateArgs = rerecoArgs["createRequest"] self.rerecoAssignArgs = rerecoArgs["assignRequest"] cmsswDoc = {"_id": "software"} cmsswDoc[self.rerecoCreateArgs["ScramArch"]] = [] cmsswDoc[self.rerecoCreateArgs["ScramArch"]].append( self.rerecoCreateArgs["CMSSWVersion"]) insertDataToCouch(os.getenv("COUCHURL"), config.views.data.couch_reqmgr_aux_db, cmsswDoc) self.reqSvc = ReqMgr(self.jsonSender["host"]) self.reqSvc._noStale = True self.reqSvc['requests'].additionalHeaders = self.create_header
def _validateRequestBase(self, param, safe, valFunc, requestName=None): data = cherrypy.request.body.read() if data: request_args = JsonWrapper.loads(data) if requestName: request_args["RequestName"] = requestName if isinstance(request_args, dict): request_args = [request_args] else: # actually this is error case # cherrypy.log(str(param.kwargs)) request_args = {} for prop in param.kwargs: request_args[prop] = param.kwargs[prop] for prop in request_args: del param.kwargs[prop] if requestName: request_args["RequestName"] = requestName request_args = [request_args] safe.kwargs['workload_pair_list'] = [] if isinstance(request_args, dict): request_args = [request_args] for args in request_args: workload, r_args = valFunc(args, self.config, self.reqmgr_db_service, param) safe.kwargs['workload_pair_list'].append((workload, r_args))
def _validateMultiRequests(self, param, safe, valFunc): data = cherrypy.request.body.read() if data: request_names, request_args = self._getMultiRequestArgs(JsonWrapper.loads(data)) else: # actually this is error case # cherrypy.log(str(param.kwargs)) request_names, request_args = self._getMultiRequestArgs(param.kwargs) for prop in request_args: if prop == "RequestStatus": del param.kwargs["new_status"] else: del param.kwargs[prop] del param.kwargs["ids"] # remove this # tmp = [] # for prop in param.kwargs: # tmp.append(prop) # for prop in tmp: # del param.kwargs[prop] safe.kwargs['workload_pair_list'] = [] for request_name in request_names: request_args["RequestName"] = request_name workload, r_args = valFunc(request_args, self.config, self.reqmgr_db_service, param) safe.kwargs['workload_pair_list'].append((workload, r_args)) safe.kwargs["multi_update_flag"] = True
def _getResult(self, callname, clearCache=True, args=None, verb="GET"): """ _getResult_ retrieve JSON/XML formatted information given the service name and the argument dictionaries TODO: Probably want to move this up into Service """ result = '' file = callname.replace("/", "_") if clearCache: self.clearCache(file, args, verb) f = self.refreshCache(file, callname, args, verb=verb) result = f.read() f.close() # if self.responseType == "json": # decoder = json.JSONDecoder() # return decoder.decode(result) if result: result = JsonWrapper.loads(result) return result
def splitting(self, requestName): """ _splitting_ Retrieve the current values for splitting parameters for all tasks in the spec. Format them in the manner that the splitting page expects and pass them to the template. """ self.validate(requestName) request = GetRequest.getRequestByName(requestName) helper = Utilities.loadWorkload(request) splittingDict = helper.listJobSplittingParametersByTask() taskNames = splittingDict.keys() taskNames.sort() splitInfo = [] for taskName in taskNames: jsonSplittingParams = JsonWrapper.dumps(splittingDict[taskName]) splitInfo.append({"splitAlgo": splittingDict[taskName]["algorithm"], "splitParams": jsonSplittingParams, "taskType": splittingDict[taskName]["type"], "taskName": taskName}) return self.templatepage("Splitting", requestName = requestName, taskInfo = splitInfo, taskNames = taskNames)
def _validateRequestBase(self, param, safe, valFunc, requestName=None): data = cherrypy.request.body.read() if data: request_args = JsonWrapper.loads(data) if requestName: request_args["RequestName"] = requestName if isinstance(request_args, dict): request_args = [request_args] else: # actually this is error case #cherrypy.log(str(param.kwargs)) request_args = {} for prop in param.kwargs: request_args[prop] = param.kwargs[prop] for prop in request_args: del param.kwargs[prop] if requestName: request_args["RequestName"] = requestName request_args = [request_args] safe.kwargs['workload_pair_list'] = [] if isinstance(request_args, dict): request_args = [request_args] for args in request_args: workload, r_args = valFunc(args, self.config, self.reqmgr_db_service, param) safe.kwargs['workload_pair_list'].append((workload, r_args))
def putRequest(self, requestName=None, status=None, priority=None): request = None if requestName: request = self.findRequest(requestName) if request == None: # Create a new request, with a JSON-encoded schema that is # sent in the body of the HTTP request body = cherrypy.request.body.read() reqInputArgs = Utilities.unidecode(JsonWrapper.loads(body)) reqInputArgs.setdefault( 'CouchURL', Utilities.removePasswordFromUrl(self.couchUrl)) reqInputArgs.setdefault('CouchDBName', self.configDBName) try: self.info( "Creating a request for: '%s'\n\tworkloadDB: '%s'\n\twmstatUrl: " "'%s' ..." % (reqInputArgs, self.workloadDBName, Utilities.removePasswordFromUrl(self.wmstatWriteURL))) request = Utilities.makeRequest(self, reqInputArgs, self.couchUrl, self.workloadDBName, self.wmstatWriteURL) except cherrypy.HTTPError as ex: self.error("Create request failed, reason: %s" % ex) # Assume that this is a valid HTTPError raise except (WMException, Exception) as ex: # TODO problem not to expose logs to the client # e.g. on ConfigCacheID not found, the entire CouchDB traceback is sent in ex_message self.error("Create request failed, reason: %s" % ex) if hasattr(ex, "name"): detail = ex.name else: detail = "check logs." msg = "Create request failed, %s" % detail raise cherrypy.HTTPError(400, msg) self.info("Request '%s' created." % request['RequestName']) # see if status & priority need to be upgraded if status != None: # forbid assignment here if status == 'assigned': raise cherrypy.HTTPError( 403, "Cannot change status without a team. Please use PUT /reqmgr/reqMgr/assignment/<team>/<requestName>" ) try: Utilities.changeStatus(requestName, status, self.wmstatWriteURL) except RuntimeError as ex: # ignore some of these errors: https://svnweb.cern.ch/trac/CMSDMWM/ticket/2002 if status != 'announced' and status != 'closed-out': self.error("RuntimeError while changeStatus: reason: %s" % ex) raise cherrypy.HTTPError( 403, "Failed to change status: %s" % str(ex)) if priority != None: Utilities.changePriority(requestName, priority, self.wmstatWriteURL) return request
def updateInboxElements(self, *elementIds, **updatedParams): """Update given inbox element's (identified by id) with new parameters""" uri = "/" + self.inbox.name + "/_design/WorkQueue/_update/in-place/" data = {"updates" : json.dumps(updatedParams)} for ele in elementIds: thisuri = uri + ele + "?" + urllib.urlencode(data) self.inbox.makeRequest(uri = thisuri, type = 'PUT') return
def makeLumiList(lumiDict): try: if isinstance(lumiDict, basestring): lumiDict = JsonWrapper.loads(lumiDict) ll = LumiList(compactList=lumiDict) return ll.getCompactList() except: raise WMSpecFactoryException("Could not parse LumiList, %s: %s" % (type(lumiDict), lumiDict))
def wrapper (self, data, expires, contentType = "application/json"): data = func (self, data) try: # jsondata = encoder.iterencode(data) jsondata = JsonWrapper.dumps(data) _setCherryPyHeaders(jsondata, contentType, expires) return jsondata except: raise
def sites(siteDbUrl): """ download a list of all the sites from siteDB """ data = JsonWrapper.loads(urllib.urlopen(siteDbUrl).read().replace("'", '"')) # kill duplicates, then put in alphabetical order siteset = set([d['name'] for d in data.values()]) # warning: alliteration sitelist = list(siteset) sitelist.sort() return sitelist
def availableWork(self, conditions, teams = None, wfs = None): """Get work which is available to be run""" elements = [] for site in conditions.keys(): if not conditions[site] > 0: del conditions[site] if not conditions: return elements, conditions options = {} options['include_docs'] = True options['descending'] = True options['resources'] = conditions if teams: options['teams'] = teams if wfs: result = [] for i in xrange(0, len(wfs), 20): options['wfs'] = wfs[i:i+20] data = self.db.loadList('WorkQueue', 'workRestrictions', 'availableByPriority', options) result.extend(json.loads(data)) # sort final list result.sort(key = lambda x: x['WMCore.WorkQueue.DataStructs.WorkQueueElement.WorkQueueElement']['Priority']) else: result = self.db.loadList('WorkQueue', 'workRestrictions', 'availableByPriority', options) result = json.loads(result) for i in result: element = CouchWorkQueueElement.fromDocument(self.db, i) elements.append(element) # Remove 1st random site that can run work names = conditions.keys() random.shuffle(names) for site in names: if element.passesSiteRestriction(site): slots_left = conditions[site] - element['Jobs'] if slots_left > 0: conditions[site] = slots_left else: conditions.pop(site, None) break if not conditions: break return elements, conditions
def test_json(self): """ Test default json implementation. """ json_wrap._module = "json" result = json_wrap.dumps(self.record) expect = json.dumps(self.record) self.assertEqual(expect, result) data = result result = json_wrap.loads(data) expect = json.loads(data) self.assertEqual(expect, result) try: json_wrap.loads("bogusbogus") except ValueError as ex: self.assertEqual(ex.args, ("No JSON object could be decoded: bogusbogus",))
def updateElements(self, *elementIds, **updatedParams): """Update given element's (identified by id) with new parameters""" if not elementIds: return uri = "/" + self.db.name + "/_design/WorkQueue/_update/in-place/" data = {"updates": json.dumps(updatedParams)} for ele in elementIds: thisuri = uri + ele + "?" + urllib.urlencode(data) self.db.makeRequest(uri=thisuri, type='PUT') return
def sites(siteDbUrl): """ download a list of all the sites from siteDB """ data = JsonWrapper.loads( urllib.urlopen(siteDbUrl).read().replace("'", '"')) # kill duplicates, then put in alphabetical order siteset = set([d['name'] for d in data.values()]) # warning: alliteration sitelist = list(siteset) sitelist.sort() return sitelist
def setUp(self): """ setUP global values """ RESTBaseUnitTest.setUp(self) reqMgrHost = self.config.getServerUrl() self.requestSchema = getRequestSchema() print reqMgrHost self.jsonSender = JSONRequests(reqMgrHost) #self.requestTypes = ['ReReco', 'StoreResults', 'CmsGen', 'Reco'] #self.requestTypes = ['ReReco', 'MonteCarlo'] self.requestTypes = ['ReReco'] if 'me' in self.jsonSender.get('user')[0]: self.jsonSender.delete('user/me') self.assertFalse('me' in self.jsonSender.get('user')[0]) self.assertEqual(self.jsonSender.put('user/[email protected]')[1], 200) self.assertTrue('me' in self.jsonSender.get('user')[0]) if 'PeopleLikeMe' in self.jsonSender.get('group')[0]: self.jsonSender.delete('group/PeopleLikeMe') self.assertFalse('PeopleLikeMe' in self.jsonSender.get('group')[0]) self.assertEqual(self.jsonSender.put('group/PeopleLikeMe')[1], 200) self.assertTrue( 'PeopleLikeMe' in self.jsonSender.get('group')[0]) self.jsonSender.put('group/PeopleLikeMe/me') users = json.loads(self.jsonSender.get('group/PeopleLikeMe')[0])['users'] self.assertTrue('me' in users) groups = json.loads(self.jsonSender.get('user/me')[0])['groups'] self.assertTrue('PeopleLikeMe' in groups) groups2 = self.jsonSender.get('group?user=me')[0] self.assertTrue('PeopleLikeMe' in groups2) if 'White Sox' in self.jsonSender.get('team')[0]: self.jsonSender.delete(urllib.quote('team/White Sox')) self.assertFalse('White Sox' in self.jsonSender.get('team')[0]) self.assertEqual(self.jsonSender.put(urllib.quote('team/White Sox'))[1], 200) self.assertTrue('White Sox' in self.jsonSender.get('team')[0]) # some foreign key stuff to dealwith #self.assertFalse('CMSSW_X_Y_Z' in self.jsonSender.get('version')[0]) self.assertTrue(self.jsonSender.put('version/CMSSW_3_5_8')[1] == 200) self.assertTrue('CMSSW_3_5_8' in self.jsonSender.get('version')[0])
def sites(): # download all the sites from siteDB url = 'https://cmsweb.cern.ch/sitedb/json/index/CEtoCMSName?name' data = JsonWrapper.loads(urllib.urlopen(url).read().replace("'", '"')) # kill duplicates, then put in alphabetical order siteset = set([d['name'] for d in data.values()]) # warning: alliteration sitelist = list(siteset) sitelist.sort() return sitelist
def updateElements(self, *elementIds, **updatedParams): """Update given element's (identified by id) with new parameters""" import urllib uri = "/" + self.db.name + "/_design/WorkQueue/_update/in-place/" data = {"updates": json.dumps(updatedParams)} for ele in elementIds: thisuri = uri + ele + "?" + urllib.urlencode(data) answer = self.db.makeRequest(uri=thisuri, type="PUT") return
def test_json(self): """ Test default json implementation. """ json_wrap._module = "json" result = json_wrap.dumps(self.record) expect = json.dumps(self.record) self.assertEqual(expect, result) data = result result = json_wrap.loads(data) expect = json.loads(data) self.assertEqual(expect, result) try: json_wrap.loads("bogusbogus") except ValueError, ex: self.assertEqual(ex.args, ("No JSON object could be decoded: bogusbogus",))
def wrapper (self, data, expires, contentType = "application/json+thunk"): data = func (self, data) try: thunker = JSONThunker() data = thunker.thunk(data) jsondata = JsonWrapper.dumps(data) _setCherryPyHeaders(jsondata, contentType, expires) return jsondata except: raise
def setUp(self): filePath = os.path.join( getTestBase(), "WMCore_t/WorkQueue_t/DataStructs_t/wq_available_elements.json") with open(filePath, "r") as f: gqData = JsonWrapper.load(f) self.gqElements = [] for ele in gqData: self.gqElements.append(WorkQueueElement(**ele))
def setUp(self): filePath = os.path.join(getTestBase(), "WMCore_t/WorkQueue_t/DataStructs_t/wq_available_elements.json") with open(filePath, "r") as f: gqData = JsonWrapper.load(f) self.gqElements = [] for ele in gqData: self.gqElements.append(WorkQueueElement(**ele))
def validateStats(self, index): """ Check the values for the updates """ if not index.has_key('stats'): return index index['stats'] = Utilities.unidecode(JsonWrapper.loads(index['stats'])) for k in ['input_lummis', 'input_num_files', 'input_events', 'total_jobs']: if k in index['stats']: index['stats'][k] = int(index['stats'][k]) return index
def validateStats(self, index): """ Check the values for the updates """ if 'stats' not in index: return index index['stats'] = Utilities.unidecode(JsonWrapper.loads(index['stats'])) for k in ['input_lumis', 'input_num_files', 'input_events', 'total_jobs']: if k in index['stats']: index['stats'][k] = int(index['stats'][k]) return index
def getElements(self, status = None, elementIDs = None, returnIdOnly = False, db = None, loadSpec = False, WorkflowName = None, **elementFilters): """Return elements that match requirements status, elementIDs & filters are 'AND'ed together to filter elements. returnIdOnly causes the element not to be loaded and only the id returned db is used to specify which database to return from loadSpec causes the workflow for each spec to be loaded. WorkflowName may be used in the place of RequestName """ key = [] if not db: db = self.db if elementFilters.get('RequestName') and not WorkflowName: WorkflowName = elementFilters.pop('RequestName') if elementIDs: if elementFilters or status or returnIdOnly: raise ValueError, "Can't specify extra filters (or return id's) when using element id's with getElements()" elements = [CouchWorkQueueElement(db, i).load() for i in elementIDs] else: options = {'include_docs' : True, 'filter' : elementFilters, 'idOnly' : returnIdOnly, 'reduce' : False} # filter on workflow or status if possible filter = 'elementsByWorkflow' if WorkflowName: key.append(WorkflowName) elif status: filter = 'elementsByStatus' key.append(status) elif elementFilters.get('SubscriptionId'): key.append(elementFilters['SubscriptionId']) filter = 'elementsBySubscription' # add given params to filters if status: options['filter']['Status'] = status if WorkflowName: options['filter']['RequestName'] = WorkflowName view = db.loadList('WorkQueue', 'filter', filter, options, key) view = json.loads(view) if returnIdOnly: return view elements = [CouchWorkQueueElement.fromDocument(db, row) for row in view] if loadSpec: specs = {} # cache as may have multiple elements for same spec for ele in elements: if ele['RequestName'] not in specs: wmspec = self.getWMSpec(ele['RequestName']) specs[ele['RequestName']] = wmspec ele['WMSpec'] = specs[ele['RequestName']] del specs return elements
def set_opts(self, curl, url, params, headers, ckey=None, cert=None, capath=None, verbose=None, verb='GET', doseq=True, cainfo=None): """Set options for given curl object param needs to be a dictionary in case of GET, while PUT and POST assume it is a string already encoded/quoted with urllib.encode and urllib.quote. """ curl.setopt(pycurl.NOSIGNAL, self.nosignal) curl.setopt(pycurl.TIMEOUT, self.timeout) curl.setopt(pycurl.CONNECTTIMEOUT, self.connecttimeout) curl.setopt(pycurl.FOLLOWLOCATION, self.followlocation) curl.setopt(pycurl.MAXREDIRS, self.maxredirs) if params and verb != 'GET': if isinstance(params, dict): params = json.dumps(params) if verb == 'GET': encoded_data = urllib.urlencode(params, doseq=doseq) url = url + '?' + encoded_data elif verb == 'POST': curl.setopt(pycurl.POST, 1) if params: curl.setopt(pycurl.POSTFIELDS, params) elif verb == 'DELETE' or verb == 'PUT': curl.setopt(pycurl.CUSTOMREQUEST, verb) curl.setopt(pycurl.HTTPHEADER, ['Transfer-Encoding: chunked']) curl.setopt(pycurl.POSTFIELDS, params) else: raise Exception('Unsupported HTTP method "%s"' % verb) curl.setopt(pycurl.URL, url) curl.setopt(pycurl.HTTPHEADER, \ ["%s: %s" % (k, v) for k, v in headers.items()]) bbuf = StringIO.StringIO() hbuf = StringIO.StringIO() curl.setopt(pycurl.WRITEFUNCTION, bbuf.write) curl.setopt(pycurl.HEADERFUNCTION, hbuf.write) if capath: curl.setopt(pycurl.CAPATH, capath) curl.setopt(pycurl.SSL_VERIFYPEER, True) if cainfo: curl.setopt(pycurl.CAINFO, cainfo) else: curl.setopt(pycurl.SSL_VERIFYPEER, False) if ckey: curl.setopt(pycurl.SSLKEY, ckey) if cert: curl.setopt(pycurl.SSLCERT, cert) if verbose: curl.setopt(pycurl.VERBOSE, 1) curl.setopt(pycurl.DEBUGFUNCTION, self.debug) return bbuf, hbuf
def process_files(self, stage_data = [], request_id=''): """ Contact PhEDEx data service to get a list of files for a given request. TODO: use Service.PhEDEx """ db = self.localcouch.connectDatabase('%s_stagequeue' % self.site) # TODO: make the phedex URL a configurable! phedex = Requests(url='https://cmsweb.cern.ch', idict={'accept_type':'application/json'}) totalFiles = 0 totalBytes = 0 # Need to clean up the input a bit for d in stage_data: qParams = {'node': self.node} # Need to pass in blocks if d.find('#') < 0: qParams['dataset'] = d else: qParams['block'] = d try: pdata = phedex.get('/phedex/datasvc/json/prod/filereplicas', qParams)[0] except httplib.HTTPException, he: self.handleHTTPExcept(he, 'HTTPException for block: %s node: %s' % (d, self.node)) # Parse block info try: p = JsonWrapper.loads(pdata) if len(p['phedex']['block']) == 0: self.logger.error('Block %s not resident at site %s' % (d, self.node)) else: self.logger.debug('Creating stage-in requests for %s' % self.node) for blk in p['phedex']['block']: self.logger.debug('Creating stage-in requests for block %s' % blk['name']) for fle in blk['file']: self.logger.debug('Creating stage-in requests for file %s' % fle['name']) checksum = fle['checksum'] f = {'_id': fle['name'], 'bytes': int(fle['bytes']), 'checksum': {checksum.split(':')[0]: checksum.split(':')[1]}, 'state': 'new', 'retry_count': [], 'request_id': request_id } try: db.queue(f, timestamp = True, viewlist=['stagequeue/file_state']) totalFiles += 1 totalBytes += f['bytes'] except httplib.HTTPException, he: self.handleHTTPExcept(he, 'Could not commit data') except: self.logger.debug('error loading json')
def test_string_compare(self): """ Test that cjson and json libraries do the same thing. """ try: import cjson except: raise nose.SkipTest json_wrap._module = "cjson" json_wrap._module = "cjson" cj_result = json_wrap.dumps(self.record) json_wrap._module = "json" dj_result = json_wrap.dumps(self.record) self.assertEqual(dj_result, cj_result) data = dj_result json_wrap._module = "cjson" cj_result = json_wrap.loads(data) json_wrap._module = "json" dj_result = json_wrap.loads(data) self.assertEqual(dj_result, cj_result)
def parse_body(self, data, decode=False): """ Parse body part of URL request (by default use json). This method can be overwritten. """ if decode: try: res = json.loads(data) except ValueError as exc: msg = 'Unable to load JSON data\n%s' % data raise ValueError(exc) return res else: return data
def jsonHandler(self, args, kwargs): """ TODO: Ugly temporary hack to convert unicode to string for kwargs Only tested on python26 built-in json """ #if get verb doesn't have request.boby #TODO: maybe this should filtered on upper level if request.body != None: from WMCore.Wrappers import JsonWrapper params = request.body.read() if params: kw = JsonWrapper.loads(params) kwargs.update(kw) return args, kwargs
def abortRequest(requestName): conn = httplib.HTTPConnection("vocms144.cern.ch:8687") conn.request("GET", "/reqmgr/reqMgr/request?requestName=%s" % requestName) resp = conn.getresponse() status = json_wrap.loads(resp.read())["WMCore.RequestManager.DataStructs.Request.Request"]["RequestStatus"] print "Status: %s" % status if status == "acquired": os.system("curl -X PUT -d \"requestName=%s&status=running\" \"http://vocms144:8687/reqmgr/reqMgr/request\"" % requestName) os.system("curl -X PUT -d \"requestName=%s&status=aborted\" \"http://vocms144:8687/reqmgr/reqMgr/request\"" % requestName) else: os.system("curl -X PUT -d \"requestName=%s&status=rejected\" \"http://vocms144:8687/reqmgr/reqMgr/request\"" % requestName) return
def loadRequestSchema(workload, requestSchema): """ _loadRequestSchema_ Does modifications to the workload I don't understand Takes a WMWorkloadHelper, operates on it directly with the schema """ schema = workload.data.request.section_('schema') for key, value in requestSchema.iteritems(): if type(value) == dict and key == 'LumiList': value = JsonWrapper.dumps(value) try: setattr(schema, key, value) except Exception as ex: # Attach TaskChain tasks if type(value) == dict and requestSchema[ 'RequestType'] == 'TaskChain' and 'Task' in key: newSec = schema.section_(key) for k, v in requestSchema[key].iteritems(): if type(value) == dict and key == 'LumiList': value = JsonWrapper.dumps(value) try: setattr(newSec, k, v) except Exception as ex: # this logging need to change to cherry py logging logging.error("Invalid Value: %s" % str(ex)) else: # this logging need to change to cherry py logging logging.error("Invalid Value: %s" % str(ex)) schema.timeStamp = int(time.time()) schema = workload.data.request.schema # might belong in another method to apply existing schema workload.data.owner.Group = schema.Group workload.data.owner.Requestor = schema.Requestor
def handleAssignmentPage(self, **kwargs): """ handler for the main page """ #Accept Json encoded strings decodedArgs = {} for key in kwargs.keys(): try: decodedArgs[key] = JsonWrapper.loads(kwargs[key]) except: #Probably wasn't JSON decodedArgs[key] = kwargs[key] kwargs = decodedArgs # handle the checkboxes teams = [] requestNames = [] for key, value in kwargs.iteritems(): if isinstance(value, types.StringTypes): kwargs[key] = value.strip() if key.startswith("Team"): teams.append(key[4:]) if key.startswith("checkbox"): requestName = key[8:] self.validate(requestName) requestNames.append(key[8:]) for requestName in requestNames: if kwargs['action'] == 'Reject': ChangeState.changeRequestStatus(requestName, 'rejected', wmstatUrl=self.wmstatWriteURL) else: assignments = GetRequest.getAssignmentsByName(requestName) if teams == [] and assignments == []: raise cherrypy.HTTPError( 400, "Must assign to one or more teams") kwargs["Teams"] = teams self.assignWorkload(requestName, kwargs) for team in teams: if not team in assignments: ChangeState.assignRequest( requestName, team, wmstatUrl=self.wmstatWriteURL) priority = kwargs.get(requestName + ':priority', '') if priority != '': Utilities.changePriority(requestName, priority, self.wmstatWriteURL) participle = kwargs['action'] + 'ed' return self.templatepage("Acknowledge", participle=participle, requests=requestNames)
def parse_body(self, data, decode=False): """ Parse body part of URL request (by default use json). This method can be overwritten. """ if decode: try: res = json.loads(data) except ValueError as exc: msg = 'Unable to load JSON data, %s, data type=%s, pass as is' \ % (str(exc), type(data)) logging.warning(msg) return data return data else: return data
def setUp(self): self.setConfig(config) self.setCouchDBs([(config.views.data.couch_reqmgr_db, "ReqMgr"), (config.views.data.couch_reqmgr_aux_db, None)]) self.setSchemaModules([]) RESTBaseUnitTestWithDBBackend.setUp(self) # put into ReqMgr auxiliary database under "software" document scram/cmsms # which we'll need a little for request injection #Warning: this assumes the same structure in jenkins wmcore_root/test requestPath = os.path.join(getWMBASE(), "test", "data", "ReqMgr", "requests") mcFile = open(os.path.join(requestPath, "MonteCarlo.json"), 'r') self.mcArgs = JsonWrapper.load(mcFile)["createRequest"] cmsswDoc = {"_id": "software"} cmsswDoc[self.mcArgs["ScramArch"]] = [] cmsswDoc[self.mcArgs["ScramArch"]].append(self.mcArgs["CMSSWVersion"]) insertDataToCouch(os.getenv("COUCHURL"), config.views.data.couch_reqmgr_aux_db, cmsswDoc)
def makeSchema(self, **schema): schema.setdefault('CouchURL', Utilities.removePasswordFromUrl(self.couchUrl)) # wrong naming ... but it's all over the place, it's the config cache DB name schema.setdefault('CouchDBName', self.configDBName) schema.setdefault('CouchWorkloadDBName', self.workloadDBName) decodedSchema = {} for key in schema.keys(): try: decodedSchema[key] = JsonWrapper.loads(schema[key]) except: # We don't know what kind of exception we'll get, so ignore them all # If it does except, it probably wasn't in JSON to begin with. # Anything else should be caught by the parsers and the validation decodedSchema[key] = schema[key] try: self.info( "Creating a request for: '%s'\n\tworkloadDB: '%s'\n\twmstatUrl: " "'%s' ..." % (decodedSchema, self.workloadDBName, Utilities.removePasswordFromUrl(self.wmstatWriteURL))) request = Utilities.makeRequest(self, decodedSchema, self.couchUrl, self.workloadDBName, self.wmstatWriteURL) # catching here KeyError is just terrible except (RuntimeError, KeyError, Exception) as ex: # TODO problem not to expose logs to the client # e.g. on ConfigCacheID not found, the entire CouchDB traceback is sent in ex_message self.error("Create request failed, reason: %s" % str(ex)) if hasattr(ex, "message"): if hasattr(ex.message, '__call__'): detail = ex.message() else: detail = str(ex) elif hasattr(ex, "name"): detail = ex.name else: detail = "check logs." msg = "Create request failed, %s" % detail raise cherrypy.HTTPError(400, msg) baseURL = cherrypy.request.base raise cherrypy.HTTPRedirect('%s/reqmgr/view/details/%s' % (baseURL, request['RequestName']))