def buildWorkloadAndCheckIn(webApi, reqSchema, couchUrl, couchDB, wmstatUrl, clone=False): """ If clone is True, the function is called on a cloned request in which case no modification of the reqSchema shall happen and should be checked in as is. """ try: request = buildWorkloadForRequest(typename=reqSchema["RequestType"], schema=reqSchema) except WMSpecFactoryException, ex: raise HTTPError(400, "Error in Workload Validation: %s" % ex._message)
def buildWorkloadAndCheckIn(webApi, reqSchema, couchUrl, couchDB, wmstatUrl, clone=False): """ If clone is True, the function is called on a cloned request in which case no modification of the reqSchema shall happen and should be checked in as is. """ try: request = buildWorkloadForRequest(typename = reqSchema["RequestType"], schema = reqSchema) except WMSpecFactoryException, ex: raise HTTPError(400, "Error in Workload Validation: %s" % str(ex))
def makeRequest(kwargs, couchUrl, couchDB, wmstatUrl): logging.info(kwargs) """ Handles the submission of requests """ # make sure no extra spaces snuck in for k, v in kwargs.iteritems(): if isinstance(v, str): kwargs[k] = v.strip() # Create a new schema schema = RequestSchema() schema.update(kwargs) currentTime = time.strftime('%y%m%d_%H%M%S', time.localtime(time.time())) secondFraction = int(10000 * (time.time()%1.0)) requestString = schema.get('RequestString', "") if requestString != "": schema['RequestName'] = "%s_%s_%s_%s" % ( schema['Requestor'], requestString, currentTime, secondFraction) else: schema['RequestName'] = "%s_%s_%s" % (schema['Requestor'], currentTime, secondFraction) schema["Campaign"] = kwargs.get("Campaign", "") if 'ProcScenario' in kwargs and 'ProcConfigCacheID' in kwargs: # Use input mode to delete the unused one inputMode = kwargs.get('inputMode', None) if inputMode == 'scenario': del schema['ProcConfigCacheID'] if 'EnableDQMHarvest' not in kwargs: schema["EnableHarvesting"] = False if kwargs.has_key("InputDataset"): schema["InputDatasets"] = [kwargs["InputDataset"]] if kwargs.has_key("FilterEfficiency"): kwargs["FilterEfficiency"] = float(kwargs["FilterEfficiency"]) skimNumber = 1 # a list of dictionaries schema["SkimConfigs"] = [] while kwargs.has_key("SkimName%s" % skimNumber): d = {} d["SkimName"] = kwargs["SkimName%s" % skimNumber] d["SkimInput"] = kwargs["SkimInput%s" % skimNumber] d["Scenario"] = kwargs["Scenario"] if kwargs.get("Skim%sConfigCacheID" % skimNumber, None) != None: d["ConfigCacheID"] = kwargs["Skim%sConfigCacheID" % skimNumber] schema["SkimConfigs"].append(d) skimNumber += 1 if kwargs.has_key("DataPileup") or kwargs.has_key("MCPileup"): schema["PileupConfig"] = {} if kwargs.has_key("DataPileup") and kwargs["DataPileup"] != "": schema["PileupConfig"]["data"] = [kwargs["DataPileup"]] if kwargs.has_key("MCPileup") and kwargs["MCPileup"] != "": schema["PileupConfig"]["mc"] = [kwargs["MCPileup"]] for runlist in ["RunWhitelist", "RunBlacklist"]: if runlist in kwargs: schema[runlist] = parseRunList(kwargs[runlist]) for blocklist in ["BlockWhitelist", "BlockBlacklist"]: if blocklist in kwargs: schema[blocklist] = parseBlockList(kwargs[blocklist]) if "DqmSequences" in kwargs: schema["DqmSequences"] = parseDqmSequences(kwargs["DqmSequences"]) validate(schema) # Get the DN schema['RequestorDN'] = cherrypy.request.user.get('dn', 'unknown') try: request = buildWorkloadForRequest(typename = kwargs["RequestType"], schema = schema) except WMSpecFactoryException, ex: msg = ex._message raise HTTPError(400, "Error in Workload Validation: %s" % msg)
def buildWorkloadAndCheckIn(webApi, reqSchema, couchUrl, couchDB, wmstatUrl, clone=False): """ If clone is True, the function is called on a cloned request in which case no modification of the reqSchema shall happen and should be checked in as is. """ try: request = buildWorkloadForRequest(typename = reqSchema["RequestType"], schema = reqSchema) except WMSpecFactoryException as ex: logging.error(traceback.format_exc()) raise HTTPError(400, "Error in Workload Validation: %s" % ex.message()) helper = WMWorkloadHelper(request['WorkloadSpec']) # update request as well for wmstats update # there is a better way to do this (passing helper to request but make sure all the information is there) request["Campaign"] = helper.getCampaign() # Add the output datasets if necessary # for some bizarre reason OutpuDatasets is list of lists, when cloning # [['/MinimumBias/WMAgentCommissioning10-v2/RECO'], ['/MinimumBias/WMAgentCommissioning10-v2/ALCARECO']] # #3743 if not clone: for ds in helper.listOutputDatasets(): if ds not in request['OutputDatasets']: request['OutputDatasets'].append(ds) #TODO: need to update output dataset by Task for task chain requests # can't save Request object directly, because it makes it hard to retrieve the _rev metadata = {} metadata.update(request) # don't want to JSONify the whole workflow del metadata['WorkloadSpec'] workloadUrl = helper.saveCouch(couchUrl, couchDB, metadata=metadata) request['RequestWorkflow'] = removePasswordFromUrl(workloadUrl) try: CheckIn.checkIn(request, reqSchema['RequestType']) except CheckIn.RequestCheckInError as ex: raise HTTPError(400, "Error in Request check-in: %s" % str(ex)) # Inconsistent request parameters between Oracle and Couch (#4380, #4388) # metadata above is what is saved into couch to represent a request document. # Number of request arguments on a corresponding couch document # is not set, has default null/None values, update those accordingly now. # It's a mess to have two mutually inconsistent database backends. # Not easy to handle this earlier since couch is stored first and # some parameters are worked out later when storing into Oracle. reqDetails = requestDetails(request["RequestName"]) # couchdb request parameters which are null at the injection time and remain so paramsToUpdate = ["RequestStatus", "RequestSizeFiles", "AcquisitionEra", "RequestWorkflow", "RequestType", "RequestStatus", "RequestPriority", "Requestor", "Group", "SizePerEvent", "PrepID", "RequestNumEvents", "ProcessingString", "ProcessingVersion", ] couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"]) fields = {} for key in paramsToUpdate: fields[key] = reqDetails[key] couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest", fields=fields, useBody=True) try: wmstatSvc = WMStatsWriter(wmstatUrl) wmstatSvc.insertRequest(request) except Exception as ex: webApi.error("Could not update WMStats, reason: %s" % ex) raise HTTPError(400, "Creating request failed, could not update WMStats.") return request
def buildWorkloadAndCheckIn(webApi, reqSchema, couchUrl, couchDB, wmstatUrl, clone=False): """ If clone is True, the function is called on a cloned request in which case no modification of the reqSchema shall happen and should be checked in as is. """ try: request = buildWorkloadForRequest(typename=reqSchema["RequestType"], schema=reqSchema) except WMSpecFactoryException as ex: logging.error(traceback.format_exc()) raise HTTPError(400, "Error in Workload Validation: %s" % ex.message()) helper = WMWorkloadHelper(request['WorkloadSpec']) # update request as well for wmstats update # there is a better way to do this (passing helper to request but make sure all the information is there) request["Campaign"] = helper.getCampaign() # Add the output datasets if necessary # for some bizarre reason OutpuDatasets is list of lists, when cloning # [['/MinimumBias/WMAgentCommissioning10-v2/RECO'], ['/MinimumBias/WMAgentCommissioning10-v2/ALCARECO']] # #3743 if not clone: for ds in helper.listOutputDatasets(): if ds not in request['OutputDatasets']: request['OutputDatasets'].append(ds) #TODO: need to update output dataset by Task for task chain requests # can't save Request object directly, because it makes it hard to retrieve the _rev metadata = {} metadata.update(request) # don't want to JSONify the whole workflow del metadata['WorkloadSpec'] workloadUrl = helper.saveCouch(couchUrl, couchDB, metadata=metadata) request['RequestWorkflow'] = removePasswordFromUrl(workloadUrl) try: CheckIn.checkIn(request, reqSchema['RequestType']) except CheckIn.RequestCheckInError as ex: raise HTTPError(400, "Error in Request check-in: %s" % str(ex)) # Inconsistent request parameters between Oracle and Couch (#4380, #4388) # metadata above is what is saved into couch to represent a request document. # Number of request arguments on a corresponding couch document # is not set, has default null/None values, update those accordingly now. # It's a mess to have two mutually inconsistent database backends. # Not easy to handle this earlier since couch is stored first and # some parameters are worked out later when storing into Oracle. reqDetails = requestDetails(request["RequestName"]) # couchdb request parameters which are null at the injection time and remain so paramsToUpdate = [ "RequestStatus", "RequestSizeFiles", "AcquisitionEra", "RequestWorkflow", "RequestType", "RequestStatus", "RequestPriority", "Requestor", "Group", "SizePerEvent", "PrepID", "RequestNumEvents", "ProcessingString", "ProcessingVersion", ] couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"]) fields = {} for key in paramsToUpdate: fields[key] = reqDetails[key] couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest", fields=fields, useBody=True) try: wmstatSvc = WMStatsWriter(wmstatUrl) wmstatSvc.insertRequest(request) except Exception as ex: webApi.error("Could not update WMStats, reason: %s" % ex) raise HTTPError(400, "Creating request failed, could not update WMStats.") return request