def testA_SoftwareManagement(self): """ _SoftwareManagement_ Test the SoftwareManagement code """ self.assertEqual(SoftwareManagement.listSoftware(), {}) softwareVersions = ReqMgrWebTools.allScramArchsAndVersions() ReqMgrWebTools.updateScramArchsAndCMSSWVersions() result = SoftwareManagement.listSoftware() for scramArch in result.keys(): self.assertEqual(set(result[scramArch]), set(softwareVersions[scramArch])) # Now for each scramArch insert a blank set # Because of the way that updateSoftware works, this interprets a blank list # as telling it that no softwareVersions are available. # It deletes every software version it is not handed, so it should give nothing out. for scramArch in result.keys(): SoftwareManagement.updateSoftware(softwareNames=[], scramArch=scramArch) self.assertEqual(SoftwareManagement.listSoftware(), {}) # import has to be here, otherwise getting: # AttributeError: 'Toolbox' object has no attribute 'secmodv2' from the Admin module from WMCore.HTTPFrontEnd.RequestManager import Admin setattr(self.config, 'database', self.testInit.coreConfig.CoreDatabase) self.config.section_('templates') self.config.section_('html') admin = Admin.Admin(self.config) ReqMgrWebTools.updateScramArchsAndCMSSWVersions() self.assertTrue('slc5_amd64_gcc434' in admin.scramArchs())
def testH_RemoveSoftwareVersion(self): """ _RemoveSoftwareVersion_ Remove the software version after submitting the request. See what that does. """ myThread = threading.currentThread() userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) result = self.jsonSender.put('request/testRequest', schema) self.assertEqual(result[1], 200) requestName = result[0]['RequestName'] req = self.jsonSender.get('request/%s' % requestName)[0] self.assertEqual(req['SoftwareVersions'], [schema["CMSSWVersion"]]) # Delete software versions and make sure they're gone from the DB SoftwareManagement.removeSoftware(softwareName = schema["CMSSWVersion"], scramArch = schema["ScramArch"]) versions = myThread.dbi.processData("SELECT * FROM reqmgr_software")[0].fetchall() self.assertEqual(versions, []) assocs = myThread.dbi.processData("SELECT * FROM reqmgr_software_dependency")[0].fetchall() self.assertEqual(assocs, []) req = self.jsonSender.get('request/%s' % requestName)[0] self.assertEqual(req['SoftwareVersions'], [schema["CMSSWVersion"]])
def index(self): """ Main web page for creating requests """ # Get the scram Architecture from the keys and the # CMSSW versions from the values self.scramArchs = SoftwareAdmin.listSoftware().keys() versionLists = SoftwareAdmin.listSoftware().values() self.versions = [] for l in versionLists: for v in l: if not v in self.versions: self.versions.append(v) self.versions.sort() # see if this was configured with a hardcoded user. If not, take from the request header requestor = self.requestor if not requestor: requestor = cherrypy.request.user["login"] if not requestor: return "No username found in your certificate" if not requestor in Registration.listUsers(): return "User %s is not registered. Contact a ReqMgr administrator." % requestor groups = GroupInfo.groupsForUser(requestor).keys() if groups == []: return "User " + requestor + " is not in any groups. Contact a ReqMgr administrator." campaigns = Campaign.listCampaigns() return self.templatepage("WebRequestSchema", yuiroot=self.yuiroot, requestor=requestor, groups=groups, versions=self.versions, archs = self.scramArchs, alldocs = Utilities.unidecode(self.allDocs()), allcampaigns = campaigns, defaultVersion=self.cmsswVersion, defaultSkimConfig=self.defaultSkimConfig)
def testA_SoftwareManagement(self): """ _SoftwareManagement_ Test the SoftwareManagement code """ self.assertEqual(SoftwareManagement.listSoftware(), {}) softwareVersions = ReqMgrWebTools.allScramArchsAndVersions() ReqMgrWebTools.updateScramArchsAndCMSSWVersions() result = SoftwareManagement.listSoftware() for scramArch in result.keys(): self.assertEqual(set(result[scramArch]), set(softwareVersions[scramArch])) # Now for each scramArch insert a blank set # Because of the way that updateSoftware works, this interprets a blank list # as telling it that no softwareVersions are available. # It deletes every software version it is not handed, so it should give nothing out. for scramArch in result.keys(): SoftwareManagement.updateSoftware(softwareNames = [], scramArch = scramArch) self.assertEqual(SoftwareManagement.listSoftware(), {}) from WMCore.HTTPFrontEnd.RequestManager import Admin setattr(self.config, 'database', self.testInit.coreConfig.CoreDatabase) self.config.section_('templates') self.config.section_('html') admin = Admin.Admin(self.config) ReqMgrWebTools.updateScramArchsAndCMSSWVersions() self.assertTrue('slc5_amd64_gcc434' in admin.scramArchs()) return
def index(self): """ Main web page for creating requests """ # Get the scram Architecture from the keys and the # CMSSW versions from the values self.scramArchs = SoftwareAdmin.listSoftware().keys() versionLists = SoftwareAdmin.listSoftware().values() self.versions = [] for l in versionLists: for v in l: if not v in self.versions: self.versions.append(v) self.versions.sort() # see if this was configured with a hardcoded user. If not, take from the request header requestor = self.requestor if not requestor: requestor = cherrypy.request.user["login"] if not requestor: return "No username found in your certificate" if not requestor in Registration.listUsers(): return "User %s is not registered. Contact a ReqMgr administrator." % requestor groups = GroupInfo.groupsForUser(requestor).keys() if groups == []: return "User " + requestor + " is not in any groups. Contact a ReqMgr administrator." campaigns = Campaign.listCampaigns() return self.templatepage("WebRequestSchema", yuiroot=self.yuiroot, requestor=requestor, groups=groups, versions=self.versions, archs=self.scramArchs, alldocs=Utilities.unidecode(self.allDocs()), allcampaigns=campaigns, defaultVersion=self.cmsswVersion, defaultArch=self.defaultArch, defaultSkimConfig=self.defaultSkimConfig)
def testH_RemoveSoftwareVersion(self): """ _RemoveSoftwareVersion_ Remove the software version after submitting the request. See what that does. """ myThread = threading.currentThread() userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) result = self.jsonSender.put('request/testRequest', schema) self.assertEqual(result[1], 200) requestName = result[0]['RequestName'] req = self.jsonSender.get('request/%s' % requestName)[0] self.assertEqual(req['SoftwareVersions'], [schema["CMSSWVersion"]]) # Delete software versions and make sure they're gone from the DB SoftwareManagement.removeSoftware(softwareName=schema["CMSSWVersion"], scramArch=schema["ScramArch"]) versions = myThread.dbi.processData( "SELECT * FROM reqmgr_software")[0].fetchall() self.assertEqual(versions, []) assocs = myThread.dbi.processData( "SELECT * FROM reqmgr_software_dependency")[0].fetchall() self.assertEqual(assocs, []) req = self.jsonSender.get('request/%s' % requestName)[0] self.assertEqual(req['SoftwareVersions'], [schema["CMSSWVersion"]])
def testH_RemoveSoftwareVersion(self): """ _RemoveSoftwareVersion_ Remove the software version after submitting the request. See what that does. """ myThread = threading.currentThread() userName = "******" groupName = "Li" teamName = "Tang" CMSSWVersion = "CMSSW_3_5_8" scramArch = "slc5_ia32_gcc434" schema = self.setupSchema( userName=userName, groupName=groupName, teamName=teamName, CMSSWVersion=CMSSWVersion, scramArch=scramArch ) result = self.jsonSender.put("request/testRequest", schema) self.assertEqual(result[1], 200) requestName = result[0]["RequestName"] req = self.jsonSender.get("request/%s" % requestName)[0] self.assertEqual(req["SoftwareVersions"], [CMSSWVersion]) # Delete software versions and make sure they're gone from the DB SoftwareManagement.removeSoftware(softwareName=CMSSWVersion, scramArch=scramArch) versions = myThread.dbi.processData("SELECT * FROM reqmgr_software")[0].fetchall() self.assertEqual(versions, []) assocs = myThread.dbi.processData("SELECT * FROM reqmgr_software_dependency")[0].fetchall() self.assertEqual(assocs, []) req = self.jsonSender.get("request/%s" % requestName)[0] self.assertEqual(req["SoftwareVersions"], [CMSSWVersion]) return
def handleAllVersions(self): """ Registers all versions in the TC """ currentVersions = SoftwareAdmin.listSoftware().keys() allVersions = Utilities.allSoftwareVersions() result = "" for version in allVersions: if not version in currentVersions: WMCore.Lexicon.cmsswversion(version) SoftwareAdmin.addSoftware(version) result += "Added version %s<br/>" % version if result == "": result = "Version list is up to date" return result
def checkIn(request): """ _CheckIn_ Check in of a request manager Given a new request, check it in to the DB and add the appropriate IDs. """ # // # // First try and register the request in the DB #// requestName = request['RequestName'] # test if the software versions are registered first versions = SoftwareManagement.listSoftware() for version in request.get('SoftwareVersions', []): if not version in versions: raise RuntimeError, "Cannot find software version %s in ReqMgr" % version try: reqId = MakeRequest.createRequest( request['Requestor'], request['Group'], requestName, request['RequestType'], request['RequestWorkflow'], request.get('PrepID', None) ) except Exception, ex: msg = "Error creating new request:\n" msg += str(ex) raise RuntimeError, msg
def versions(self): """ Lists all versions """ versions = SoftwareAdmin.listSoftware().keys() versions.sort() for version in versions: WMCore.Lexicon.cmsswversion(version) return self.templatepage("Versions", versions=versions)
def scramArchs(self): """ _scramArchs_ List all scramArchs in the DB Prelim for putting this in the template pages """ return SoftwareAdmin.listSoftware().keys()
def updateScramArchsAndCMSSWVersions(): """ _updateScramArchsAndCMSSWVersions_ Update both the scramArchs and their associated software versions to the current tag collector standard. """ allArchsAndVersions = allScramArchsAndVersions() if allArchsAndVersions == {}: # The tag collector is probably down # NO valid CMSSW Versions is not a valid use case! # Do nothing. logging.error("Handed blank list of scramArchs/versions. Ignoring for this cycle.") return for scramArch in allArchsAndVersions.keys(): SoftwareAdmin.updateSoftware(scramArch=scramArch, softwareNames=allArchsAndVersions[scramArch])
def updateScramArchsAndCMSSWVersions(): """ _updateScramArchsAndCMSSWVersions_ Update both the scramArchs and their associated software versions to the current tag collector standard. """ allArchsAndVersions = allScramArchsAndVersions() if allArchsAndVersions == {}: # The tag collector is probably down # NO valid CMSSW Versions is not a valid use case! # Do nothing. logging.error("Handed blank list of scramArchs/versions. Ignoring for this cycle.") return for scramArch in allArchsAndVersions.keys(): SoftwareAdmin.updateSoftware(scramArch = scramArch, softwareNames = allArchsAndVersions[scramArch])
def getVersion(self): """ Returns a list of all CMSSW versions registered with ReqMgr """ archList = SoftwareAdmin.listSoftware() result = [] for arch in archList.keys(): for version in archList[arch]: if not version in result: result.append(version) return result
def versions(self): """ Lists all versions """ archList = SoftwareAdmin.listSoftware() versions = [] for versionList in archList.values(): for version in versionList: if not version in versions: versions.append(version) versions.sort() for version in versions: WMCore.Lexicon.cmsswversion(version) return self.templatepage("Versions", versions=versions)
def checkIn(request, requestType = 'None'): """ _CheckIn_ Check in of a request manager Given a new request, check it in to the DB and add the appropriate IDs. """ # // # // First try and register the request in the DB #// requestName = request['RequestName'] # test if the software versions are registered first versions = SoftwareManagement.listSoftware() scramArch = request.get('ScramArch') if requestType.lower() in ['resubmission']: # Do nothing, as we have no valid software version yet pass else: if not scramArch in versions.keys(): m = ("Cannot find scramArch %s in ReqMgr (the one(s) available: %s)" % (scramArch, versions)) raise RequestCheckInError(m) for version in request.get('SoftwareVersions', []): if not version in versions[scramArch]: raise RequestCheckInError("Cannot find software version %s in ReqMgr for " "scramArch %s. Supported versions: %s" % (version, scramArch, versions[scramArch])) try: reqId = MakeRequest.createRequest( request['Requestor'], request['Group'], requestName, request['RequestType'], request['RequestWorkflow'], request.get('PrepID', None), request.get('RequestPriority', None) ) except Exception, ex: msg = "Error creating new request:\n" msg += str(ex) raise RequestCheckInError( msg )
def checkIn(request, requestType = 'None'): """ _CheckIn_ Check in of a request manager Given a new request, check it in to the DB and add the appropriate IDs. """ # // # // First try and register the request in the DB #// requestName = request['RequestName'] # test if the software versions are registered first versions = SoftwareManagement.listSoftware() scramArch = request.get('ScramArch') if requestType.lower() in ['resubmission']: # Do nothing, as we have no valid software version yet pass else: if not scramArch in versions.keys(): m = ("Cannot find scramArch %s in ReqMgr (the one(s) available: %s)" % (scramArch, versions)) raise RequestCheckInError(m) for version in request.get('SoftwareVersions', []): if not version in versions[scramArch]: raise RequestCheckInError("Cannot find software version %s in ReqMgr for " "scramArch %s. Supported versions: %s" % (version, scramArch, versions[scramArch])) try: reqId = MakeRequest.createRequest( request['Requestor'], request['Group'], requestName, request['RequestType'], request['RequestWorkflow'], request.get('PrepID', None), request.get('ReqMgrRequestBasePriority', None) ) except Exception, ex: msg = "Error creating new request:\n" msg += str(ex) raise RequestCheckInError( msg )
def checkIn(request, requestType="None", wmstatSvc=None): """ _CheckIn_ Check in of a request manager Given a new request, check it in to the DB and add the appropriate IDs. """ # // # // First try and register the request in the DB # // requestName = request["RequestName"] # test if the software versions are registered first versions = SoftwareManagement.listSoftware() scramArch = request.get("ScramArch") if requestType.lower() in ["resubmission"]: # Do nothing, as we have no valid software version yet pass else: if not scramArch in versions.keys(): m = "Cannot find scramArch %s in ReqMgr (the one(s) available: %s)" % (scramArch, versions) raise RequestCheckInError(m) for version in request.get("SoftwareVersions", []): if not version in versions[scramArch]: raise RequestCheckInError( "Cannot find software version %s in ReqMgr for scramArch %s" % (version, scramArch) ) try: reqId = MakeRequest.createRequest( request["Requestor"], request["Group"], requestName, request["RequestType"], request["RequestWorkflow"], request.get("PrepID", None), ) except Exception, ex: msg = "Error creating new request:\n" msg += str(ex) raise RequestCheckInError(msg)
def checkIn(request, requestType="None"): """ _CheckIn_ Check in of a request manager Given a new request, check it in to the DB and add the appropriate IDs. """ # // # // First try and register the request in the DB # // requestName = request["RequestName"] # test if the software versions are registered first versions = SoftwareManagement.listSoftware() scramArch = request.get("ScramArch") if requestType.lower() in ["resubmission"]: # Do nothing, as we have no valid software version yet pass else: if not scramArch in versions.keys(): m = "Cannot find scramArch %s in ReqMgr (the one(s) available: %s)" % (scramArch, versions) raise RequestCheckInError(m) for version in request.get("SoftwareVersions", []): if not version in versions[scramArch]: raise RequestCheckInError( "Cannot find software version %s in ReqMgr for " "scramArch %s. Supported versions: %s" % (version, scramArch, versions[scramArch]) ) try: reqId = MakeRequest.createRequest( request["Requestor"], request["Group"], requestName, request["RequestType"], request["RequestWorkflow"], request.get("PrepID", None), request.get("RequestPriority", None), ) except Exception as ex: msg = "Error creating new request:\n" msg += str(ex) raise RequestCheckInError(msg) # FIXME LAST_INSERT_ID doesn't work on oracle reqId = GetRequest.requestID(requestName) request["RequestID"] = reqId logging.info("Request %s created with request id %s" % (requestName, request["RequestID"])) # // # // add metadata about the request # // try: if request["InputDatasetTypes"] != {}: for ds, dsType in request["InputDatasetTypes"].items(): MakeRequest.associateInputDataset(requestName, ds, dsType) elif isinstance(request["InputDatasets"], list): for ds in request["InputDatasets"]: MakeRequest.associateInputDataset(requestName, ds) else: MakeRequest.associateInputDataset(requestName, request["InputDatasets"]) except Exception as ex: _raiseCheckInError(request, ex, "Unable to Associate input datasets to request") try: for ds in request["OutputDatasets"]: # request['OutputDatasets'] may contain a list of lists (each sublist for a task) # which is actually not understood why but seems to be correct (Steve) # dirty if isinstance(ds, list): for dss in ds: MakeRequest.associateOutputDataset(requestName, dss) else: MakeRequest.associateOutputDataset(requestName, ds) except Exception as ex: _raiseCheckInError(request, ex, "Unable to Associate output datasets to request") try: for sw in request["SoftwareVersions"]: MakeRequest.associateSoftware(requestName, sw) except Exception as ex: _raiseCheckInError(request, ex, "Unable to associate software for this request") MakeRequest.updateRequestSize( requestName, request.get("RequestNumEvents", 0), request.get("RequestSizeFiles", 0), request.get("SizePerEvent", 0), ) campaign = request.get("Campaign", "") if campaign != "" and campaign != None: Campaign.associateCampaign(campaign, reqId) logging.info("Request '%s' built with request id '%s" "" % (requestName, request["RequestID"]))
def getVersion(self): """ Returns a list of all CMSSW versions registered with ReqMgr """ return SoftwareAdmin.listSoftware().keys()
def putVersion(self, version): """ Registers a new CMSSW version with ReqMgr """ return SoftwareAdmin.addSoftware(version)
def handleAddVersion(self, version): """ Registers a version """ WMCore.Lexicon.cmsswversion(version) SoftwareAdmin.updateSoftware(version) return "Added version %s" % version
def checkIn(request, requestType='None'): """ _CheckIn_ Check in of a request manager Given a new request, check it in to the DB and add the appropriate IDs. """ # // # // First try and register the request in the DB #// requestName = request['RequestName'] # test if the software versions are registered first versions = SoftwareManagement.listSoftware() scramArch = request.get('ScramArch') if requestType.lower() in ['resubmission']: # Do nothing, as we have no valid software version yet pass else: if not scramArch in versions.keys(): m = ( "Cannot find scramArch %s in ReqMgr (the one(s) available: %s)" % (scramArch, versions)) raise RequestCheckInError(m) for version in request.get('SoftwareVersions', []): if not version in versions[scramArch]: raise RequestCheckInError( "Cannot find software version %s in ReqMgr for " "scramArch %s. Supported versions: %s" % (version, scramArch, versions[scramArch])) try: reqId = MakeRequest.createRequest(request['Requestor'], request['Group'], requestName, request['RequestType'], request['RequestWorkflow'], request.get('PrepID', None), request.get('RequestPriority', None)) except Exception as ex: msg = "Error creating new request:\n" msg += str(ex) raise RequestCheckInError(msg) #FIXME LAST_INSERT_ID doesn't work on oracle reqId = GetRequest.requestID(requestName) request['RequestID'] = reqId logging.info("Request %s created with request id %s" % (requestName, request['RequestID'])) # // # // add metadata about the request #// try: if request['InputDatasetTypes'] != {}: for ds, dsType in request['InputDatasetTypes'].items(): MakeRequest.associateInputDataset(requestName, ds, dsType) elif isinstance(request['InputDatasets'], list): for ds in request['InputDatasets']: MakeRequest.associateInputDataset(requestName, ds) else: MakeRequest.associateInputDataset(requestName, request['InputDatasets']) except Exception as ex: _raiseCheckInError(request, ex, "Unable to Associate input datasets to request") try: for ds in request['OutputDatasets']: # request['OutputDatasets'] may contain a list of lists (each sublist for a task) # which is actually not understood why but seems to be correct (Steve) # dirty if isinstance(ds, list): for dss in ds: MakeRequest.associateOutputDataset(requestName, dss) else: MakeRequest.associateOutputDataset(requestName, ds) except Exception as ex: _raiseCheckInError(request, ex, "Unable to Associate output datasets to request") try: for sw in request['SoftwareVersions']: MakeRequest.associateSoftware(requestName, sw) except Exception as ex: _raiseCheckInError(request, ex, "Unable to associate software for this request") MakeRequest.updateRequestSize(requestName, request.get("RequestNumEvents", 0), request.get("RequestSizeFiles", 0), request.get("SizePerEvent", 0)) campaign = request.get("Campaign", "") if campaign != "" and campaign != None: Campaign.associateCampaign(campaign, reqId) logging.info("Request '%s' built with request id '%s" '' % (requestName, request['RequestID']))
def deleteVersion(self, version, scramArch): """ Un-register this software version with ReqMgr """ SoftwareAdmin.removeSoftware(version, scramArch)
def putVersion(self, version, scramArch = None): """ Registers a new CMSSW version with ReqMgr """ return SoftwareAdmin.addSoftware(version, scramArch = scramArch)
def putVersion(self, version, scramArch=None): """ Registers a new CMSSW version with ReqMgr """ return SoftwareAdmin.addSoftware(version, scramArch=scramArch)