コード例 #1
0
 def handleAnnounce(self, **kwargs):
     """ Handler for announcing requests """
     requests = self.requestNamesFromCheckboxes(kwargs)
     # the distinction good/bad datasets was based upon contacting
     # wrong DBS url so it was always giving rubbish,
     # perhaps the entire page is just not used at all
     datasets = []
     goodDatasets = []
     badDatasets = []
     for requestName in requests:
         WMCore.Lexicon.identifier(requestName)
         Utilities.changeStatus(requestName,
                                'announced',
                                wmstatUrl=self.wmstatWriteURL,
                                acdcUrl=self.acdcURL)
         datasets.extend(Utilities.getOutputForRequest(requestName))
     for dataset in datasets:
         # was needed for the DBS call to wrong DBS URL
         # check code before 2013-04-04
         #toks = dataset.split('/')
         #data = {'primary_ds_name': toks[0], 'processed_ds_name': toks[1],
         #        'data_tier_name': toks[2], 'is_dataset_valid': 1}
         goodDatasets.append(dataset)
     return self.templatepage("Announce",
                              requests=requests,
                              goodDatasets=goodDatasets,
                              badDatasets=badDatasets)
コード例 #2
0
ファイル: Admin_t.py プロジェクト: stuartw/WMCore
    def testA_SoftwareManagement(self):
        """
        _SoftwareManagement_

        Test the SoftwareManagement code
        """

        self.assertEqual(SoftwareManagement.listSoftware(), {})
        softwareVersions = ReqMgrWebTools.allScramArchsAndVersions()
        ReqMgrWebTools.updateScramArchsAndCMSSWVersions()
        result = SoftwareManagement.listSoftware()
        for scramArch in result.keys():
            self.assertEqual(set(result[scramArch]), set(softwareVersions[scramArch]))

        # Now for each scramArch insert a blank set
        # Because of the way that updateSoftware works, this interprets a blank list
        # as telling it that no softwareVersions are available.
        # It deletes every software version it is not handed, so it should give nothing out.
        for scramArch in result.keys():
            SoftwareManagement.updateSoftware(softwareNames = [], scramArch = scramArch)
        self.assertEqual(SoftwareManagement.listSoftware(), {})

        from WMCore.HTTPFrontEnd.RequestManager import Admin
        setattr(self.config, 'database', self.testInit.coreConfig.CoreDatabase)
        self.config.section_('templates')
        self.config.section_('html')
        admin = Admin.Admin(self.config)

        ReqMgrWebTools.updateScramArchsAndCMSSWVersions()
        self.assertTrue('slc5_amd64_gcc434' in admin.scramArchs())
        return
コード例 #3
0
ファイル: Assign.py プロジェクト: zhiwenuil/WMCore
    def __init__(self, config):
        WebAPI.__init__(self, config)
        ReqMgrAuth.assign_roles = config.security_roles
        # Take a guess
        self.templatedir = config.templates
        self.couchUrl = config.couchUrl
        self.clipboardDB = config.clipboardDB
        cleanUrl = Utilities.removePasswordFromUrl(self.couchUrl)
        self.clipboardUrl = "%s/%s/_design/OpsClipboard/index.html" % (cleanUrl, self.clipboardDB)
        self.hold = config.hold
        self.configDBName = config.configDBName
        self.sites = Utilities.sites(config.sitedb)
        self.allMergedLFNBases =  [
            "/store/backfill/1", "/store/backfill/2", 
            "/store/data",  "/store/mc"]
        self.allUnmergedLFNBases = ["/store/unmerged", "/store/temp"]

        self.mergedLFNBases = {
             "ReReco" : ["/store/backfill/1", "/store/backfill/2", "/store/data"],
             "DataProcessing" : ["/store/backfill/1", "/store/backfill/2", "/store/data"],
             "ReDigi" : ["/store/backfill/1", "/store/backfill/2", "/store/data", "/store/mc"],
             "MonteCarlo" : ["/store/backfill/1", "/store/backfill/2", "/store/mc"],
             "RelValMC" : ["/store/backfill/1", "/store/backfill/2", "/store/mc"],
             "Resubmission" : ["/store/backfill/1", "/store/backfill/2", "/store/mc", "/store/data"],
             "MonteCarloFromGEN" : ["/store/backfill/1", "/store/backfill/2", "/store/mc"]}

        self.yuiroot = config.yuiroot
        cherrypy.engine.subscribe('start_thread', self.initThread)
コード例 #4
0
ファイル: Assign_t.py プロジェクト: AndrewLevin/WMCore
    def testA_SiteWhitelist(self):
        """
        _SiteWhitelist_

        Test to see if we can get the siteWhitelist to work properly.
        
        """
        secconfig = getattr(self.config, "SecurityModule")
        cherrypy.server.environment = 'test'
        cherrypy.tools.secmodv2 = NullAuth(secconfig)
        
        self.config.UnitTests.views.active.rest.templates    = 'templateDir'
        self.config.UnitTests.views.active.rest.yuiroot      = 'yuiroot'
        self.config.UnitTests.views.active.rest.wildcardKeys = {'T1*': 'T1_*',
                                                                'T2*': 'T2_*',
                                                                'T3*': 'T3_*',
                                                                'US*': '_US_'}
        from WMCore.HTTPFrontEnd.RequestManager.Assign import Assign
        assign = Assign(config = self.config.UnitTests.views.active.rest, noSiteDB = True)

        siteList = ['T1_US_FNAL', 'T1_CH_CERN', 'T1_UK_RAL', 'T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT']
        self.assertEqual(assign.sites, [])
        assign.sites.extend(siteList)

        Utilities.addSiteWildcards(assign.wildcardKeys,assign.sites,assign.wildcardSites)
        for s in siteList:
            self.assertTrue(s in assign.sites)
        self.assertTrue('T1*' in assign.sites)
        self.assertTrue('T2*' in assign.sites)
        self.assertFalse('T3*' in assign.sites)
        self.assertTrue('US*' in assign.sites)

        self.assertEqual(assign.wildcardSites['T1*'], ['T1_US_FNAL', 'T1_CH_CERN', 'T1_UK_RAL'])
        self.assertEqual(assign.wildcardSites['T2*'], ['T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT'])
        self.assertEqual(assign.wildcardSites['US*'], ['T1_US_FNAL', 'T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT'])
コード例 #5
0
ファイル: Assign.py プロジェクト: zhiwenuil/WMCore
    def one(self,  requestName):
        """ Assign a single request """
        self.validate(requestName)
        request =  GetRequest.getRequestByName(requestName)
        request = Utilities.prepareForTable(request)
        requestType = request["RequestType"]
        # get assignments
        teams = ProdManagement.listTeams()
        assignments = GetRequest.getAssignmentsByName(requestName)
        # might be a list, or a dict team:priority
        if isinstance(assignments, dict):
            assignments = assignments.keys()

        procVer = ""
        acqEra = ""
        helper = Utilities.loadWorkload(request)
        if helper.getAcquisitionEra() != None:
            acqEra = helper.getAcquisitionEra()
            if helper.getProcessingVersion() != None:
                procVer = helper.getProcessingVersion()
        dashboardActivity = helper.getDashboardActivity()

        (reqMergedBase, reqUnmergedBase) = helper.getLFNBases()
        return self.templatepage("Assign", requests=[request], teams=teams, 
                                 assignments=assignments, sites=self.sites,
                                 mergedLFNBases=self.mergedLFNBases[requestType],
                                 reqMergedBase=reqMergedBase,
                                 unmergedLFNBases=self.allUnmergedLFNBases,
                                 reqUnmergedBase=reqUnmergedBase,
                                 acqEra = acqEra, procVer = procVer, 
                                 dashboardActivity=dashboardActivity,
                                 badRequests=[])
コード例 #6
0
ファイル: WebRequestSchema.py プロジェクト: zhiwenuil/WMCore
 def makeSchema(self, **schema):
     schema.setdefault('CouchURL', Utilities.removePasswordFromUrl(self.couchUrl))
     schema.setdefault('CouchDBName', self.configDBName)
     try:
         request = Utilities.makeRequest(schema, self.couchUrl, self.workloadDBName)
     except RuntimeError, e:
         raise cherrypy.HTTPError(400, "Error creating request: %s" % e)
コード例 #7
0
 def javascript(self, *args):
     if args[0] == "external":
         return Utilities.serveFile('application/javascript',
                          os.path.join(self.config.javascript), *args)
     return Utilities.serveFile('application/javascript',
                       os.path.join(self.config.javascript,
                                 'WMCore', 'WebTools'), *args)
コード例 #8
0
ファイル: WebRequestSchema.py プロジェクト: ticoann/WMCore
    def makeSchema(self, **schema):
        schema.setdefault('CouchURL', Utilities.removePasswordFromUrl(self.couchUrl))
        schema.setdefault('CouchDBName', self.configDBName)

        decodedSchema = {}
        for key in schema.keys():
            try:
                decodedSchema[key] = JsonWrapper.loads(schema[key])
            except:
                # We don't know what kind of exception we'll get, so ignore them all
                # If it does except, it probably wasn't in JSON to begin with.
                # Anything else should be caught by the parsers and the validation
                decodedSchema[key] = schema[key]
        try:
            self.info("Creating a request for: '%s'\n\tworkloadDB: '%s'\n\twmstatUrl: "
                      "'%s' ..." % (decodedSchema, self.workloadDBName,
                                    Utilities.removePasswordFromUrl(self.wmstatWriteURL)))
            request = Utilities.makeRequest(self, decodedSchema, self.couchUrl, self.workloadDBName, self.wmstatWriteURL)
            # catching here KeyError is just terrible
        except (RuntimeError, KeyError, Exception) as ex:
            # TODO problem not to expose logs to the client
            # e.g. on ConfigCacheID not found, the entire CouchDB traceback is sent in ex_message
            self.error("Create request failed, reason: %s" % ex)
            if hasattr(ex, "name"):
                detail = ex.name
            else:
                detail = "check logs." 
            msg = "Create request failed, %s" % detail
            raise cherrypy.HTTPError(400, msg)            
        baseURL = cherrypy.request.base
        raise cherrypy.HTTPRedirect('%s/reqmgr/view/details/%s' % (baseURL, request['RequestName']))
コード例 #9
0
    def testA_SoftwareManagement(self):
        """
        _SoftwareManagement_

        Test the SoftwareManagement code
        
        """
        self.assertEqual(SoftwareManagement.listSoftware(), {})
        softwareVersions = ReqMgrWebTools.allScramArchsAndVersions()
        ReqMgrWebTools.updateScramArchsAndCMSSWVersions()
        result = SoftwareManagement.listSoftware()
        for scramArch in result.keys():
            self.assertEqual(set(result[scramArch]),
                             set(softwareVersions[scramArch]))

        # Now for each scramArch insert a blank set
        # Because of the way that updateSoftware works, this interprets a blank list
        # as telling it that no softwareVersions are available.
        # It deletes every software version it is not handed, so it should give nothing out.
        for scramArch in result.keys():
            SoftwareManagement.updateSoftware(softwareNames=[],
                                              scramArch=scramArch)
        self.assertEqual(SoftwareManagement.listSoftware(), {})

        # import has to be here, otherwise getting:
        # AttributeError: 'Toolbox' object has no attribute 'secmodv2' from the Admin module
        from WMCore.HTTPFrontEnd.RequestManager import Admin
        setattr(self.config, 'database', self.testInit.coreConfig.CoreDatabase)
        self.config.section_('templates')
        self.config.section_('html')
        admin = Admin.Admin(self.config)

        ReqMgrWebTools.updateScramArchsAndCMSSWVersions()
        self.assertTrue('slc5_amd64_gcc434' in admin.scramArchs())
コード例 #10
0
 def javascript(self, *args):
     if args[0] == "external":
         return Utilities.serveFile('application/javascript',
                                    os.path.join(self.config.javascript),
                                    *args)
     return Utilities.serveFile(
         'application/javascript',
         os.path.join(self.config.javascript, 'WMCore', 'WebTools'), *args)
コード例 #11
0
    def cloneRequest(self, requestName):
        """
        Input assumes an existing request, checks that.
        The original existing request is not touched.
        A new request is generated.
        The cloned request has a newly generated RequestName, new timestamp,
        RequestDate, however -everything- else is copied from the original request.
        Addition: since Edgar changed his mind, he no longer
        wants this cloned request be in the 'new' state but put 
        straight into 'assignment-approved' state.
        
        """
        request = None
        if requestName:
            self.info("Cloning request: request name: '%s'" % requestName)
            requestOrigDict = self.getRequest(requestName)
            # add OriginalRequestName to the new request schema
            requestOrigDict['OriginalRequestName'] = requestName

            if requestOrigDict:
                self.info("Request found, cloning ...")
                newReqSchema = Utilities.getNewRequestSchema(requestOrigDict)
                # since we cloned the request, all the attributes
                # manipulation in Utilities.makeRequest() should not be necessary
                # the cloned request shall be identical but following arguments
                toRemove = ["RequestName", "RequestDate", "timeStamp"]
                toRemove.extend(deprecatedRequestArgs)

                for remove in toRemove:
                    try:
                        del requestOrigDict[remove]
                    except KeyError:
                        pass
                newReqSchema.update(requestOrigDict)  # clone

                # problem that priority wasn't preserved went down from here:
                # via CheckIn.checkIn() -> MakeRequest.createRequest() -> Request.New.py factory
                request = Utilities.buildWorkloadAndCheckIn(
                    self,
                    newReqSchema,
                    self.couchUrl,
                    self.workloadDBName,
                    self.wmstatWriteURL,
                    clone=True)
                # change the clone request state as desired
                Utilities.changeStatus(newReqSchema["RequestName"],
                                       "assignment-approved",
                                       self.wmstatWriteURL, self.acdcURL)
                return request
            else:
                msg = "Request '%s' not found." % requestName
                self.warning(msg)
                raise cherrypy.HTTPError(404, msg)
        else:
            msg = "Received empty request name: '%s', exit." % requestName
            self.warn(msg)
            raise cherrypy.HTTPError(400, msg)
コード例 #12
0
    def cloneRequest(self, requestName):
        """
        Input assumes an existing request, checks that.
        The original existing request is not touched.
        A new request is generated.
        The cloned request has a newly generated RequestName, new timestamp,
        RequestDate, however -everything- else is copied from the original request.
        Addition: since Edgar changed his mind, he no longer
        wants this cloned request be in the 'new' state but put 
        straight into 'assignment-approved' state.
        
        """
        request = None
        if requestName:
            self.info("Cloning request: request name: '%s'" % requestName)
            requestOrigDict = self.getRequest(requestName)
            # add OriginalRequestName to the new request schema
            requestOrigDict['OriginalRequestName'] = requestName

            if requestOrigDict:
                self.info("Request found, cloning ...")
                newReqSchema = Utilities.getNewRequestSchema(requestOrigDict)
                # since we cloned the request, all the attributes
                # manipulation in Utilities.makeRequest() should not be necessary
                # the cloned request shall be identical but following arguments
                toRemove = ["RequestName",
                            "RequestDate",
                            "timeStamp"]
                toRemove.extend(deprecatedRequestArgs)
                            
                for remove in toRemove:
                    try:
                        del requestOrigDict[remove]
                    except KeyError:
                        pass
                newReqSchema.update(requestOrigDict) # clone
                
                # problem that priority wasn't preserved went down from here:
                # via CheckIn.checkIn() -> MakeRequest.createRequest() -> Request.New.py factory
                request = Utilities.buildWorkloadAndCheckIn(self, newReqSchema,
                                                            self.couchUrl, self.workloadDBName,
                                                            self.wmstatWriteURL, clone=True)
                # change the clone request state as desired
                Utilities.changeStatus(newReqSchema["RequestName"],
                                       "assignment-approved",
                                        self.wmstatWriteURL,
                                        self.acdcURL)
                return request
            else:
                msg = "Request '%s' not found." % requestName
                self.warning(msg)
                raise cherrypy.HTTPError(404, msg)
        else:
            msg = "Received empty request name: '%s', exit." % requestName
            self.warn(msg)
            raise cherrypy.HTTPError(400, msg)
コード例 #13
0
ファイル: ReqMgrBrowser.py プロジェクト: tsarangi/WMCore
    def handleSplittingPage(self, requestName, splittingTask, splittingAlgo,
                            **submittedParams):
        """
        _handleSplittingPage_

        Parse job splitting parameters sent from the splitting parameter update
        page.  Pull down the request and modify the new spec applying the
        updated splitting parameters.
        """
        splitParams = {}
        if splittingAlgo == "FileBased":
            splitParams["files_per_job"] = int(submittedParams["files_per_job"])
        elif splittingAlgo == "TwoFileBased":
            splitParams["files_per_job"] = int(submittedParams["two_files_per_job"])
        elif splittingAlgo == "LumiBased":
            splitParams["lumis_per_job"] = int(submittedParams["lumis_per_job"])
            if str(submittedParams["halt_job_on_file_boundaries"]) == "True":
                splitParams["halt_job_on_file_boundaries"] = True
            else:
                splitParams["halt_job_on_file_boundaries"] = False
        elif splittingAlgo == "EventAwareLumiBased":
            splitParams["events_per_job"] = int(submittedParams["avg_events_per_job"])
            splitParams["max_events_per_lumi"] = int(submittedParams["max_events_per_lumi"])
            if str(submittedParams["halt_job_on_file_boundaries_event_aware"]) == "True":
                splitParams["halt_job_on_file_boundaries"] = True
            else:
                splitParams["halt_job_on_file_boundaries"] = False
        elif splittingAlgo == "EventBased":
            splitParams["events_per_job"] = int(submittedParams["events_per_job"])
            if "events_per_lumi" in submittedParams:
                splitParams["events_per_lumi"] = int(submittedParams["events_per_lumi"])
            if "lheInputFiles" in submittedParams:
                if str(submittedParams["lheInputFiles"]) == "True":
                    splitParams["lheInputFiles"] = True
                else:
                    splitParams["lheInputFiles"] = False
        elif splittingAlgo == "Harvest":
            splitParams["periodic_harvest_interval"] = int(submittedParams["periodic_harvest_interval"])
        elif 'Merg' in splittingTask:
            for field in ['min_merge_size', 'max_merge_size', 'max_merge_events', 'max_wait_time']:
                splitParams[field] = int(submittedParams[field])
        if "include_parents" in submittedParams.keys():
            if str(submittedParams["include_parents"]) == "True":
                splitParams["include_parents"] = True
            else:
                splitParams["include_parents"] = False

        self.validate(requestName)
        request = GetRequest.getRequestByName(requestName)
        helper = Utilities.loadWorkload(request)
        logging.info("SetSplitting " + requestName + splittingTask + splittingAlgo + str(splitParams))
        helper.setJobSplittingParameters(splittingTask, splittingAlgo, splitParams)
        Utilities.saveWorkload(helper, request['RequestWorkflow'])
        return "Successfully updated splitting parameters for " + splittingTask \
               + " " + detailsBackLink(requestName)
コード例 #14
0
    def assignWorkload(self, requestName, kwargs):
        """ Make all the necessary changes in the Workload to reflect the new assignment """
        request = GetRequest.getRequestByName(requestName)
        helper = Utilities.loadWorkload(request)
        for field in ["AcquisitionEra", "ProcessingVersion"]:
            if type(kwargs[field]) == dict:
                for value in kwargs[field].values():
                    self.validate(value, field)
            else:
                self.validate(kwargs[field], field)
        # Set white list and black list
        whiteList = kwargs.get("SiteWhitelist", [])
        blackList = kwargs.get("SiteBlacklist", [])
        helper.setSiteWildcardsLists(siteWhitelist=whiteList,
                                     siteBlacklist=blackList,
                                     wildcardDict=self.wildcardSites)
        # Set ProcessingVersion and AcquisitionEra, which could be json encoded dicts
        helper.setProcessingVersion(kwargs["ProcessingVersion"])
        helper.setAcquisitionEra(kwargs["AcquisitionEra"])
        #FIXME not validated
        helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"])
        helper.setMergeParameters(int(kwargs.get("MinMergeSize", 2147483648)),
                                  int(kwargs.get("MaxMergeSize", 4294967296)),
                                  int(kwargs.get("MaxMergeEvents", 50000)))
        helper.setupPerformanceMonitoring(
            int(kwargs.get("maxRSS", 2411724)),
            int(kwargs.get("maxVSize", 2411724)),
            int(kwargs.get("SoftTimeout", 167000)),
            int(kwargs.get("GracePeriod", 300)))

        # Check whether we should check location for the data
        if "useSiteListAsLocation" in kwargs:
            helper.setLocationDataSourceFlag()

        # Set phedex subscription information
        custodialList = kwargs.get("CustodialSites", [])
        nonCustodialList = kwargs.get("NonCustodialSites", [])
        if "AutoApprove" in kwargs:
            autoApproveList = nonCustodialList
        else:
            autoApproveList = []
        priority = kwargs.get("Priority", "Low")
        if priority not in ["Low", "Normal", "High"]:
            raise cherrypy.HTTPError(400, "Invalid subscription priority")

        helper.setSubscriptionInformationWildCards(
            wildcardDict=self.wildcardSites,
            custodialSites=custodialList,
            nonCustodialSites=nonCustodialList,
            autoApproveSites=autoApproveList,
            priority=priority)
        helper.setDashboardActivity(kwargs.get("dashboard", ""))
        Utilities.saveWorkload(helper, request['RequestWorkflow'],
                               self.wmstatWriteURL)
コード例 #15
0
ファイル: ReqMgrWebTools_t.py プロジェクト: ticoann/WMCore
 def testA_ParseRunList(self):
     l0 = ''
     l1 = ' [1,  2,  3 ] '
     l2 = '1,  2, 3   '
     l3 = u'1,  2, 3   '
     l4 = [1,2,3]
     l5 = {1:2, 3:4}
     self.assertEqual(ReqMgrWebTools.parseRunList(l0), [])
     self.assertEqual(ReqMgrWebTools.parseRunList(l1), [1,2,3])
     self.assertEqual(ReqMgrWebTools.parseRunList(l2), [1,2,3])
     self.assertEqual(ReqMgrWebTools.parseRunList(l3), [1,2,3])
     self.assertEqual(ReqMgrWebTools.parseRunList(l4), [1,2,3])
コード例 #16
0
ファイル: Assign.py プロジェクト: stuartw/WMCore
    def handleAssignmentPage(self, **kwargs):
        """ handler for the main page """
        #Accept Json encoded strings
        decodedArgs = {}
        for key in kwargs.keys():
            try:
                decodedArgs[key] = JsonWrapper.loads(kwargs[key])
            except:
                #Probably wasn't JSON
                decodedArgs[key] = kwargs[key]
        kwargs = decodedArgs
        # handle the checkboxes
        teams = []
        requestNames = []
        for key, value in kwargs.iteritems():
            if isinstance(value, types.StringTypes):
                kwargs[key] = value.strip()
            if key.startswith("Team"):
                teams.append(key[4:])
            if key.startswith("checkbox"):
                requestName = key[8:]
                self.validate(requestName)
                requestNames.append(key[8:])
        
        for requestName in requestNames:
            if kwargs['action'] == 'Reject':
                ChangeState.changeRequestStatus(requestName, 'rejected', wmstatUrl = self.wmstatWriteURL) 
            else:
                assignments = GetRequest.getAssignmentsByName(requestName)
                if teams == [] and assignments == []:
                    raise cherrypy.HTTPError(400, "Must assign to one or more teams")
                self.assignWorkload(requestName, kwargs)
                for team in teams:
                    if not team in assignments:
                        ChangeState.assignRequest(requestName, team, wmstatUrl = self.wmstatWriteURL)
                priority = kwargs.get(requestName+':priority', '')
                if priority != '':
                    Utilities.changePriority(requestName, priority, self.wmstatWriteURL)
        participle=kwargs['action']+'ed'
        if self.opshold and kwargs['action'] == 'Assign':
            participle='put into "ops-hold" state (see <a href="%s">OpsClipboard</a>)' % self.clipboardUrl
            # this, previously used, call made all requests injected into OpsClipboard to
            # have campaign_id null since the call doesn't propagate request's 
            # CampaignName at all, AcquisitionEra remains default null and probably
            # a bunch of other things is wrong too
            #requests = [GetRequest.getRequestByName(requestName) for requestName in requestNames]
            requests = [Utilities.requestDetails(requestName) for requestName in requestNames]
            OpsClipboard.inject(self.couchUrl, self.clipboardDB, *requests)
            for request in requestNames:
                ChangeState.changeRequestStatus(requestName, 'ops-hold', wmstatUrl = self.wmstatWriteURL)

        return self.templatepage("Acknowledge", participle=participle, requests=requestNames)
コード例 #17
0
ファイル: ReqMgrBrowser.py プロジェクト: tsarangi/WMCore
    def showWorkload(self, requestName):
        """ Displays the workload """
        self.validate(requestName)

        try:
            request = GetRequest.getRequestByName(requestName)
        except (Exception, RuntimeError) as ex:
            raise cherrypy.HTTPError(400, "Invalid request. %s" % str(ex))

        request = Utilities.prepareForTable(request)
        helper = Utilities.loadWorkload(request)
        workloadText = str(helper.data)
        return cgi.escape(workloadText).replace("\n", "<br/>\n")
コード例 #18
0
ファイル: ReqMgrBrowser.py プロジェクト: stuartw/WMCore
    def showWorkload(self, requestName):
        """ Displays the workload """
        self.validate(requestName)

        try:
            request = GetRequest.getRequestByName(requestName)
        except (Exception, RuntimeError) as ex:
            raise cherrypy.HTTPError(400, "Invalid request.")
        
        request = Utilities.prepareForTable(request)
        helper = Utilities.loadWorkload(request)
        workloadText = str(helper.data)
        return cgi.escape(workloadText).replace("\n", "<br/>\n")
コード例 #19
0
ファイル: Assign.py プロジェクト: mmascher/WMCore
    def one(self, requestName):
        """ Assign a single request """
        self.validate(requestName)
        request = GetRequest.getRequestByName(requestName)
        request = Utilities.prepareForTable(request)
        # get assignments
        teams = ProdManagement.listTeams()
        assignments = GetRequest.getAssignmentsByName(requestName)
        # might be a list, or a dict team:priority
        if isinstance(assignments, dict):
            assignments = assignments.keys()

        procVer = ""
        acqEra = ""
        procString = ""
        helper = Utilities.loadWorkload(request)
        if helper.getAcquisitionEra() != None:
            acqEra = helper.getAcquisitionEra()
        if helper.getProcessingVersion() != None:
            procVer = helper.getProcessingVersion()
        if helper.getProcessingString():
            procString = helper.getProcessingString()
        dashboardActivity = helper.getDashboardActivity()
        blockCloseMaxWaitTime = helper.getBlockCloseMaxWaitTime()
        blockCloseMaxFiles = helper.getBlockCloseMaxFiles()
        blockCloseMaxEvents = helper.getBlockCloseMaxEvents()
        blockCloseMaxSize = helper.getBlockCloseMaxSize()
        (reqMergedBase, reqUnmergedBase) = helper.getLFNBases()

        return self.templatepage(
            "Assign",
            requests=[request],
            teams=teams,
            assignments=assignments,
            sites=self.sites,
            phedexNodes=self.phedexNodes,
            mergedLFNBases=self.allMergedLFNBases,
            reqMergedBase=reqMergedBase,
            unmergedLFNBases=self.allUnmergedLFNBases,
            reqUnmergedBase=reqUnmergedBase,
            acqEra=acqEra,
            procVer=procVer,
            procString=procString,
            dashboardActivity=dashboardActivity,
            badRequests=[],
            blockCloseMaxWaitTime=blockCloseMaxWaitTime,
            blockCloseMaxFiles=blockCloseMaxFiles,
            blockCloseMaxSize=blockCloseMaxSize,
            blockCloseMaxEvents=blockCloseMaxEvents,
        )
コード例 #20
0
    def assignWorkload(self, requestName, kwargs):
        """ Make all the necessary changes in the Workload to reflect the new assignment """
        request = GetRequest.getRequestByName(requestName)
        helper = Utilities.loadWorkload(request)
        for field in ["AcquisitionEra", "ProcessingVersion"]:
            if type(kwargs[field]) == dict:
                for value in kwargs[field].values():
                    self.validate(value, field)
            else:
                self.validate(kwargs[field], field)
        # Set white list and black list
        whiteList = kwargs.get("SiteWhitelist", [])
        blackList = kwargs.get("SiteBlacklist", [])
        helper.setSiteWildcardsLists(siteWhitelist = whiteList, siteBlacklist = blackList,
                                     wildcardDict = self.wildcardSites)
        # Set ProcessingVersion and AcquisitionEra, which could be json encoded dicts
        helper.setProcessingVersion(kwargs["ProcessingVersion"])
        helper.setAcquisitionEra(kwargs["AcquisitionEra"])
        #FIXME not validated
        helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"])
        helper.setMergeParameters(int(kwargs.get("MinMergeSize", 2147483648)),
                                  int(kwargs.get("MaxMergeSize", 4294967296)),
                                  int(kwargs.get("MaxMergeEvents", 50000)))
        helper.setupPerformanceMonitoring(int(kwargs.get("maxRSS", 2411724)),
                                          int(kwargs.get("maxVSize", 2411724)),
                                          int(kwargs.get("SoftTimeout", 171600)),
                                          int(kwargs.get("GracePeriod", 300)))

        # Check whether we should check location for the data
        if "useSiteListAsLocation" in kwargs:
            helper.setLocationDataSourceFlag()

        # Set phedex subscription information
        custodialList = kwargs.get("CustodialSites", [])
        nonCustodialList = kwargs.get("NonCustodialSites", [])
        if "AutoApprove" in kwargs:
            autoApproveList = nonCustodialList
        else:
            autoApproveList = []
        priority = kwargs.get("Priority", "Low")
        if priority not in ["Low", "Normal", "High"]:
            raise cherrypy.HTTPError(400, "Invalid subscription priority")

        helper.setSubscriptionInformationWildCards(wildcardDict = self.wildcardSites,
                                                   custodialSites = custodialList,
                                                   nonCustodialSites = nonCustodialList,
                                                   autoApproveSites = autoApproveList,
                                                   priority = priority)
        helper.setDashboardActivity(kwargs.get("dashboard", ""))
        Utilities.saveWorkload(helper, request['RequestWorkflow'], self.wmstatWriteURL)
コード例 #21
0
class Approve(BulkOperations):
    """ Page for Physics group leaders to approve requests """
    def __init__(self, config):
        BulkOperations.__init__(self, config)
        self.wmstatWriteURL = "%s/%s" % (config.couchUrl.rstrip('/'),
                                         config.wmstatDBName)

    @cherrypy.expose
    @cherrypy.tools.secmodv2()
    def index(self):
        """ Page for approving requests """
        return self.draw(self.requests())

    def requests(self):
        """ Base list of the requests """
        return Utilities.requestsWhichCouldLeadTo('assignment-approved')

    def draw(self, requests):
        return self.templatepage("BulkOperations",
                                 operation="Approve",
                                 actions=["Approve", "Reject"],
                                 searchFields=["RequestName", "RequestType"],
                                 requests=requests)

    @cherrypy.expose
    #@cherrypy.tools.secmodv2() security issue fix
    @cherrypy.tools.secmodv2(role=Utilities.security_roles(),
                             group=Utilities.security_groups())
    def handleApprove(self, **kwargs):
        """ Handler for approving requests """
        requests = self.requestNamesFromCheckboxes(kwargs)
        particple = ''
        for requestName in requests:
            if kwargs['action'] == 'Reject':
                participle = 'rejected'
                ChangeState.changeRequestStatus(requestName,
                                                'rejected',
                                                wmstatUrl=self.wmstatWriteURL)
            else:
                participle = 'approved'
                ChangeState.changeRequestStatus(requestName,
                                                'assignment-approved',
                                                wmstatUrl=self.wmstatWriteURL)
            priority = kwargs.get(requestName + ':priority', '')
            if priority != '':
                Utilities.changePriority(requestName, priority,
                                         self.wmstatWriteURL)
        return self.templatepage("Acknowledge",
                                 participle=participle,
                                 requests=requests)
コード例 #22
0
    def one(self, requestName):
        """ Assign a single request """
        self.validate(requestName)
        request = GetRequest.getRequestByName(requestName)
        request = Utilities.prepareForTable(request)
        requestType = request["RequestType"]
        # get assignments
        teams = ProdManagement.listTeams()
        assignments = GetRequest.getAssignmentsByName(requestName)
        # might be a list, or a dict team:priority
        if isinstance(assignments, dict):
            assignments = assignments.keys()

        procVer = ""
        acqEra = ""
        procString = ""
        helper = Utilities.loadWorkload(request)
        if helper.getAcquisitionEra() != None:
            acqEra = helper.getAcquisitionEra()
        if helper.getProcessingVersion() != None:
            procVer = helper.getProcessingVersion()
        if helper.getProcessingString():
            procString = helper.getProcessingString()
        dashboardActivity = helper.getDashboardActivity()
        blockCloseMaxWaitTime = helper.getBlockCloseMaxWaitTime()
        blockCloseMaxFiles = helper.getBlockCloseMaxFiles()
        blockCloseMaxEvents = helper.getBlockCloseMaxEvents()
        blockCloseMaxSize = helper.getBlockCloseMaxSize()

        (reqMergedBase, reqUnmergedBase) = helper.getLFNBases()

        return self.templatepage("Assign",
                                 requests=[request],
                                 teams=teams,
                                 assignments=assignments,
                                 sites=self.sites,
                                 phedexNodes=self.phedexNodes,
                                 mergedLFNBases=self.allMergedLFNBases,
                                 reqMergedBase=reqMergedBase,
                                 unmergedLFNBases=self.allUnmergedLFNBases,
                                 reqUnmergedBase=reqUnmergedBase,
                                 acqEra=acqEra,
                                 procVer=procVer,
                                 procString=procString,
                                 dashboardActivity=dashboardActivity,
                                 badRequests=[],
                                 blockCloseMaxWaitTime=blockCloseMaxWaitTime,
                                 blockCloseMaxFiles=blockCloseMaxFiles,
                                 blockCloseMaxSize=blockCloseMaxSize,
                                 blockCloseMaxEvents=blockCloseMaxEvents)
コード例 #23
0
ファイル: ReqMgrRESTModel.py プロジェクト: ticoann/WMCore
 def putRequest(self, requestName=None, status=None, priority=None):
     request = None
     if requestName:
         request = self.findRequest(requestName)
     if request == None:
         # Create a new request, with a JSON-encoded schema that is
         # sent in the body of the HTTP request
         body = cherrypy.request.body.read()
         reqInputArgs = Utilities.unidecode(JsonWrapper.loads(body))
         reqInputArgs.setdefault(
             'CouchURL', Utilities.removePasswordFromUrl(self.couchUrl))
         reqInputArgs.setdefault('CouchDBName', self.configDBName)
         try:
             self.info(
                 "Creating a request for: '%s'\n\tworkloadDB: '%s'\n\twmstatUrl: "
                 "'%s' ..." %
                 (reqInputArgs, self.workloadDBName,
                  Utilities.removePasswordFromUrl(self.wmstatWriteURL)))
             request = Utilities.makeRequest(self, reqInputArgs,
                                             self.couchUrl,
                                             self.workloadDBName,
                                             self.wmstatWriteURL)
         except cherrypy.HTTPError as ex:
             self.error("Create request failed, reason: %s" % ex)
             # Assume that this is a valid HTTPError
             raise
         except (WMException, Exception) as ex:
             # TODO problem not to expose logs to the client
             # e.g. on ConfigCacheID not found, the entire CouchDB traceback is sent in ex_message
             self.error("Create request failed, reason: %s" % ex)
             if hasattr(ex, "name"):
                 detail = ex.name
             else:
                 detail = "check logs."
             msg = "Create request failed, %s" % detail
             raise cherrypy.HTTPError(400, msg)
         self.info("Request '%s' created." % request['RequestName'])
     # see if status & priority need to be upgraded
     if status != None:
         # forbid assignment here
         if status == 'assigned':
             raise cherrypy.HTTPError(
                 403,
                 "Cannot change status without a team.  Please use PUT /reqmgr/reqMgr/assignment/<team>/<requestName>"
             )
         try:
             Utilities.changeStatus(requestName, status,
                                    self.wmstatWriteURL)
         except RuntimeError as ex:
             # ignore some of these errors: https://svnweb.cern.ch/trac/CMSDMWM/ticket/2002
             if status != 'announced' and status != 'closed-out':
                 self.error("RuntimeError while changeStatus: reason: %s" %
                            ex)
                 raise cherrypy.HTTPError(
                     403, "Failed to change status: %s" % str(ex))
     if priority != None:
         Utilities.changePriority(requestName, priority,
                                  self.wmstatWriteURL)
     return request
コード例 #24
0
 def handleApprove(self, **kwargs):
     """ Handler for approving requests """
     requests = self.requestNamesFromCheckboxes(kwargs)
     particple = ""
     for requestName in requests:
         if kwargs["action"] == "Reject":
             participle = "rejected"
             ChangeState.changeRequestStatus(requestName, "rejected")
         else:
             participle = "approved"
             ChangeState.changeRequestStatus(requestName, "assignment-approved")
         priority = kwargs.get(requestName + ":priority", "")
         if priority != "":
             Utilities.changePriority(requestName, priority)
     return self.templatepage("Acknowledge", participle=participle, requests=requests)
コード例 #25
0
    def index(self, all=0):
        """ Main page """
        # returns dict of  name:id
        allRequests = Utilities.requestsWithStatus('assignment-approved')
        teams = ProdManagement.listTeams()

        procVer = ""
        acqEra = ""
        procString = ""
        dashboardActivity = None
        badRequestNames = []
        goodRequests = []
        reqMergedBase = None
        reqUnmergedBase = None
        blockCloseMaxWaitTime = 66400
        blockCloseMaxFiles = 500
        blockCloseMaxEvents = 250000000
        blockCloseMaxSize = 5000000000000
        for request in allRequests:
            # make sure there's a workload attached
            try:
                helper = Utilities.loadWorkload(request)
            except Exception, ex:
                logging.error("Assign error: %s " % str(ex))
                badRequestNames.append(request["RequestName"])
            else:
                # get defaults from the first good one
                if not goodRequests:
                    # forget it if it fails.
                    try:
                        if helper.getAcquisitionEra() != None:
                            acqEra = helper.getAcquisitionEra()
                        if helper.getProcessingVersion() != None:
                            procVer = helper.getProcessingVersion()
                        if helper.getProcessingString() != None:
                            procString = helper.getProcessingString()
                        blockCloseMaxWaitTime = helper.getBlockCloseMaxWaitTime(
                        )
                        blockCloseMaxFiles = helper.getBlockCloseMaxFiles()
                        blockCloseMaxEvents = helper.getBlockCloseMaxEvents()
                        blockCloseMaxSize = helper.getBlockCloseMaxSize()
                        (reqMergedBase, reqUnmergedBase) = helper.getLFNBases()
                        dashboardActivity = helper.getDashboardActivity()
                        goodRequests.append(request)
                    except Exception, ex:
                        logging.error("Assign error: %s " % str(ex))
                        badRequestNames.append(request["RequestName"])
                else:
コード例 #26
0
    def handleAssignmentPage(self, **kwargs):
        """ handler for the main page """
        #Accept Json encoded strings
        decodedArgs = {}
        for key in kwargs.keys():
            try:
                decodedArgs[key] = JsonWrapper.loads(kwargs[key])
            except:
                #Probably wasn't JSON
                decodedArgs[key] = kwargs[key]
        kwargs = decodedArgs
        # handle the checkboxes
        teams = []
        requestNames = []
        for key, value in kwargs.iteritems():
            if isinstance(value, types.StringTypes):
                kwargs[key] = value.strip()
            if key.startswith("Team"):
                teams.append(key[4:])
            if key.startswith("checkbox"):
                requestName = key[8:]
                self.validate(requestName)
                requestNames.append(key[8:])

        for requestName in requestNames:
            if kwargs['action'] == 'Reject':
                ChangeState.changeRequestStatus(requestName,
                                                'rejected',
                                                wmstatUrl=self.wmstatWriteURL)
            else:
                assignments = GetRequest.getAssignmentsByName(requestName)
                if teams == [] and assignments == []:
                    raise cherrypy.HTTPError(
                        400, "Must assign to one or more teams")
                kwargs["Teams"] = teams
                self.assignWorkload(requestName, kwargs)
                for team in teams:
                    if not team in assignments:
                        ChangeState.assignRequest(
                            requestName, team, wmstatUrl=self.wmstatWriteURL)
                priority = kwargs.get(requestName + ':priority', '')
                if priority != '':
                    Utilities.changePriority(requestName, priority,
                                             self.wmstatWriteURL)
        participle = kwargs['action'] + 'ed'
        return self.templatepage("Acknowledge",
                                 participle=participle,
                                 requests=requestNames)
コード例 #27
0
ファイル: ReqMgrBrowser.py プロジェクト: holzman/WMCore
 def _getConfigCache(self, requestName, processMethod):
     try:
         request = Utilities.requestDetails(requestName)
     except Exception, ex:
         msg = "Cannot find request %s, check logs." % requestName
         logging.error("%s, reason: %s" % (msg, ex))
         return msg
コード例 #28
0
ファイル: Approve.py プロジェクト: stuartw/WMCore
 def handleApprove(self, **kwargs):
     """ Handler for approving requests """
     requests = self.requestNamesFromCheckboxes(kwargs)
     particple = ''
     for requestName in requests:
         if kwargs['action'] == 'Reject':
             participle = 'rejected'
             ChangeState.changeRequestStatus(requestName, 'rejected', wmstatUrl = self.wmstatWriteURL)
         else:
             participle = 'approved'
             ChangeState.changeRequestStatus(requestName, 'assignment-approved', wmstatUrl = self.wmstatWriteURL)
         priority = kwargs.get(requestName+':priority', '')
         if priority != '':
             Utilities.changePriority(requestName, priority, self.wmstatWriteURL)
     return self.templatepage("Acknowledge", participle=participle, 
                              requests=requests)
コード例 #29
0
 def announceRequest(self, requestName, cascade=False):
     """
     Announce a request, if the cascade option is given
     then it will search for any Resubmission requests
     for which the given request is a parent and announce them too.
     """
     requestsToAnnounce = [requestName]
     if cascade == "True":
         requestsToAnnounce.extend(
             Utilities.retrieveResubmissionChildren(requestName,
                                                    self.couchUrl,
                                                    self.workloadDBName))
     for requestName in requestsToAnnounce:
         Utilities.changeStatus(requestName, 'announced',
                                self.wmstatWriteURL, self.acdcURL)
     return
コード例 #30
0
ファイル: ReqMgrRESTModel.py プロジェクト: zhiwenuil/WMCore
 def getRequest(self, requestName=None):
     """ If a request name is specified, return the details of the request. 
     Otherwise, return an overview of all requests """
     if requestName == None:
         return GetRequest.getRequests()
     else:
         return Utilities.requestDetails(requestName)
コード例 #31
0
 def putCampaign(self, campaign, request=None):
     """ Adds a campaign if it doesn't already exist, and optionally
         associates a request with it """
     if request:
         requestID = GetRequest.requestID(request)
         if requestID:
             result = Campaign.associateCampaign(campaign, requestID)
             Utilities.associateCampaign(campaign=campaign,
                                         requestName=request,
                                         couchURL=self.couchUrl,
                                         couchDBName=self.workloadDBName)
             return result
         else:
             return False
     else:
         Campaign.addCampaign(campaign)
コード例 #32
0
ファイル: ReqMgrRESTModel.py プロジェクト: zhiwenuil/WMCore
 def putRequest(self, requestName=None, status=None, priority=None):
     """ Checks the request n the body with one arg, and changes the status with kwargs """
     request = None
     if requestName:
         request = self.findRequest(requestName)
     if request == None:
         """ Creates a new request, with a JSON-encoded schema that is sent in the
         body of the request """
         body = cherrypy.request.body.read()
         schema = Utilities.unidecode(JsonWrapper.loads(body))
         schema.setdefault('CouchURL', Utilities.removePasswordFromUrl(self.couchUrl))
         schema.setdefault('CouchDBName', self.configDBName)
         try:
             request = Utilities.makeRequest(schema, self.couchUrl, self.workloadDBName)
         except RuntimeError, ex:
             raise cherrypy.HTTPError(400, ex.message)
コード例 #33
0
ファイル: ReqMgrRESTModel.py プロジェクト: AndrewLevin/WMCore
 def announceRequest(self, requestName, cascade = False):
     """
     Announce a request, if the cascade option is given
     then it will search for any Resubmission requests
     for which the given request is a parent and announce them too.
     """
     requestsToAnnounce = [requestName]
     if cascade == "True":
         requestsToAnnounce.extend(Utilities.retrieveResubmissionChildren(requestName,
                                                                          self.couchUrl,
                                                                          self.workloadDBName))
     for requestName in requestsToAnnounce:
         Utilities.changeStatus(requestName, 'announced',
                                self.wmstatWriteURL,
                                self.acdcURL)
     return
コード例 #34
0
 def index(self):
     """ Main web page for creating requests """
     # Get the scram Architecture from the keys and the
     # CMSSW versions from the values
     self.scramArchs = SoftwareAdmin.listSoftware().keys()
     versionLists = SoftwareAdmin.listSoftware().values()
     self.versions = []
     for l in versionLists:
         for v in l:
             if not v in self.versions:
                 self.versions.append(v)
     self.versions.sort()
     # see if this was configured with a hardcoded user.  If not, take from the request header
     requestor = self.requestor
     if not requestor:
         requestor = cherrypy.request.user["login"]
     if not requestor:
         return "No username found in your certificate"
     if not requestor in Registration.listUsers():
         return "User %s is not registered.  Contact a ReqMgr administrator." % requestor
     groups = GroupInfo.groupsForUser(requestor).keys()
     if groups == []:
         return "User " + requestor + " is not in any groups.  Contact a ReqMgr administrator."
     campaigns = Campaign.listCampaigns()
     return self.templatepage("WebRequestSchema",
                              yuiroot=self.yuiroot,
                              requestor=requestor,
                              groups=groups,
                              versions=self.versions,
                              archs=self.scramArchs,
                              alldocs=Utilities.unidecode(self.allDocs()),
                              allcampaigns=campaigns,
                              defaultVersion=self.cmsswVersion,
                              defaultArch=self.defaultArch,
                              defaultSkimConfig=self.defaultSkimConfig)
コード例 #35
0
ファイル: ReqMgrBrowser.py プロジェクト: tsarangi/WMCore
    def splitting(self, requestName):
        """
        _splitting_

        Retrieve the current values for splitting parameters for all tasks in
        the spec.  Format them in the manner that the splitting page expects
        and pass them to the template.
        """
        self.validate(requestName)
        request = GetRequest.getRequestByName(requestName)
        helper = Utilities.loadWorkload(request)
        splittingDict = helper.listJobSplittingParametersByTask(performance = False)
        taskNames = splittingDict.keys()
        taskNames.sort()

        splitInfo = []
        for taskName in taskNames:
            jsonSplittingParams = JsonWrapper.dumps(splittingDict[taskName])
            splitInfo.append({"splitAlgo": splittingDict[taskName]["algorithm"],
                              "splitParams": jsonSplittingParams,
                              "taskType": splittingDict[taskName]["type"],
                              "taskName": taskName})

        return self.templatepage("Splitting", requestName = requestName,
                                 taskInfo = splitInfo, taskNames = taskNames)
コード例 #36
0
ファイル: ReqMgrRESTModel.py プロジェクト: AndrewLevin/WMCore
 def putCampaign(self, campaign, request=None):
     """ Adds a campaign if it doesn't already exist, and optionally
         associates a request with it """
     if request:
         requestID = GetRequest.requestID(request)
         if requestID:
             result = Campaign.associateCampaign(campaign, requestID)
             Utilities.associateCampaign(campaign = campaign,
                                         requestName = request,
                                         couchURL = self.couchUrl,
                                         couchDBName = self.workloadDBName)
             return result
         else:
             return False
     else:
         Campaign.addCampaign(campaign)
コード例 #37
0
ファイル: ReqMgrBrowser.py プロジェクト: annawoodard/WMCore
 def _getConfigCache(self, requestName, processMethod):
     try:
         request = Utilities.requestDetails(requestName)
     except Exception, ex:
         msg = "Cannot find request %s, check logs." % requestName
         logging.error("%s, reason: %s" % (msg, ex))
         return msg
コード例 #38
0
ファイル: ReqMgrBrowser.py プロジェクト: stuartw/WMCore
    def splitting(self, requestName):
        """
        _splitting_

        Retrieve the current values for splitting parameters for all tasks in
        the spec.  Format them in the manner that the splitting page expects
        and pass them to the template.
        """
        self.validate(requestName)
        request = GetRequest.getRequestByName(requestName)
        helper = Utilities.loadWorkload(request)
        splittingDict = helper.listJobSplittingParametersByTask()
        taskNames = splittingDict.keys()
        taskNames.sort()

        splitInfo = []
        for taskName in taskNames:
            jsonSplittingParams = JsonWrapper.dumps(splittingDict[taskName])
            splitInfo.append({"splitAlgo": splittingDict[taskName]["algorithm"],
                              "splitParams": jsonSplittingParams,
                              "taskType": splittingDict[taskName]["type"],
                              "taskName": taskName})

        return self.templatepage("Splitting", requestName = requestName,
                                 taskInfo = splitInfo, taskNames = taskNames)
コード例 #39
0
 def index(self):
     """ Main web page for creating requests """
     # Get the scram Architecture from the keys and the
     # CMSSW versions from the values
     self.scramArchs = SoftwareAdmin.listSoftware().keys()
     versionLists = SoftwareAdmin.listSoftware().values()
     self.versions = []
     for l in versionLists:
         for v in l:
             if not v in self.versions:
                 self.versions.append(v)
     self.versions.sort()
     # see if this was configured with a hardcoded user.  If not, take from the request header 
     requestor = self.requestor
     if not requestor:
         requestor = cherrypy.request.user["login"]
     if not requestor:
         return "No username found in your certificate"
     if not requestor in Registration.listUsers():
         return "User %s is not registered.  Contact a ReqMgr administrator." % requestor
     groups = GroupInfo.groupsForUser(requestor).keys()
     if groups == []:
         return "User " + requestor + " is not in any groups.  Contact a ReqMgr administrator."
     campaigns = Campaign.listCampaigns()
     return self.templatepage("WebRequestSchema", yuiroot=self.yuiroot,
                              requestor=requestor,
                              groups=groups, 
                              versions=self.versions,
                              archs = self.scramArchs,
                              alldocs = Utilities.unidecode(self.allDocs()),
                              allcampaigns = campaigns,                     
                              defaultVersion=self.cmsswVersion,
                              defaultSkimConfig=self.defaultSkimConfig)
コード例 #40
0
ファイル: Assign.py プロジェクト: cinquo/WMCore
    def index(self, all=0):
        """ Main page """
        # returns dict of  name:id
        allRequests = Utilities.requestsWithStatus("assignment-approved")
        teams = ProdManagement.listTeams()

        procVer = ""
        acqEra = ""
        procString = ""
        dashboardActivity = None
        badRequestNames = []
        goodRequests = []
        reqMergedBase = None
        reqUnmergedBase = None
        blockCloseMaxWaitTime = 66400
        blockCloseMaxFiles = 500
        blockCloseMaxEvents = 250000000
        blockCloseMaxSize = 5000000000000
        for request in allRequests:
            # make sure there's a workload attached
            try:
                helper = Utilities.loadWorkload(request)
            except Exception, ex:
                logging.error("Assign error: %s " % str(ex))
                badRequestNames.append(request["RequestName"])
            else:
                # get defaults from the first good one
                if not goodRequests:
                    # forget it if it fails.
                    try:
                        if helper.getAcquisitionEra() != None:
                            acqEra = helper.getAcquisitionEra()
                        if helper.getProcessingVersion() != None:
                            procVer = helper.getProcessingVersion()
                        if helper.getProcessingString() != None:
                            procString = helper.getProcessingString()
                        blockCloseMaxWaitTime = helper.getBlockCloseMaxWaitTime()
                        blockCloseMaxFiles = helper.getBlockCloseMaxFiles()
                        blockCloseMaxEvents = helper.getBlockCloseMaxEvents()
                        blockCloseMaxSize = helper.getBlockCloseMaxSize()
                        (reqMergedBase, reqUnmergedBase) = helper.getLFNBases()
                        dashboardActivity = helper.getDashboardActivity()
                        goodRequests.append(request)
                    except Exception, ex:
                        logging.error("Assign error: %s " % str(ex))
                        badRequestNames.append(request["RequestName"])
                else:
コード例 #41
0
ファイル: ReqMgrBrowser.py プロジェクト: stuartw/WMCore
 def details(self, requestName):
     """ A page showing the details for the requests """
     self.validate(requestName)
     try:
         request = Utilities.requestDetails(requestName)
     except AssertionError:
         raise cherrypy.HTTPError(404, "Cannot load request %s" % requestName)
     adminHtml = statusMenu(requestName, request['RequestStatus']) \
               + ' Priority ' + Utilities.priorityMenu(request)
     return self.templatepage("Request", requestName=requestName,
                             detailsFields=self.detailsFields, 
                             requestSchema=request,
                             docId=request.get('ProcConfigCacheID', None),
                             assignments=request['Assignments'],
                             adminHtml=adminHtml,
                             messages=request['RequestMessages'],
                             updateDictList=request['RequestUpdates'])
コード例 #42
0
 def closeOutRequest(self, requestName, cascade=False):
     """
     Close out a request, if the cascade option is given
     then it will search for any Resubmission requests
     for which the given request is a parent and close them out
     too.
     """
     requestsToCloseOut = [requestName]
     if cascade == "True":
         requestsToCloseOut.extend(
             Utilities.retrieveResubmissionChildren(requestName,
                                                    self.couchUrl,
                                                    self.workloadDBName))
     for requestName in requestsToCloseOut:
         Utilities.changeStatus(requestName, 'closed-out',
                                self.wmstatWriteURL, self.acdcURL)
     return
コード例 #43
0
 def getMostRecentOutputForPrepID(self, prepID):
     """Return the datasets produced by the most recently submitted request with this prep ID"""
     requestIDs = GetRequest.getRequestByPrepID(prepID)
     # most recent will have the largest ID
     requestID = max(requestIDs)
     request = GetRequest.getRequest(requestID)
     helper = Utilities.loadWorkload(request)
     return helper.listOutputDatasets()
コード例 #44
0
ファイル: ReqMgrRESTModel.py プロジェクト: AndrewLevin/WMCore
 def closeOutRequest(self, requestName, cascade = False):
     """
     Close out a request, if the cascade option is given
     then it will search for any Resubmission requests
     for which the given request is a parent and close them out
     too.
     """
     requestsToCloseOut = [requestName]
     if cascade == "True":
         requestsToCloseOut.extend(Utilities.retrieveResubmissionChildren(requestName,
                                                                          self.couchUrl,
                                                                          self.workloadDBName))
     for requestName in requestsToCloseOut:
         Utilities.changeStatus(requestName, 'closed-out',
                                self.wmstatWriteURL,
                                self.acdcURL)
     return
コード例 #45
0
ファイル: ReqMgrBrowser.py プロジェクト: tsarangi/WMCore
 def details(self, requestName):
     """ A page showing the details for the requests """
     self.validate(requestName)
     try:
         request = Utilities.requestDetails(requestName)
     except AssertionError:
         raise cherrypy.HTTPError(404, "Cannot load request %s" % requestName)
     adminHtml = statusMenu(requestName, request['RequestStatus']) \
               + ' Priority: ' + Utilities.priorityMenu(request)
     return self.templatepage("Request", requestName=requestName,
                             detailsFields=self.detailsFields,
                             requestSchema=request,
                             docId=request.get('ConfigCacheID', None),
                             assignments=request['Assignments'],
                             adminHtml=adminHtml,
                             messages=request['RequestMessages'],
                             updateDictList=request['RequestUpdates'])
コード例 #46
0
ファイル: Assign_t.py プロジェクト: ticoann/WMCore
    def testA_SiteWhitelist(self):
        """
        _SiteWhitelist_

        Test to see if we can get the siteWhitelist to work properly.
        
        """
        secconfig = getattr(self.config, "SecurityModule")
        cherrypy.server.environment = 'test'
        cherrypy.tools.secmodv2 = NullAuth(secconfig)

        self.config.UnitTests.views.active.rest.templates = 'templateDir'
        self.config.UnitTests.views.active.rest.yuiroot = 'yuiroot'
        self.config.UnitTests.views.active.rest.wildcardKeys = {
            'T1*': 'T1_*',
            'T2*': 'T2_*',
            'T3*': 'T3_*',
            'US*': '_US_'
        }
        from WMCore.HTTPFrontEnd.RequestManager.Assign import Assign
        assign = Assign(config=self.config.UnitTests.views.active.rest,
                        noSiteDB=True)

        siteList = [
            'T1_US_FNAL', 'T1_CH_CERN', 'T1_UK_RAL', 'T2_US_UCSD', 'T2_US_UNL',
            'T2_US_CIT'
        ]
        self.assertEqual(assign.sites, [])
        assign.sites.extend(siteList)

        Utilities.addSiteWildcards(assign.wildcardKeys, assign.sites,
                                   assign.wildcardSites)
        for s in siteList:
            self.assertTrue(s in assign.sites)
        self.assertTrue('T1*' in assign.sites)
        self.assertTrue('T2*' in assign.sites)
        self.assertFalse('T3*' in assign.sites)
        self.assertTrue('US*' in assign.sites)

        self.assertEqual(assign.wildcardSites['T1*'],
                         ['T1_US_FNAL', 'T1_CH_CERN', 'T1_UK_RAL'])
        self.assertEqual(assign.wildcardSites['T2*'],
                         ['T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT'])
        self.assertEqual(
            assign.wildcardSites['US*'],
            ['T1_US_FNAL', 'T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT'])
コード例 #47
0
ファイル: Assign.py プロジェクト: stuartw/WMCore
    def __init__(self, config, noSiteDB = False):
        """
        _init_

        Note, noSiteDB added for TESTING PURPOSED ONLY!
        """
        WebAPI.__init__(self, config)
        ReqMgrAuth.assign_roles = config.security_roles
        # Take a guess
        self.templatedir = config.templates
        self.couchUrl = config.couchUrl
        self.clipboardDB = config.clipboardDB
        cleanUrl = Utilities.removePasswordFromUrl(self.couchUrl)
        self.clipboardUrl = "%s/%s/_design/OpsClipboard/index.html" % (cleanUrl, self.clipboardDB)
        self.opshold = config.opshold
        self.configDBName = config.configDBName
        self.wmstatWriteURL = "%s/%s" % (self.couchUrl.rstrip('/'), config.wmstatDBName)
        if not noSiteDB:
            self.sites = Utilities.sites(config.sitedb)
        else:
            self.sites = []
        self.allMergedLFNBases =  [
            "/store/backfill/1", "/store/backfill/2", 
            "/store/data",  "/store/mc", "/store/generator", "/store/relval"]
        self.allUnmergedLFNBases = ["/store/unmerged", "/store/temp"]

        self.mergedLFNBases = {
             "ReReco" : ["/store/backfill/1", "/store/backfill/2", "/store/data"],
             "DataProcessing" : ["/store/backfill/1", "/store/backfill/2", "/store/data"],
             "ReDigi" : ["/store/backfill/1", "/store/backfill/2", "/store/data", "/store/mc"],
             "MonteCarlo" : ["/store/backfill/1", "/store/backfill/2", "/store/mc"],
             "RelValMC" : ["/store/backfill/1", "/store/backfill/2", "/store/mc"],
             "Resubmission" : ["/store/backfill/1", "/store/backfill/2", "/store/mc", "/store/data"],
             "MonteCarloFromGEN" : ["/store/backfill/1", "/store/backfill/2", "/store/mc"],
             "TaskChain": ["/store/backfill/1", "/store/backfill/2", "/store/mc", "/store/data", "/store/relval"],
             "LHEStepZero": ["/store/backfill/1", "/store/backfill/2", "/store/generator"]}

        self.yuiroot = config.yuiroot
        cherrypy.engine.subscribe('start_thread', self.initThread)

        self.wildcardKeys = getattr(config, 'wildcardKeys', {'T1*': 'T1_*',
                                                             'T2*': 'T2_*',
                                                             'T3*': 'T3_*'})
        self.wildcardSites = {}
        Utilities.addSiteWildcards(self.wildcardKeys, self.sites, self.wildcardSites)
コード例 #48
0
    def makeSchema(self, **schema):
        schema.setdefault('CouchURL', Utilities.removePasswordFromUrl(self.couchUrl))
        schema.setdefault('CouchDBName', self.configDBName)

        decodedSchema = {}
        for key in schema.keys():
            try:
                decodedSchema[key] = JsonWrapper.loads(schema[key])
            except:
                # We don't know what kind of exception we'll get, so ignore them all
                # If it does except, it probably wasn't in JSON to begin with.
                # Anything else should be caught by the parsers and the validation
                decodedSchema[key] = schema[key]

        try:
            request = Utilities.makeRequest(decodedSchema, self.couchUrl, self.workloadDBName)
        except RuntimeError, e:
            raise cherrypy.HTTPError(400, "Error creating request: %s" % e)
コード例 #49
0
    def makeSchema(self, **schema):
        schema.setdefault('CouchURL',
                          Utilities.removePasswordFromUrl(self.couchUrl))
        # wrong naming ... but it's all over the place, it's the config cache DB name
        schema.setdefault('CouchDBName', self.configDBName)
        schema.setdefault('CouchWorkloadDBName', self.workloadDBName)

        decodedSchema = {}
        for key in schema.keys():
            try:
                decodedSchema[key] = JsonWrapper.loads(schema[key])
            except:
                # We don't know what kind of exception we'll get, so ignore them all
                # If it does except, it probably wasn't in JSON to begin with.
                # Anything else should be caught by the parsers and the validation
                decodedSchema[key] = schema[key]
        try:
            self.info(
                "Creating a request for: '%s'\n\tworkloadDB: '%s'\n\twmstatUrl: "
                "'%s' ..." %
                (decodedSchema, self.workloadDBName,
                 Utilities.removePasswordFromUrl(self.wmstatWriteURL)))
            request = Utilities.makeRequest(self, decodedSchema, self.couchUrl,
                                            self.workloadDBName,
                                            self.wmstatWriteURL)
            # catching here KeyError is just terrible
        except (RuntimeError, KeyError, Exception) as ex:
            # TODO problem not to expose logs to the client
            # e.g. on ConfigCacheID not found, the entire CouchDB traceback is sent in ex_message
            self.error("Create request failed, reason: %s" % str(ex))
            if hasattr(ex, "message"):
                if hasattr(ex.message, '__call__'):
                    detail = ex.message()
                else:
                    detail = str(ex)
            elif hasattr(ex, "name"):
                detail = ex.name
            else:
                detail = "check logs."
            msg = "Create request failed, %s" % detail
            raise cherrypy.HTTPError(400, msg)
        baseURL = cherrypy.request.base
        raise cherrypy.HTTPRedirect('%s/reqmgr/view/details/%s' %
                                    (baseURL, request['RequestName']))
コード例 #50
0
 def getOutputForPrepID(self, prepID):
     """Return the datasets produced by this prep ID. in a dict of requestName:dataset list"""
     requestIDs = GetRequest.getRequestByPrepID(prepID)
     result = {}
     for requestID in requestIDs:
         request = GetRequest.getRequest(requestID)
         requestName = request["RequestName"]
         helper = Utilities.loadWorkload(request)
         result[requestName] = helper.listOutputDatasets()
     return result
コード例 #51
0
ファイル: ReqMgrRESTModel.py プロジェクト: AndrewLevin/WMCore
 def getOutputForPrepID(self, prepID):
     """Return the datasets produced by this prep ID. in a dict of requestName:dataset list"""
     requestIDs = GetRequest.getRequestByPrepID(prepID)
     result = {}
     for requestID in requestIDs:
         request = GetRequest.getRequest(requestID)
         requestName = request["RequestName"]
         helper = Utilities.loadWorkload(request)
         result[requestName] =  helper.listOutputDatasets()
     return result
コード例 #52
0
ファイル: ReqMgrBrowser.py プロジェクト: tsarangi/WMCore
 def doAdmin(self, **kwargs):
     """  format of kwargs is {'requestname:status' : 'approved', 'requestname:priority' : '2'} """
     message = ""
     for k, v in kwargs.iteritems():
         if k.endswith(':status'):
             requestName = k.split(':')[0]
             self.validate(requestName)
             status = v
             priority = kwargs[requestName+':priority']
             if priority != '':
                 Utilities.changePriority(requestName, priority, self.wmstatWriteURL)
                 message += "Changed priority for %s to %s.\n" % (requestName, priority)
             if status != "":
                 Utilities.changeStatus(requestName, status, self.wmstatWriteURL, self.acdcURL)
                 message += "Changed status for %s to %s\n" % (requestName, status)
                 if status == "assigned":
                     # make a page to choose teams
                     raise cherrypy.HTTPRedirect('/reqmgr/assign/one/%s' % requestName)
     return message + detailsBackLink(requestName)
コード例 #53
0
ファイル: ReqMgrRESTModel.py プロジェクト: prozober/WMCore
    def validateStats(self, index):
        """ Check the values for the updates """

        if 'stats' not in index:
            return index
        index['stats'] = Utilities.unidecode(JsonWrapper.loads(index['stats']))
        for k in ['input_lumis', 'input_num_files',
                  'input_events', 'total_jobs']:
            if k in index['stats']:
                index['stats'][k] = int(index['stats'][k])
        return index
コード例 #54
0
ファイル: ReqMgrBrowser.py プロジェクト: tsarangi/WMCore
 def modifyWorkload(self, requestName, workload,
                    CMSSWVersion=None, GlobalTag=None,
                    runWhitelist=None, runBlacklist=None,
                    blockWhitelist=None, blockBlacklist=None,
                    ScramArch=None):
     """ handles the "Modify" button of the details page """
     self.validate(requestName)
     helper = WMWorkloadHelper()
     helper.load(workload)
     schema = helper.data.request.schema
     message = ""
     if runWhitelist != "" and runWhitelist != None:
         l = Utilities.parseRunList(runWhitelist)
         helper.setRunWhitelist(l)
         schema.RunWhitelist = l
         message += 'Changed runWhiteList to %s<br>' % l
     if runBlacklist != "" and runBlacklist != None:
         l = Utilities.parseRunList(runBlacklist)
         helper.setRunBlacklist(l)
         schema.RunBlacklist = l
         message += 'Changed runBlackList to %s<br>' % l
     if blockWhitelist != "" and blockWhitelist != None:
         l = Utilities.parseBlockList(blockWhitelist)
         helper.setBlockWhitelist(l)
         schema.BlockWhitelist = l
         message += 'Changed blockWhiteList to %s<br>' % l
     if blockBlacklist != "" and blockBlacklist != None:
         l = Utilities.parseBlockList(blockBlacklist)
         helper.setBlockBlacklist(l)
         schema.BlockBlacklist = l
         message += 'Changed blockBlackList to %s<br>' % l
     Utilities.saveWorkload(helper, workload)
     return message + detailsBackLink(requestName)
コード例 #55
0
ファイル: ReqMgrWebTools_t.py プロジェクト: ticoann/WMCore
    def testD_AllSoftwareVersions(self):
        """
        _AllSoftwareVersions_

        This test is a bit weird because it just checks to make sure you're getting versions
        with CMSSW in the name.  I don't want to do specifics because software versions
        change all the time.
        
        """
        result = ReqMgrWebTools.allSoftwareVersions()
        self.assertTrue(len(result) > 0)
        for ver in result:
            self.assertTrue('CMSSW' in ver)
コード例 #56
0
 def handleApprove(self, **kwargs):
     """ Handler for approving requests """
     requests = self.requestNamesFromCheckboxes(kwargs)
     particple = ''
     for requestName in requests:
         if kwargs['action'] == 'Reject':
             participle = 'rejected'
             ChangeState.changeRequestStatus(requestName,
                                             'rejected',
                                             wmstatUrl=self.wmstatWriteURL)
         else:
             participle = 'approved'
             ChangeState.changeRequestStatus(requestName,
                                             'assignment-approved',
                                             wmstatUrl=self.wmstatWriteURL)
         priority = kwargs.get(requestName + ':priority', '')
         if priority != '':
             Utilities.changePriority(requestName, priority,
                                      self.wmstatWriteURL)
     return self.templatepage("Acknowledge",
                              participle=participle,
                              requests=requests)
コード例 #57
0
 def getRequest(self, requestName=None):
     """ If a request name is specified, return the details of the request.
     Otherwise, return an overview of all requests """
     if requestName == None:
         result = GetRequest.getRequests()
     else:
         result = Utilities.requestDetails(requestName)
         try:
             teamNames = GetRequest.getAssignmentsByName(requestName)
             result['teams'] = teamNames
         except:
             # Ignore errors, then we just don't have a team name
             pass
     return result
コード例 #58
0
ファイル: ReqMgrRESTModel.py プロジェクト: prozober/WMCore
    def getConfigIDs(self, prim, proc, tier):
        """
        _getConfigIDs_

        Get the ConfigIDs for the specified request
        """
        result = {}
        dataset = self.getDataset(prim, proc, tier)
        requests = GetRequest.getRequestsByCriteria("Datasets.GetRequestByInput", dataset)
        for request in requests:
            requestName = request["RequestName"]
            helper = Utilities.loadWorkload(request)
            result[requestName] = helper.listAllCMSSWConfigCacheIDs()

        return result