Пример #1
0
    def load(self, improvNode):
        """
        _load_

        Extract information for this object from the improv instance provided
        """
        wfQuery = IMProvQuery("/RequestSpec/WorkflowSpec")
        wfnode = wfQuery(improvNode)[0]
        wfspec = WorkflowSpec()
        wfspec.loadFromNode(wfnode)
        self.workflow = wfspec

        policyQuery = IMProvQuery("/RequestSpec/Policies/*")
        detailQuery = IMProvQuery("/RequestSpec/RequestDetails/*")
        preferredPAQuery = IMProvQuery("/RequestSpec/PreferredPA")

        policies = policyQuery(improvNode)
        details = detailQuery(improvNode)
        preferredPAs = preferredPAQuery(improvNode)

        for policy in policies:
            self.policies[str(policy.name)] = str(policy.chardata)

        for detail in improvNode.attrs.keys():
            self.requestDetails[detail] = str(improvNode.attrs[detail])

        for preferredPA in preferredPAs:
            self.preferredPAs[str(preferredPA.attrs['id'])] = \
                str(preferredPA.attrs['priority'])

        return
Пример #2
0
    def load(self, node):
        """load into self from improv node"""

        runQuery = IMProvQuery("/Run[attribute(\"ID\")]")
        runInfo = runQuery(node)
        if len(runInfo) == 0:
            return

        self.run = int(runInfo[-1])

        lumiQuery = IMProvQuery("/Run/LumiSection[attribute(\"ID\")]")
        lumiInfo = lumiQuery(node)
        self.extend([int(x) for x in lumiInfo])
        return
Пример #3
0
    def legacyLumiInfo(self, improvNode, queryBase):
        """
        _legacyLumiInfo_

        handle legacy lumi section information

        """
        #  //
        # // Lumi Sections
        #//
        lumiQ = IMProvQuery("/%s/LumiSections/LumiSection" % queryBase)
        for lumiSect in lumiQ(improvNode):

            newLumi = {}
            [
                newLumi.__setitem__(x.name, x.attrs['Value'])
                for x in lumiSect.children
            ]
            run = newLumi.get("RunNumber", None)
            lumi = newLumi.get("LumiSectionNumber", None)
            if run == None or lumi == None:
                continue

            run = int(run)
            lumi = int(lumi)
            if not self.runs.has_key(run):
                newRun = RunInfo()
                newRun.run = run
                self.runs[run] = newRun
            runInfo = self.runs.get(run)
            runInfo.append(lumi)
        return
Пример #4
0
def readSpecFile(filename):
    """
    _readSpecFile_

    Tool for extracting multiple RequestSpecs from a file

    Returns a list of spec instances found
    """
    result = []
    # ignore failed parsed requests
    ignore = 0

    node = loadIMProvFile(filename)
    node = loadIMProvFile(filename)
    specQ = IMProvQuery("RequestSpec")
    specNodes = specQ(node)

    for snode in specNodes:
        newSpec = RequestSpec()
        try:
            newSpec.load(snode)
            result.append(newSpec)
        except Exception, ex:
            logging.debug(
                'ERROR loading a requestspec, will ignore this requestspec :' +
                str(ex))
            ignore += 1
Пример #5
0
    def load(self, improvNode):
        """improv to self"""
        self.release = str(improvNode.attrs['Version'])
        query = IMProvQuery("Validate/Test")
        nodes = query(improvNode)
        for node in nodes:
            newTest = RelValTest()
            newTest.load(node)
            self.append(newTest)

        return
Пример #6
0
    def load(self, improvNode):
        """
        _load_

        Populate self based on content of improvNode instance
        """
        componentQ = IMProvQuery(
            "ProdCommonConfiguration/Component[attribute(\"Name\")]"
            )
        configQ = IMProvQuery(
            "ProdCommonConfiguration/ConfigBlock"
            )
        components = componentQ(improvNode)
        configs = configQ(improvNode)
        for comp in components:
            self.components.append(comp)
        for config in configs:
            cfgBlock = ConfigBlock("temp")
            cfgBlock.load(config)
            self[cfgBlock.name] = cfgBlock
        
        return
Пример #7
0
def queryStatusXML(filename):

    try:
        report = loadIMProvFile(filename)
    except Exception as ex:
        msg = "Error reading CRAB Status Report: %s\n" % filename
        msg += str(ex)
        raise RuntimeError(msg)

    query = IMProvQuery("Task/TaskJobs/Job/RunningJob")
    Jobs = query(report)

    return Jobs
Пример #8
0
    def load(self, improvNode):
        """
        _load_

        Extract Performance reports from the improvNode provided
        Uses a Relative path query

        """
        cpuQ = IMProvQuery("PerformanceReport/CPU/CPUCore")
        memQ = IMProvQuery("PerformanceReport/Memory/Property")
        summaryQ = IMProvQuery("PerformanceReport/PerformanceSummary")
        moduleQ = IMProvQuery("PerformanceReport/PerformanceModule")

        for cpu in cpuQ(improvNode):
            core = cpu.attrs.get("Core", None)
            if core == None:
                continue
            core = str(core)
            speed = None
            desc = None
            for child in cpu.children:
                if child.attrs.get('Name', None) == "Description":
                    desc = str(child.chardata)
                if child.attrs.get('Name', None) == 'Speed':
                    speed = str(child.chardata)
            self.addCPU(core, speed, desc)

        for mem in memQ(improvNode):
            name = mem.attrs.get("Name", None)
            if name == None: continue
            self.memory[str(name)] = str(mem.chardata)

        for summNode in summaryQ(improvNode):
            metricName = summNode.attrs.get("Metric", None)
            if metricName == None: continue
            metrics = {}
            for metric in IMProvQuery("Metric")(summNode):
                metName = metric.attrs.get("Name", None)
                metVal = metric.attrs.get("Value", None)
                if metName == None: continue
                metrics[str(metName)] = str(metVal)

            self.addSummary(str(metricName), **metrics)

        for modNode in moduleQ(improvNode):
            metricName = modNode.attrs.get("Metric", None)
            modName = modNode.attrs.get("Module", None)
            if metricName == None: continue
            if modName == None: continue
            metrics = {}
            for metric in IMProvQuery("Metric")(modNode):
                metName = metric.attrs.get("Name", None)
                metVal = metric.attrs.get("Value", None)
                if metName == None: continue
                metrics[str(metName)] = str(metVal)
            self.addModule(str(metricName), modName, **metrics)

        return
Пример #9
0
    def load(self, improvNode):
        dataQ = IMProvQuery("/InputLink/*")
        dataNodes = dataQ(improvNode)
        for node in dataNodes:
            key = str(node.name)
            val = str(node.attrs['Value'])
            self[key] = val

        if self.get("AppearStandalone", "false").lower() in ("true", "yes"):
            self["AppearStandalone"] = True
        else:
            self["AppearStandalone"] = False

        return
Пример #10
0
    def load(self, improvNode):
        """
        _load_

        Populate self based on content of improvNode instance
        """
        nameQ = IMProvQuery("/ConfigBlock[attribute(\"Name\")]")
        paramQ = IMProvQuery("/ConfigBlock/Parameter")
        commentQ = IMProvQuery("/ConfigBlock/Comment[text()]")
        
        self.name = str(nameQ(improvNode)[0])
        comments = commentQ(improvNode)
        if len(comments) > 0:
            self.comment = str(comments[0])
        else:
            self.comment = None
        
        params = paramQ(improvNode)
        for paramNode in params:
            paramName = str(paramNode.attrs['Name'])
            paramVal = str(paramNode.attrs['Value'])
            self[paramName] = paramVal
            
        return
Пример #11
0
    def loadFromNode(self, improvNode):
        """
        _loadFromNode_

        Populate this object based on content of improvNode provided
        
        """
        paramQ = IMProvQuery("/WorkflowSpec/Parameter")
        payloadQ = IMProvQuery("/WorkflowSpec/Payload/PayloadNode")
        libsQ = IMProvQuery(
            "/WorkflowSpec/PythonLibraries/PythonLibrary[attribute(\"Name\")]")
        #  //
        # // Extract Params
        #//
        paramNodes = paramQ(improvNode)
        for item in paramNodes:
            paramName = item.attrs.get("Name", None)
            if paramName == None:
                continue
            paramValue = str(item.chardata)
            self.parameters[str(paramName)] = paramValue

        #  //
        # // Extract python lib list
        #//
        libNames = libsQ(improvNode)
        for item in libNames:
            self.addPythonLibrary(str(item))

        #  //
        # // Extract Payload Nodes
        #//
        payload = payloadQ(improvNode)[0]
        self.payload = PayloadNode()
        self.payload.populate(payload)
        return
Пример #12
0
    def load(self, improvNode):
        """
        _load_

        Load from IMProvNode structure & populate self

        """
        nodesQ = IMProvQuery("/BulkSpecs/BulkSpec")
        nodes = nodesQ(improvNode)
        for node in nodes:
            id = node.attrs.get("ID", None)
            if id == None:
                continue
            self[str(id)] = str(node.chardata)
        return
Пример #13
0
    def populate(self, improvNode):
        """
        _populate_

        Extract details of this node from improvNode and
        instantiate and populate any children found

        """

        self.unpackPayloadNodeData(improvNode)
        #  //
        # // Recursively handle children
        #//
        childQ = IMProvQuery("/PayloadNode/PayloadNode")
        childNodes = childQ(improvNode)
        for item in childNodes:
            newChild = PayloadNode()
            self.addNode(newChild)
            newChild.populate(item)

        return
Пример #14
0
    def load(self, improvNode):
        """
        _load_

        Populate this instance with data extracted from the improvNode
        provided. The Argument should be an improvNode created with the
        same structure as the result of the save method of this class

        """
        if improvNode.name != self.__class__.__name__:
            #  //
            # // Not the right node type
            #//
            return

        entryQ = IMProvQuery("/%s/Entry" % self.__class__.__name__)
        entries = entryQ(improvNode)
        for entry in entries:
            key = str(entry.attrs["Name"])
            value = str(entry.chardata)
            self[key] = value
        return
Пример #15
0
def readRelValSpec(url, topNode = "ReleaseValidation"):
    """
    _readRelValSpec_

    Open the given URL and read the contents, parse it and return a list of
    RelValidation objects

    """
    
    handler = IMProvHandler()
    parser = make_parser()
    parser.setContentHandler(handler)
    parser.parse(url)
    mainNode = handler._ParentDoc

    result = []
    query = IMProvQuery("/%s/Validate" % topNode)
    nodes = query(mainNode)
    for node in nodes:
        relval = RelValidation("")
        relval.load(node)
        result.append(relval)
    return result
Пример #16
0
    def loadFromNode(self, improvNode):

        paramQ = IMProvQuery("/JobSpec/Parameter")
        payloadQ = IMProvQuery("/JobSpec/Payload/JobSpecNode")
        whitelistQ = IMProvQuery("/JobSpec/SiteWhitelist/Site")
        blacklistQ = IMProvQuery("/JobSpec/SiteBlacklist/Site")
        assocFileQ = IMProvQuery("/JobSpec/AssociatedFiles/AssocFileList")
        bulkSpecQ = IMProvQuery("/JobSpec/BulkSpecs")

        #  //
        # // Extract Params
        #//
        paramNodes = paramQ(improvNode)
        for item in paramNodes:
            paramName = item.attrs.get("Name", None)
            if paramName == None:
                continue
            paramValue = str(item.chardata)
            self.parameters[str(paramName)] = paramValue

        #  //
        # // Extract site lists
        #//
        whiteNodes = whitelistQ(improvNode)
        for wnode in whiteNodes:
            value = wnode.attrs.get("Name", None)
            if value != None:
                self.siteWhitelist.append(str(value))
        blackNodes = blacklistQ(improvNode)
        for bnode in blackNodes:
            value = bnode.attrs.get("Name", None)
            if value != None:
                self.siteBlacklist.append(str(value))

        #  //
        # // Extract Associated Files
        #//
        assocFiles = assocFileQ(improvNode)
        if len(assocFiles) > 0:
            for assocFileList in assocFiles:
                assocListName = str(assocFileList.attrs['Name'])
                assocList = []
                for aFile in assocFileList.children:
                    fileEntry = {
                        "LFN" : str(aFile.chardata),
                        }
                    for attrName, attrVal in aFile.attrs.items():
                        fileEntry[str(attrName)] = str(attrVal)
                    assocList.append(fileEntry)
                self.addAssociatedFiles(assocListName, *assocList)

        bulkNodes = bulkSpecQ(improvNode)
        if len(bulkNodes) > 0:
            self.bulkSpecs.load(bulkNodes[-1])
        
        #  //
        # // Extract Payload Nodes
        #//
        payload = payloadQ(improvNode)[0]
        self.payload = JobSpecNode()
        self.payload.populate(payload)
        return
Пример #17
0
    def populate(self, improvNode):
        """
        _populate_

        Override PayloadNode.populate to load a saved JobSpecNode

        """
        #  //
        # // Unpack base class data
        #//
        self.unpackPayloadNodeData(improvNode)
        jobName = improvNode.attrs.get("JobName", None)
        if jobName != None:
            self.jobName = str(jobName)
        jobType = improvNode.attrs.get("JobType", None)
        if jobType != None:
            self.jobType = str(jobType)
            
        
        #  //
        # // Unpack JobSpecNode additional Data
        #//  Parameters
        paramQ = IMProvQuery("/JobSpecNode/JobSpecification/Parameters/*")
        params = paramQ(improvNode)
        for paramNode in params:
            key = str(paramNode.name)
            value = str(paramNode.chardata)
            self.addParameter(key, value)

        #  //
        # // InputModule 
        #//
        inpQ = IMProvQuery("/JobSpecNode/JobSpecification/InputModule")
        inputModNode = inpQ(improvNode)
        if len(inputModNode) > 0:
            inputModNode = inputModNode[0]
            self.inputModule = str(inputModNode.chardata)
            

        #  //
        # // OutputModules
        #//
        outQ = IMProvQuery(
            "/JobSpecNode/JobSpecification/OutputModules/OutputModule")
        outMods = outQ(improvNode)
        for outMod in outMods:
            self.outputModules.append(str(outMod.chardata))
            
        
        #  //
        # // Recursively handle children
        #//
        childQ = IMProvQuery("/JobSpecNode/JobSpecNode")
        childNodes = childQ(improvNode)
        for item in childNodes:
            newChild = JobSpecNode()
            self.addNode(newChild)
            newChild.populate(item)
            
            
        return
    'Cancelled by user': '******',
    'Unknown': 'U',
    'Done(failed)': 'D'
}


def queryStatusXML(filename):

    try:
        report = loadIMProvFile(filename)
    except StandardError, ex:
        msg = "Error reading CRAB Status Report: %s\n" % filename
        msg += str(ex)
        raise RuntimeError, msg

    query = IMProvQuery("Task/TaskJobs/Job/RunningJob")
    Jobs = query(report)

    return Jobs


def printCrabStatusFromReport(filename):
    Jobs = queryStatusXML(filename)
    print "Crab Id: StatusScheduler | Status | ProcessStatus | State | GridId |"
    for j in Jobs:
        crabId = int(j.attrs.get("jobId", None))
        statusScheduler = str(j.attrs.get("statusScheduler", None))
        status = str(j.attrs.get("status", None))
        processStatus = str(j.attrs.get("processStatus", None))
        state = str(j.attrs.get("state", None))
        gridId = str(j.attrs.get("schedulerId", None))
Пример #19
0
    instance containing the details found in it

    """
    if not os.path.exists(filename):
        msg = "TrivialFileCatalog not found: %s" % filename
        raise RuntimeError, msg


    try:
        node = loadIMProvFile(filename)
    except StandardError, ex:
        msg = "Error reading TrivialFileCatalog: %s\n" % filename
        msg += str(ex)
        raise RuntimeError, msg

    query = IMProvQuery("storage-mapping/lfn-to-pfn")
    mappings = query(node)

    tfcInstance = TrivialFileCatalog()

    for mapping in mappings:
        protocol = mapping.attrs.get("protocol", None)
        match = mapping.attrs.get("path-match", None)
        result = mapping.attrs.get("result", None)
        chain = mapping.attrs.get("chain", None)
        if True in (protocol, match, mapping == None):
            continue
        tfcInstance.addMapping(str(protocol), str(match), str(result), chain)
        
    return tfcInstance
Пример #20
0
    def unpackPayloadNodeData(self, improvNode):
        """
        _unpackPayloadNodeData_

        Unpack PayloadNode data from improv Node provided and
        add information to self

        """
        self.name = str(improvNode.attrs["Name"])
        self.type = str(improvNode.attrs["Type"])
        workflowName = improvNode.attrs.get('Workflow', None)
        if workflowName != None:
            self.workflow = str(workflowName)
        #  //
        # // Unpack data for this instance
        #//  App details
        appDataQ = IMProvQuery("/%s/Application" % self.__class__.__name__)
        appData = appDataQ(improvNode)[0]
        for appField in appData.children:
            field = str(appField.name)
            value = str(appField.attrs['Value'])
            self.application[field] = value
        #  //
        # // App Control details
        #//
        appConDataQ = IMProvQuery("/%s/ApplicationControl/*" %
                                  self.__class__.__name__)
        appConData = appConDataQ(improvNode)
        for appConField in appConData:
            field = str(appConField.name)
            value = str(appConField.attrs['Value'])
            self.applicationControls[field] = value

        #  //
        # // Script Controls
        #//
        scriptConQ = IMProvQuery("/%s/ScriptControls/ScriptList" %
                                 self.__class__.__name__)
        scriptLists = scriptConQ(improvNode)
        for scriptList in scriptLists:
            listName = scriptList.attrs.get("Name", None)
            if listName == None: continue
            listName = str(listName)
            for script in scriptList.children:
                scriptName = script.attrs.get("Value", None)
                if scriptName == None: continue
                self.scriptControls[listName].append(str(scriptName))

        #  //
        # // Dataset details
        #//  Input Datasets
        inputDSQ = IMProvQuery("/%s/InputDatasets/DatasetInfo" %
                               self.__class__.__name__)
        inputDS = inputDSQ(improvNode)
        #        print improvNode
        for item in inputDS:
            newDS = DatasetInfo()
            newDS.load(item)
            self._InputDatasets.append(newDS)

        #  //
        # // Output Datasets
        #//
        outputDSQ = IMProvQuery("/%s/OutputDatasets/DatasetInfo" %
                                self.__class__.__name__)
        outputDS = outputDSQ(improvNode)
        for item in outputDS:
            newDS = DatasetInfo()
            newDS.load(item)
            self._OutputDatasets.append(newDS)
        #  //
        # // Pileup Datasets
        #//
        pileupDSQ = IMProvQuery("/%s/PileupDatasets/DatasetInfo" %
                                self.__class__.__name__)
        pileupDS = pileupDSQ(improvNode)
        for item in pileupDS:
            newDS = DatasetInfo()
            newDS.load(item)
            self._PileupDatasets.append(newDS)
        #  //
        # // Input Links
        #//
        inpLinkQ = IMProvQuery("/%s/InputLinks/InputLink" %
                               self.__class__.__name__)
        inpLinks = inpLinkQ(improvNode)
        for ilink in inpLinks:
            newLink = InputLink()
            newLink.load(ilink)
            self._InputLinks.append(newLink)

        #  //
        # // Configuration
        #//
        configQ = IMProvQuery("/%s/Configuration" % self.__class__.__name__)
        configNodes = configQ(improvNode)
        if len(configNodes) > 0:
            configNode = configNodes[0]
            self.configuration = base64.decodestring(str(configNode.chardata))

        cfgIntQ = IMProvQuery("/%s/CMSSWConfig" % self.__class__.__name__)
        cfgNodes = cfgIntQ(improvNode)
        if len(cfgNodes) > 0:
            cfgNode = cfgNodes[0]
            self.cfgInterface = CMSSWConfig()
            self.cfgInterface.load(cfgNode)

        #  //
        # // User sandbox
        #//
        sandboxQ = IMProvQuery("/%s/UserSandbox" % self.__class__.__name__)
        sandboxNodes = sandboxQ(improvNode)
        if len(sandboxNodes) > 0:
            sandboxNode = sandboxNodes[-1]
            self.userSandbox = str(sandboxNode.chardata)

        return
Пример #21
0
class TrivialFileCatalog:
    """
    _FwkJobReport_

    Framework Job Report container and interface object

    """
    def __init__(self, url=None):
        self.preferredProtocol = None  # attribute for preferred protocol
        self.lfnToPfn = []
        self.pfnToLfn = []
        if url:
            self.load(url)

    def addLfnToPfnRule(self, protocol, pathMatch, result, chain=None):
        """
        Add a LFN-To-PFN mapping rule that will match for the given
        protocol and destination
        """
        self.lfnToPfn.append({
            "protocol": protocol,
            "path-match": pathMatch,
            "path-match-regexp": re.compile(pathMatch),
            "chain": chain,
            "result": result
        })
        return

    def matchLFN(self, protocol, lfn):
        """
        _matchLFN_

        Return the result for the LFN provided if the LFN
        matches the path-match for that protocol

        Return None if no match
        """
        if not protocol:
            protocol = self.preferredProtocol

        for mapping in self.lfnToPfn:
            if mapping['protocol'] != protocol:
                continue
            if mapping['path-match-regexp'].match(lfn):
                if mapping['chain'] != None:
                    lfn = self.matchLFN(mapping['chain'], lfn)
                try:
                    splitLFN = mapping['path-match-regexp'].split(lfn, 1)[1]
                except IndexError:
                    continue
                result = mapping['result'].replace("$1", splitLFN)
                return result

        return None

    def addPfnToLfnRule(self, protocol, pathMatch, result, chain=None):
        """
        Add a PFN-To-LFN mapping rule that will match for the given
        protocol and destination
        """
        self.pfnToLfn.append({
            "protocol": protocol,
            "path-match": pathMatch,
            "path-match-regexp": re.compile(pathMatch),
            "chain": chain,
            "result": result
        })
        return

    def matchPFN(self, protocol, pfn):
        """
        _matchPFN_

        Return the result for the PFN provided if the LFN
        matches the path-match for that protocol

        Return None if no match
        """
        if not protocol:
            protocol = self.preferredProtocol

        for mapping in self.pfnToLfn:
            if mapping['protocol'] != protocol:
                continue
            if mapping['path-match-regexp'].match(pfn):
                if mapping['chain'] != None:
                    pfn = self.matchPFN(mapping['chain'], pfn)
                try:
                    splitPFN = mapping['path-match-regexp'].split(pfn, 1)[1]
                except IndexError:
                    continue
                result = mapping['result'].replace("$1", splitPFN)
                return result

        return None

    def save(self):
        """
        _save_

        Save the tfc by converting it into
        an XML IMProv Object

        """
        result = IMProvNode("storage-mapping")

        for maping in self.lfnToPfn:
            node = IMProvNode("lfn-to-pfn", None)
            node.attrs.update(maping)
            del node.attrs['path-match-regexp']
            if not maping.get('chain', None):
                del node.attrs['chain']
            result.addNode(node)

        for maping in self.pfnToLfn:
            node = IMProvNode("pfn-to-lfn", None)
            node.attrs.update(maping)
            del node.attrs['path-match-regexp']
            if not maping.get('chain', None):
                del node.attrs['chain']
            result.addNode(node)

        return result

    def write(self, filename):
        """
        _write_

        Write the tfc to an XML file

        """
        handle = open(filename, 'w')
        handle.write(self.save().makeDOMElement().toprettyxml())
        handle.close()
        return

    def __str__(self):
        """string representation of instance"""
        return str(self.save())

    def load(self, url):
        """
        _load_

        Read a tfc into this instance

        """
        self.lfnToPfn = []
        self.pfnToLfn = []
        self.preferredProtocol = tfcProtocol(url)
        filename = tfcFilename(url)

        if not os.path.exists(filename):
            msg = "TrivialFileCatalog not found: %s" % filename
            raise RuntimeError, msg

        try:
            node = loadIMProvFile(filename)
        except StandardError, ex:
            msg = "Error reading TrivialFileCatalog: %s\n" % filename
            msg += str(ex)
            raise RuntimeError, msg

        query = IMProvQuery("storage-mapping/lfn-to-pfn")
        mappings = query(node)

        for mapping in mappings:
            protocol = mapping.attrs.get("protocol", None)
            match = mapping.attrs.get("path-match", None)
            result = mapping.attrs.get("result", None)
            chain = mapping.attrs.get("chain", None)
            if True in (protocol, match, mapping == None):
                continue
            self.addLfnToPfnRule(str(protocol), str(match), \
                                                        str(result), chain)

        query = IMProvQuery("storage-mapping/pfn-to-lfn")
        mappings = query(node)

        for mapping in mappings:
            protocol = mapping.attrs.get("protocol", None)
            match = mapping.attrs.get("path-match", None)
            result = mapping.attrs.get("result", None)
            chain = mapping.attrs.get("chain", None)
            if True in (protocol, match, mapping == None):
                continue
            self.addPfnToLfnRule(str(protocol), str(match), \
                                                        str(result), chain)

        return
Пример #22
0
    def load(self, improvNode):
        """
        _load_

        Populate this object from the improvNode provided

        """
        #  //
        # // Input or Output?
        #//
        queryBase = improvNode.name
        if queryBase == "InputFile":
            self.isInput = True
        else:
            self.isInput = False
        #  //
        # // Parameters
        #//
        paramQ = IMProvQuery("/%s/*" % queryBase)
        for paramNode in paramQ(improvNode):
            if paramNode.name not in self.keys():
                continue
            self[paramNode.name] = paramNode.chardata

        #  //
        # // State
        #//
        stateQ = IMProvQuery("/%s/State[attribute(\"Value\")]" % queryBase)
        self.state = stateQ(improvNode)[-1]

        #  //
        # // Checksums
        #//
        cksumQ = IMProvQuery("/%s/Checksum" % queryBase)
        for cksum in cksumQ(improvNode):
            algo = cksum.attrs.get('Algorithm', None)
            if algo == None: continue
            self.addChecksum(str(algo), str(cksum.chardata))

        #  //
        # // Inputs
        #//
        inputFileQ = IMProvQuery("/%s/Inputs/Input" % queryBase)
        for inputFile in inputFileQ(improvNode):
            lfn = IMProvQuery("/Input/LFN[text()]")(inputFile)[-1]
            pfn = IMProvQuery("/Input/PFN[text()]")(inputFile)[-1]
            self.addInputFile(pfn, lfn)

        #  //
        # // Datasets
        #//
        datasetQ = IMProvQuery("/%s/Datasets/DatasetInfo" % queryBase)
        for dataset in datasetQ(improvNode):
            newDataset = self.newDataset()
            newDataset.load(dataset)

        #  //
        # // Branches
        #//
        branchQ = IMProvQuery("/%s/Branches/Branch[text()]" % queryBase)
        for branch in branchQ(improvNode):
            self.branches.append(str(branch))

        runQ = IMProvQuery("/%s/Runs/Run" % queryBase)
        for run in runQ(improvNode):
            newRun = RunInfo()
            newRun.load(run)
            if newRun.run == None:
                continue

            self.runs[newRun.run] = newRun
            self["Runs"] = self.runs

        self.legacyLumiInfo(improvNode, queryBase)

        return
Пример #23
0
def handleStorageStats(improvNode, reportRef):
    """
    _handleStorageStats_

    Look for the detailed storage stats information and
    collapse it to a set of performance report instances

    """
    #  //
    # // Backwards compatibility
    #//
    if len(improvNode.chardata.strip()) > 0:
        reportRef.storageStatistics = str(improvNode.chardata.strip())
        return

    #  //
    # // new format report
    #//
    base = "/StorageStatistics/storage-factory-summary/"
    statsBase = "%sstorage-factory-stats" % base
    statsQ = IMProvQuery("%s/storage-timing-summary/counter-value" % statsBase)

    rootQ = IMProvQuery("%s/storage-root-summary/counter-value" % statsBase)

    paramBase = "%sstorage-factory-params/param" % base
    paramQ = IMProvQuery(paramBase)

    allStats = {}
    for stats in statsQ(improvNode):
        subsystem = stats.attrs.get("subsystem", None)
        if subsystem == None:
            continue
        subsystem = str(subsystem)
        counterName = stats.attrs.get("counter-name", None)
        if counterName == None:
            continue
        counterName = str(counterName)

        keyName = "%s-%s" % (subsystem, counterName)
        for statKey, statVal in stats.attrs.items():
            statKey = str(statKey)
            statVal = str(statVal)
            if statKey in ("subsystem", "counter-name"):
                continue
            dataKey = "%s-%s" % (keyName, statKey)
            allStats[dataKey] = statVal

    #print allStats
    allRoots = {}
    for root in rootQ(improvNode):
        subsystem = root.attrs.get("subsystem", None)
        if subsystem == None:
            continue
        subsystem = str(subsystem)
        counterName = root.attrs.get("counter-name", None)
        if counterName == None:
            continue
        counterName = str(counterName)

        keyName = "%s-%s" % (subsystem, counterName)
        for rootKey, rootVal in root.attrs.items():
            rootKey = str(rootKey)
            rootVal = str(rootVal)
            if rootKey in ("subsystem", "counter-name"):
                continue
            dataKey = "%s-%s" % (keyName, rootKey)
            allRoots[dataKey] = rootVal

    allParams = {}
    for param in paramQ(improvNode):
        paramName = param.attrs.get("name", None)
        if paramName == None:
            continue
        paramName = str(paramName)
        paramVal = param.attrs.get("value", None)
        if paramVal == None:
            continue
        allParams[paramName] = str(paramVal)

    reportRef.performance.addSummary("StorageTiming", **allStats)
    reportRef.performance.addSummary("StorageRoot", **allRoots)
    reportRef.performance.addSummary("StorageParams", **allParams)
    return
Пример #24
0
    def load(self, improvNode):
        """
        _load_

        Unpack improvNode into this instance

        """
        self.name = improvNode.attrs.get("Name", None)
        self.status = improvNode.attrs.get("Status", None)
        self.jobSpecId = improvNode.attrs.get("JobSpecID", None)
        self.jobType = improvNode.attrs.get("JobType", None)
        self.workflowSpecId =improvNode.attrs.get("WorkflowSpecID", None)
        self.dashboardId =improvNode.attrs.get("DashboardId", None)

        exitQ = IMProvQuery(
            "/FrameworkJobReport/ExitCode[attribute(\"Value\")]")
        exitVals = exitQ(improvNode)
        if len(exitVals) > 0:
            self.exitCode = int(exitVals[-1])

        #  //
        # // Site details
        #//
        siteQ = IMProvQuery("/FrameworkJobReport/SiteDetail")
        [ self.siteDetails.__setitem__(x.attrs['Parameter'], x.attrs['Value'])
          for x in siteQ(improvNode) ]

        #  //
        # // PSetHash
        #//
        hashQ = IMProvQuery("/FrameworkJobReport/PSetHash[text()]")
        hashVals = hashQ(improvNode)
        if len(hashVals) > 0:
            hashData = hashVals[-1]
            self.psetHash = str(hashData)


        #  //
        # // output files
        #//
        fileQ = IMProvQuery("/FrameworkJobReport/File")
        fileList = fileQ(improvNode)
        for fileEntry in fileList:
            newFile = self.newFile()
            newFile.load(fileEntry)

        #  //
        # // input files
        #//
        infileQ = IMProvQuery("/FrameworkJobReport/InputFile")
        for infileEntry in infileQ(improvNode):
            newInFile = self.newInputFile()
            newInFile.load(infileEntry)

        #  //
        # // Skipped Events & Files
        #//
        skipFileQ = IMProvQuery("/FrameworkJobReport/SkippedFile")
        skipEventQ = IMProvQuery("/FrameworkJobReport/SkippedEvent")


        [ self.addSkippedEvent(int(skipEv.attrs['Run']),
                               int(skipEv.attrs['Event']))
          for skipEv in skipEventQ(improvNode)]

        [ self.addSkippedFile(skipF.attrs['Pfn'], skipF.attrs['Lfn'])
          for skipF in skipFileQ(improvNode) ]

        #  //
        # // removed files
        #//
        remFileQ = IMProvQuery("/FrameworkJobReport/RemovedFile")
        [ self.addRemovedFile(str(remF.chardata), remF.attrs['SEName'])
          for remF in remFileQ(improvNode) ]

        #  //
        # // unremoved files
        #//
        unremFileQ = IMProvQuery("/FrameworkJobReport/UnremovedFile")
        [ self.addUnremovedFile(str(remF.chardata), remF.attrs['SEName'])
          for remF in unremFileQ(improvNode) ]

        #  //
        # // analysis files
        #//
        afileQ = IMProvQuery("/FrameworkJobReport/AnalysisFile")
        for afileEntry in afileQ(improvNode):
            newAFile = self.newAnalysisFile()
            newAFile.load(afileEntry)
        #  //
        # // log files
        #//
        logFileQ = IMProvQuery("/FrameworkJobReport/LogFile")
        [ self.addLogFile(str(logF.chardata), logF.attrs['SEName'])
          for logF in logFileQ(improvNode) ]

        #  //
        # // Timing, Storage and generator info
        #//
        timingQ = IMProvQuery("/FrameworkJobReport/TimingService/*")

        [ self.timing.__setitem__(x.name, x.attrs['Value'])
          for x in timingQ(improvNode) ]

        #  //
        # // ReadBranches tag from cmsRun
        #//
        readBranchesQ = IMProvQuery("/FrameworkJobReport/ReadBranches/*")
        [ self.readBranches.__setitem__(x.attrs['Name'], x.attrs['ReadCount'])
          for x in readBranchesQ(improvNode) ]

        storageQ = IMProvQuery("/FrameworkJobReport/StorageStatistics")
        storageInfo = storageQ(improvNode)
        if len(storageInfo) > 0:
            #self.storageStatistics = storageInfo[-1]
            StorageStats.handleStorageStats(storageInfo[-1], self)

        genQ = IMProvQuery("/FrameworkJobReport/GeneratorInfo/Data")
        [ self.generatorInfo.__setitem__(x.attrs['Name'], x.attrs['Value'])
          for x in genQ(improvNode)]

        errQ = IMProvQuery("/FrameworkJobReport/FrameworkError")
        errors  = errQ(improvNode)
        for err in errors:
            newErr = self.addError(
                int(err.attrs['ExitStatus']),
                err.attrs['Type'])
            newErr['Description'] = err.chardata

        #  //
        # // Performance reports
        #//
        self.performance.load(improvNode)

        return
Пример #25
0
    Instantiate a new ContentHandler and run it over the file producing
    a list of FwkJobReport instances

    """
    #handler = FwkJobRepHandler()
    #parser = make_parser()
    #parser.setContentHandler(handler)
    #try:
    #    parser.parse(filename)
    #except SAXParseException, ex:
    #    msg = "Error parsing JobReport File: %s\n" % filename
    #    msg += str(ex)
    #    print msg
    #    return []
    #print handler.results[0]
    #return handler.results

    try:
        improvDoc = loadIMProvFile(filename)
    except Exception, ex:
        return []
    result = []
    reportQ = IMProvQuery("FrameworkJobReport")
    reportInstances = reportQ(improvDoc)
    for reportInstance in reportInstances:
        newReport = FwkJobReport()
        newReport.load(reportInstance)
        result.append(newReport)
    return result
Пример #26
0
        _read_

        Load data from SiteLocal Config file and populate this object

        """
        try:
            node = loadIMProvFile(self.siteConfigFile)
        except StandardError, ex:
            msg = "Unable to read SiteConfigFile: %s\n" % self.siteConfigFile
            msg += str(ex)
            raise SiteConfigError, msg

        #  //
        # // site name
        #//
        nameQ = IMProvQuery("/site-local-config/site")
        nameNodes = nameQ(node)
        if len(nameNodes) == 0:
            msg = "Unable to find site name in SiteConfigFile:\n"
            msg += self.siteConfigFile
            raise SiteConfigError, msg
        self.siteName = str(nameNodes[0].attrs.get("name"))

        #  //
        # // event data (Read Trivial Catalog location)
        #//

        catalogQ = IMProvQuery("/site-local-config/site/event-data/catalog")
        catNodes = catalogQ(node)
        if len(catNodes) == 0:
            msg = "Unable to find catalog entry for event data in:\n"