def getOutputDatasetsWithPSet(payloadNode, sorted = False): """ _getOutputDatasetsWithPSet_ Extract all the information about output datasets from the payloadNode object provided, including the {{}} format PSet cfg Returns a list of DatasetInfo objects including App details from the node. """ result = [] for item in payloadNode._OutputDatasets: resultEntry = DatasetInfo() resultEntry.update(item) resultEntry["ApplicationName"] = payloadNode.application['Executable'] resultEntry["ApplicationProject"] = payloadNode.application['Project'] resultEntry["ApplicationVersion"] = payloadNode.application['Version'] resultEntry["ApplicationFamily"] = item.get("OutputModuleName", "AppFamily") try: config = payloadNode.cfgInterface psetStr = config.originalContent() resultEntry['PSetContent'] = psetStr resultEntry['Conditions'] = config.conditionsTag except Exception, ex: resultEntry['PSetContent'] = None result.append(resultEntry)
def __call__(self, node): """ _operator(node)_ Operate on all output datasets in a Payload Node """ for dataset in node._OutputDatasets: if dataset.has_key("NoMerge"): # // # // If we need to avoid merging some datasets we #// can add a NoMerge key and this will ignore it continue newDataset = DatasetInfo() newDataset.update(dataset) newDataset["ApplicationFamily"] = self.mergeModuleName newDataset["ApplicationName"] = self.appName newDataset["ApplicationVersion"] = node.application['Version'] procName = dataset["ProcessedDataset"] if procName.endswith("-unmerged"): procName = procName.replace("-unmerged", "") else: procName = "%s-merged" % procName newDataset["ProcessedDataset"] = procName newDataset["ParentDataset"] = dataset.name() newDataset['OutputModuleName'] = "%s-Merged" % ( newDataset['OutputModuleName'],) if node.userSandbox != None: newDataset['UserSandbox'] = node.userSandbox self.result.append(newDataset) return
def expandDatasetInfo(datasetInfo, requestTimestamp): """ _expandDatasetInfo_ Given a DatasetInfo, check to see if it contains multi tiers, and if it does, expand them to a list of basic tier dataset info objects returns a list of datasetInfo objects """ result = [] if not isMultiTier(datasetInfo['DataTier']): # // # // No multi tier, return same object as only entry in list #// result.append(datasetInfo) return result tiers = splitMultiTier(datasetInfo['DataTier']) processedDSName = "%s-%s-%s" % (datasetInfo['ApplicationVersion'], datasetInfo['OutputModuleName'], requestTimestamp) for tier in tiers: newInfo = DatasetInfo() newInfo.update(datasetInfo) newInfo['DataTier'] = tier newInfo['ProcessedDataset'] = processedDSName result.append(newInfo) return result
def __call__(self, node): """ _operator(node)_ Operate on all output datasets in a Payload Node """ for dataset in node._OutputDatasets: if dataset.has_key("NoMerge"): # // # // If we need to avoid merging some datasets we #// can add a NoMerge key and this will ignore it continue newDataset = DatasetInfo() newDataset.update(dataset) newDataset["ApplicationFamily"] = self.mergeModuleName newDataset["ApplicationName"] = self.appName newDataset["ApplicationVersion"] = node.application['Version'] procName = dataset["ProcessedDataset"] if procName.endswith("-unmerged"): procName = procName.replace("-unmerged", "") else: procName = "%s-merged" % procName newDataset["ProcessedDataset"] = procName newDataset["ParentDataset"] = dataset.name() newDataset['OutputModuleName'] = "%s-Merged" % ( newDataset['OutputModuleName'], ) if node.userSandbox != None: newDataset['UserSandbox'] = node.userSandbox self.result.append(newDataset) return
def getPileupDatasets(payloadNode): """ _getPileupDatasets_ Extract all pileup dataset info from the node provided. Returns a list of dataset info objects """ result = [] for item in payloadNode._PileupDatasets: resultEntry = DatasetInfo() resultEntry.update(item) resultEntry['NodeName'] = payloadNode.name result.append(resultEntry) return result
def getInputDatasets(payloadNode, sorted = False): """ _getInputDatasets_ Extract all the information about input datasets from the payloadNode object provided. Returns a list of DatasetInfo objects including App details from the node. """ result = [] for item in payloadNode._InputDatasets: resultEntry = DatasetInfo() resultEntry.update(item) result.append(resultEntry) if sorted: result = _sortDatasets(result) return result
def getOutputDatasets(payloadNode, sorted = False): """ _getOutputDatasets_ Extract all the information about output datasets from the payloadNode object provided. Returns a list of DatasetInfo objects including App details from the node. """ result = [] for item in payloadNode._OutputDatasets: resultEntry = DatasetInfo() resultEntry.update(item) resultEntry["ApplicationName"] = payloadNode.application['Executable'] resultEntry["ApplicationProject"] = payloadNode.application['Project'] resultEntry["ApplicationVersion"] = payloadNode.application['Version'] result.append(resultEntry) if sorted: result = _sortDatasets(result) return result
def unpackPayloadNodeData(self, improvNode): """ _unpackPayloadNodeData_ Unpack PayloadNode data from improv Node provided and add information to self """ self.name = str(improvNode.attrs["Name"]) self.type = str(improvNode.attrs["Type"]) workflowName = improvNode.attrs.get("Workflow", None) if workflowName != None: self.workflow = str(workflowName) # // # // Unpack data for this instance # // App details appDataQ = IMProvQuery("/%s/Application" % self.__class__.__name__) appData = appDataQ(improvNode)[0] for appField in appData.children: field = str(appField.name) value = str(appField.attrs["Value"]) self.application[field] = value # // # // App Control details # // appConDataQ = IMProvQuery("/%s/ApplicationControl/*" % self.__class__.__name__) appConData = appConDataQ(improvNode) for appConField in appConData: field = str(appConField.name) value = str(appConField.attrs["Value"]) self.applicationControls[field] = value # // # // Script Controls # // scriptConQ = IMProvQuery("/%s/ScriptControls/ScriptList" % self.__class__.__name__) scriptLists = scriptConQ(improvNode) for scriptList in scriptLists: listName = scriptList.attrs.get("Name", None) if listName == None: continue listName = str(listName) for script in scriptList.children: scriptName = script.attrs.get("Value", None) if scriptName == None: continue self.scriptControls[listName].append(str(scriptName)) # // # // Dataset details # // Input Datasets inputDSQ = IMProvQuery("/%s/InputDatasets/DatasetInfo" % self.__class__.__name__) inputDS = inputDSQ(improvNode) # print improvNode for item in inputDS: newDS = DatasetInfo() newDS.load(item) self._InputDatasets.append(newDS) # // # // Output Datasets # // outputDSQ = IMProvQuery("/%s/OutputDatasets/DatasetInfo" % self.__class__.__name__) outputDS = outputDSQ(improvNode) for item in outputDS: newDS = DatasetInfo() newDS.load(item) self._OutputDatasets.append(newDS) # // # // Pileup Datasets # // pileupDSQ = IMProvQuery("/%s/PileupDatasets/DatasetInfo" % self.__class__.__name__) pileupDS = pileupDSQ(improvNode) for item in pileupDS: newDS = DatasetInfo() newDS.load(item) self._PileupDatasets.append(newDS) # // # // Input Links # // inpLinkQ = IMProvQuery("/%s/InputLinks/InputLink" % self.__class__.__name__) inpLinks = inpLinkQ(improvNode) for ilink in inpLinks: newLink = InputLink() newLink.load(ilink) self._InputLinks.append(newLink) # // # // Configuration # // configQ = IMProvQuery("/%s/Configuration" % self.__class__.__name__) configNodes = configQ(improvNode) if len(configNodes) > 0: configNode = configNodes[0] self.configuration = base64.decodestring(str(configNode.chardata)) cfgIntQ = IMProvQuery("/%s/CMSSWConfig" % self.__class__.__name__) cfgNodes = cfgIntQ(improvNode) if len(cfgNodes) > 0: cfgNode = cfgNodes[0] self.cfgInterface = CMSSWConfig() self.cfgInterface.load(cfgNode) # // # // User sandbox # // sandboxQ = IMProvQuery("/%s/UserSandbox" % self.__class__.__name__) sandboxNodes = sandboxQ(improvNode) if len(sandboxNodes) > 0: sandboxNode = sandboxNodes[-1] self.userSandbox = str(sandboxNode.chardata) return