def processDataset(self): """ _processDataset_ Import the Dataset contents and create a set of jobs from it """ # // # // Now create the job definitions #// logging.debug("SplitSize = %s" % self.splitSize) logging.debug("SplitType = %s" % self.splitType) logging.debug("AllowedSites = %s" % self.allowedSites) logging.debug("AllowedBlocks = %s" % self.allowedBlocks) jobDefs = splitDatasetForReReco(self.inputDataset(), self.dbsUrl, self.splitSize, self.onlyClosedBlocks, self.allowedSites, self.allowedBlocks) logging.debug("Retrieved %s job definitions split for rereco" % ( len(jobDefs)),) return jobDefs
def processDataset(self): """ _processDataset_ Import the Dataset contents and create a set of jobs from it """ # // # // Now create the job definitions #// logging.debug("SplitSize = %s" % self.splitSize) logging.debug("SplitType = %s" % self.splitType) logging.debug("AllowedSites = %s" % self.allowedSites) logging.debug("AllowedBlocks = %s" % self.allowedBlocks) jobDefs = splitDatasetForReReco(self.inputDataset(), self.dbsUrl, self.splitSize, self.onlyClosedBlocks, self.allowedSites, self.allowedBlocks) logging.debug("Retrieved %s job definitions split for rereco" % ( len(jobDefs)),) onlyFiles = self.workflowSpec.parameters.get('OnlyFiles') if onlyFiles is not None: onlyFilesList = [x for x in onlyFiles.split(',') if x] if onlyFilesList: filteredJobDefs = [] for jobDef in jobDefs: skipJobDef = False for lfn in jobDef['LFNS']: if lfn not in onlyFilesList: skipJobDef = True break if not skipJobDef: filteredJobDefs.append(jobDef) jobDefs = filteredJobDefs return jobDefs
def processDataset(self): """ _processDataset_ Import the Dataset contents and create a set of jobs from it """ # // # // Now create the job definitions #// logging.debug("SplitSize = %s" % self.splitSize) logging.debug("SplitType = %s" % self.splitType) logging.debug("AllowedSites = %s" % self.allowedSites) logging.debug("AllowedBlocks = %s" % self.allowedBlocks) jobDefs = splitDatasetForReReco(self.inputDataset(), self.dbsUrl, self.splitSize, self.onlyClosedBlocks, self.allowedSites, self.allowedBlocks) logging.debug( "Retrieved %s job definitions split for rereco" % (len(jobDefs)), ) onlyFiles = self.workflowSpec.parameters.get('OnlyFiles') if onlyFiles is not None: onlyFilesList = [x for x in onlyFiles.split(',') if x] if onlyFilesList: filteredJobDefs = [] for jobDef in jobDefs: skipJobDef = False for lfn in jobDef['LFNS']: if lfn not in onlyFilesList: skipJobDef = True break if not skipJobDef: filteredJobDefs.append(jobDef) jobDefs = filteredJobDefs return jobDefs
def processDataset(self): """ _processDataset_ Import the Dataset contents and create a set of jobs from it """ # // # // Now create the job definitions #// logging.debug("SplitSize = %s" % self.splitSize) logging.debug("SplitType = %s" % self.splitType) logging.debug("AllowedSites = %s" % self.allowedSites) logging.debug("AllowedBlocks = %s" % self.allowedBlocks) jobDefs = splitDatasetForReReco(self.inputDataset(), self.dbsUrl, self.splitSize, self.onlyClosedBlocks, self.allowedSites, self.allowedBlocks) logging.debug( "Retrieved %s job definitions split for rereco" % (len(jobDefs)), ) return jobDefs