def __call__(self, wmspec, task, data=None, mask=None, team=None, continuous=False, rucioObj=None): self.wmspec = wmspec # bring in spec specific settings self.args.update(self.wmspec.startPolicyParameters()) self.initialTask = task if data: self.data = data self.mask = mask self.validate() try: pileupDatasets = self.wmspec.listPileupDatasets() if pileupDatasets: self.pileupData = self.getDatasetLocations(pileupDatasets) self.split() # For known exceptions raise custom error that will fail the workflow. except dbsClientException as ex: # A dbs configuration error implies the spec is invalid error = WorkQueueWMSpecError(self.wmspec, "DBS config error: %s" % str(ex)) raise error except AssertionError as ex: # Assertion generally means validation of an input field failed error = WorkQueueWMSpecError(self.wmspec, "Assertion error: %s" % str(ex)) raise error except DBSReaderError as ex: # Hacky way of identifying non-existant data, DbsBadRequest chomped by DBSReader if 'Invalid parameters' in str(ex): data = task.data.input.pythonise_( ) if task.data.input else 'None' msg = """data: %s, mask: %s, pileup: %s. %s""" % ( str(data), str(mask), str(pileupDatasets), str(ex)) error = WorkQueueNoWorkError(self.wmspec, msg) raise error raise # propagate other dbs errors # if we have no new elements and we are not adding work to request # already running, then raise exception if not self.workQueueElements and not continuous: data = task.data.input.pythonise_() if task.data.input else 'None' msg = "Failed to add work. Input data: %s, mask: %s." % (str(data), str(mask)) error = WorkQueueNoWorkError(self.wmspec, msg) raise error return self.workQueueElements, self.rejectedWork, self.badWork
def __call__(self, wmspec, task, data=None, mask=None, team=None): self.wmspec = wmspec # bring in spec specific settings self.args.update(self.wmspec.startPolicyParameters()) self.initialTask = task if data: self.data = data self.mask = mask self.validate() try: pileupDatasets = self.wmspec.listPileupDatasets() if pileupDatasets: self.pileupData = self.getDatasetLocations(pileupDatasets) self.split() # For known exceptions raise custom error that will fail the workflow. except DbsConfigurationError as ex: # A dbs configuration error implies the spec is invalid error = WorkQueueWMSpecError(self.wmspec, "DBS config error: %s" % str(ex)) raise error except AssertionError as ex: # Assertion generally means validation of an input field failed error = WorkQueueWMSpecError(self.wmspec, "Assertion error: %s" % str(ex)) raise error except DBSReaderError as ex: # Hacky way of identifying non-existant data, DbsBadRequest chomped by DBSReader # DbsConnectionError: Database exception,Invalid parameters thrown by Summary api if 'DbsBadRequest' in str(ex) or 'Invalid parameters' in str(ex): data = task.data.input.pythonise_( ) if task.data.input else 'None' msg = """data: %s, mask: %s, pileup: %s. %s""" % ( str(data), str(mask), str(pileupDatasets), str(ex)) error = WorkQueueNoWorkError(self.wmspec, msg) raise error raise # propagate other dbs errors # if we have no elements then there was no work in the spec, fail it if not self.workQueueElements: data = task.data.input.pythonise_() if task.data.input else 'None' msg = """data: %s, mask: %s.""" % (str(data), str(mask)) error = WorkQueueNoWorkError(self.wmspec, msg) raise error return self.workQueueElements, self.rejectedWork
def validate(self): """Check args and spec work with block splitting""" StartPolicyInterface.validateCommon(self) if self.initialTask.totalEvents() < 1: raise WorkQueueNoWorkError(self.wmspec, 'Invalid total events selection: %s' % str(self.initialTask.totalEvents())) if self.mask and self.mask['LastEvent'] < self.mask['FirstEvent']: raise WorkQueueWMSpecError(self.wmspec, "Invalid start & end events") if self.mask and self.mask['LastLumi'] < self.mask['FirstLumi']: raise WorkQueueWMSpecError(self.wmspec, "Invalid start & end lumis")
"DBS config error: %s" % str(ex)) raise error except AssertionError, ex: # Assertion generally means validation of an input field failed error = WorkQueueWMSpecError(self.wmspec, "Assertion error: %s" % str(ex)) raise error except DBSReaderError, ex: # Hacky way of identifying non-existant data, DbsBadRequest chomped by DBSReader # DbsConnectionError: Database exception,Invalid parameters thrown by Summary api if 'DbsBadRequest' in str(ex) or 'Invalid parameters' in str(ex): data = task.data.input.pythonise_( ) if task.data.input else 'None' msg = """data: %s: mask %s. %s""" % (str(data), str(mask), str(ex)) error = WorkQueueNoWorkError(self.wmspec, msg) raise error raise # propagate other dbs errors # if we have no elements then there was no work in the spec, fail it if not self.workQueueElements: data = task.data.input.pythonise_() if task.data.input else 'None' msg = """data: %s, mask: %s.""" % (str(data), str(mask)) error = WorkQueueNoWorkError(self.wmspec, msg) raise error return self.workQueueElements def dbs(self): """Get DBSReader""" from WMCore.WorkQueue.WorkQueueUtils import get_dbs dbs_url = self.initialTask.dbsUrl()