def __call__(self,
wmspec,
task,
data=None,
mask=None,
team=None,
continuous=False,
rucioObj=None):
self.wmspec = wmspec
# bring in spec specific settings
self.args.update(self.wmspec.startPolicyParameters())
self.initialTask = task
if data:
self.data = data
self.mask = mask
self.validate()
try:
pileupDatasets = self.wmspec.listPileupDatasets()
if pileupDatasets:
self.pileupData = self.getDatasetLocations(pileupDatasets)
self.split()
# For known exceptions raise custom error that will fail the workflow.
except dbsClientException as ex:
# A dbs configuration error implies the spec is invalid
error = WorkQueueWMSpecError(self.wmspec,
"DBS config error: %s" % str(ex))
raise error
except AssertionError as ex:
# Assertion generally means validation of an input field failed
error = WorkQueueWMSpecError(self.wmspec,
"Assertion error: %s" % str(ex))
raise error
except DBSReaderError as ex:
# Hacky way of identifying non-existant data, DbsBadRequest chomped by DBSReader
if 'Invalid parameters' in str(ex):
data = task.data.input.pythonise_(
) if task.data.input else 'None'
msg = """data: %s, mask: %s, pileup: %s. %s""" % (
str(data), str(mask), str(pileupDatasets), str(ex))
error = WorkQueueNoWorkError(self.wmspec, msg)
raise error
raise # propagate other dbs errors
# if we have no new elements and we are not adding work to request
# already running, then raise exception
if not self.workQueueElements and not continuous:
data = task.data.input.pythonise_() if task.data.input else 'None'
msg = "Failed to add work. Input data: %s, mask: %s." % (str(data),
str(mask))
error = WorkQueueNoWorkError(self.wmspec, msg)
raise error
return self.workQueueElements, self.rejectedWork, self.badWork
def __call__(self, wmspec, task, data=None, mask=None, team=None):
self.wmspec = wmspec
# bring in spec specific settings
self.args.update(self.wmspec.startPolicyParameters())
self.initialTask = task
if data:
self.data = data
self.mask = mask
self.validate()
try:
pileupDatasets = self.wmspec.listPileupDatasets()
if pileupDatasets:
self.pileupData = self.getDatasetLocations(pileupDatasets)
self.split()
# For known exceptions raise custom error that will fail the workflow.
except DbsConfigurationError as ex:
# A dbs configuration error implies the spec is invalid
error = WorkQueueWMSpecError(self.wmspec,
"DBS config error: %s" % str(ex))
raise error
except AssertionError as ex:
# Assertion generally means validation of an input field failed
error = WorkQueueWMSpecError(self.wmspec,
"Assertion error: %s" % str(ex))
raise error
except DBSReaderError as ex:
# Hacky way of identifying non-existant data, DbsBadRequest chomped by DBSReader
# DbsConnectionError: Database exception,Invalid parameters thrown by Summary api
if 'DbsBadRequest' in str(ex) or 'Invalid parameters' in str(ex):
data = task.data.input.pythonise_(
) if task.data.input else 'None'
msg = """data: %s, mask: %s, pileup: %s. %s""" % (
str(data), str(mask), str(pileupDatasets), str(ex))
error = WorkQueueNoWorkError(self.wmspec, msg)
raise error
raise # propagate other dbs errors
# if we have no elements then there was no work in the spec, fail it
if not self.workQueueElements:
data = task.data.input.pythonise_() if task.data.input else 'None'
msg = """data: %s, mask: %s.""" % (str(data), str(mask))
error = WorkQueueNoWorkError(self.wmspec, msg)
raise error
return self.workQueueElements, self.rejectedWork
def validate(self):
"""Check args and spec work with block splitting"""
StartPolicyInterface.validateCommon(self)
if self.initialTask.totalEvents() < 1:
raise WorkQueueNoWorkError(self.wmspec, 'Invalid total events selection: %s' % str(self.initialTask.totalEvents()))
if self.mask and self.mask['LastEvent'] < self.mask['FirstEvent']:
raise WorkQueueWMSpecError(self.wmspec, "Invalid start & end events")
if self.mask and self.mask['LastLumi'] < self.mask['FirstLumi']:
raise WorkQueueWMSpecError(self.wmspec, "Invalid start & end lumis")
"DBS config error: %s" % str(ex))
raise error
except AssertionError, ex:
# Assertion generally means validation of an input field failed
error = WorkQueueWMSpecError(self.wmspec,
"Assertion error: %s" % str(ex))
raise error
except DBSReaderError, ex:
# Hacky way of identifying non-existant data, DbsBadRequest chomped by DBSReader
# DbsConnectionError: Database exception,Invalid parameters thrown by Summary api
if 'DbsBadRequest' in str(ex) or 'Invalid parameters' in str(ex):
data = task.data.input.pythonise_(
) if task.data.input else 'None'
msg = """data: %s: mask %s. %s""" % (str(data), str(mask),
str(ex))
error = WorkQueueNoWorkError(self.wmspec, msg)
raise error
raise # propagate other dbs errors
# if we have no elements then there was no work in the spec, fail it
if not self.workQueueElements:
data = task.data.input.pythonise_() if task.data.input else 'None'
msg = """data: %s, mask: %s.""" % (str(data), str(mask))
error = WorkQueueNoWorkError(self.wmspec, msg)
raise error
return self.workQueueElements
def dbs(self):
"""Get DBSReader"""
from WMCore.WorkQueue.WorkQueueUtils import get_dbs
dbs_url = self.initialTask.dbsUrl()