def __init__(self, couchDB, id = None, elementParams = None):
elementParams = elementParams or {}
WorkQueueElement.__init__(self, **elementParams)
if id:
self._id = id
self._document = Document(id = id)
self._couch = couchDB
def testSetId(self):
"""Can override id generation"""
ele = WorkQueueElement(RequestName='testIdImmutable')
before_id = ele.id
ele.id = 'something_new'
self.assertEqual('something_new', ele.id)
self.assertNotEqual(before_id, ele.id)
def __init__(self, couchDB, id=None, elementParams=None):
elementParams = elementParams or {}
WorkQueueElement.__init__(self, **elementParams)
if id:
self._id = id
self._document = Document(id=id)
self._couch = couchDB
def testSetId(self):
"""Can override id generation"""
ele = WorkQueueElement(RequestName='testIdImmutable')
before_id = ele.id
ele.id = 'something_new'
self.assertEqual('something_new', ele.id)
self.assertNotEqual(before_id, ele.id)
def testPriority(self):
"""Element priority and ordering handled correctly"""
element = WorkQueueElement(RequestName = 'backend_test',
WMSpec = self.processingSpec,
Status = 'Available',
Jobs = 10, Priority = 1)
highprielement = WorkQueueElement(RequestName = 'backend_test_high',
WMSpec = self.processingSpec,
Status = 'Available', Jobs = 10,
Priority = 100)
element2 = WorkQueueElement(RequestName = 'backend_test_2',
WMSpec = self.processingSpec,
Status = 'Available',
Jobs = 10, Priority = 1)
lowprielement = WorkQueueElement(RequestName = 'backend_test_low',
WMSpec = self.processingSpec,
Status = 'Available',
Jobs = 10, Priority = 0.1)
self.backend.insertElements([element])
self.backend.availableWork({'place' : 1000})
# timestamp in elements have second coarseness, 2nd element must
# have a higher timestamp to force it after the 1st
time.sleep(1)
self.backend.insertElements([lowprielement, element2, highprielement])
self.backend.availableWork({'place' : 1000})
work = self.backend.availableWork({'place' : 1000})
# order should be high to low, with the standard elements in the order
# they were queueud
self.assertEqual([x['RequestName'] for x in work[0]],
['backend_test_high', 'backend_test', 'backend_test_2', 'backend_test_low'])
def testNegotiating(self):
"""Workflow only partially split"""
# one element split and available in workqueue, other not split yet
parent2 = WQE(Status = 'Negotiating')
parent2.id = 2
parents = [self.parent, parent2]
# throw error as can only compute status when all elements split
self.assertRaises(RuntimeError, self.policy(), [self.available], parents)
def testNegotiating(self):
"""Workflow only partially split"""
# one element split and available in workqueue, other not split yet
parent2 = WQE(Status = 'Negotiating')
parent2.id = 2
parents = [self.parent, parent2]
results = self.policy()([self.available], parents)
self.assertEqual(len(results), 2)
self.assertEqual(results[0]['Status'], 'Acquired')
self.assertEqual(results[1]['Status'], 'Negotiating')
def testIdIgnoresIrrelevant(self):
"""Id calculation ignores irrelavant variables"""
ele = WorkQueueElement(RequestName = 'testIdIgnoresIrrelevant')
this_id = ele.id
for params2 in itertools.product(Progress, Priority, Teams,
ParentQueueUrl, ParentQueueId):
ele = WorkQueueElement(RequestName = 'testIdIgnoresIrrelevant',
PercentSuccess = params2[0],
Priority = params2[1], TeamName = params2[2],
ParentQueueUrl = params2[3], ParentQueueId = params2[4],
)
# id not changed by changing irrelvant parameters
self.assertEqual(ele.id, this_id)
def newQueueElement(self, **args):
# DBS Url may not be available in the initial task
# but in the pileup data (MC pileup)
dbsUrl = self.initialTask.dbsUrl()
if dbsUrl is None and self.pileupData:
# Get the first DBS found
dbsUrl = next(iter(self.wmspec.listPileupDatasets()))
args.setdefault('Status', 'Available')
args.setdefault('WMSpec', self.wmspec)
args.setdefault('Task', self.initialTask)
args.setdefault('RequestName', self.wmspec.name())
args.setdefault('TaskName', self.initialTask.name())
args.setdefault('Dbs', dbsUrl)
args.setdefault('SiteWhitelist', self.initialTask.siteWhitelist())
args.setdefault('SiteBlacklist', self.initialTask.siteBlacklist())
args.setdefault('StartPolicy', self.wmspec.startPolicy())
args.setdefault('EndPolicy', self.wmspec.endPolicyParameters())
args.setdefault('Priority', self.wmspec.priority())
args.setdefault('PileupData', self.pileupData)
if not args['Priority']:
args['Priority'] = 0
ele = WorkQueueElement(**args)
for data, sites in viewitems(ele['Inputs']):
if not sites:
raise WorkQueueWMSpecError(
self.wmspec, 'Input data has no locations "%s"' % data)
# catch infinite splitting loops
if len(self.workQueueElements) > self.args.get('maxRequestSize', 1e8):
raise WorkQueueWMSpecError(
self.wmspec, 'Too many elements (%d)' %
self.args.get('MaxRequestElements', 1e8))
self.workQueueElements.append(ele)
def testPossibleSites(self):
"""
Workqueue element data location check (same as workRestrictions)
"""
# test element ala MonteCarlo
ele = WorkQueueElement(SiteWhitelist=["T1_IT_CNAF", "T2_DE_DESY"])
self.assertEqual(possibleSites(ele), ["T1_IT_CNAF", "T2_DE_DESY"])
# test element with InputDataset but no location
ele['Inputs'] = {"/MY/BLOCK/NAME#73e99a52": []}
self.assertEqual(possibleSites(ele), [])
# test element with InputDataset and no match location
ele['Inputs'] = {
"/MY/BLOCK/NAME#73e99a52": ["T1_US_FNAL", "T2_CH_CERN"]
}
self.assertEqual(possibleSites(ele), [])
# test element with InputDataset and valid location
ele['Inputs'] = {
"/MY/BLOCK/NAME#73e99a52":
["T1_US_FNAL", "T2_CH_CERN", "T2_DE_DESY"]
}
self.assertEqual(possibleSites(ele), ["T2_DE_DESY"])
# test element with InputDataset and ParentData with no location
ele['Inputs'] = {
"/MY/BLOCK/NAME#73e99a52":
["T1_US_FNAL", "T2_CH_CERN", "T2_DE_DESY"]
}
ele['ParentFlag'] = True
ele['ParentData'] = {"/MY/BLOCK2/NAME#002590494c06": []}
self.assertEqual(possibleSites(ele), [])
# test element with InputDataset and ParentData with no match location
ele['ParentData'] = {"/MY/BLOCK2/NAME#002590494c06": ["T1_IT_CNAF"]}
self.assertEqual(possibleSites(ele), [])
# test element with InputDataset and ParentData with valid location
ele['ParentData'] = {
"/MY/BLOCK2/NAME#002590494c06": ["T1_US_FNAL", "T2_DE_DESY"]
}
self.assertEqual(possibleSites(ele), ["T2_DE_DESY"])
# test element with InputDataset, PileupData and ParentData with no location
ele['Inputs'] = {
"/MY/BLOCK/NAME#73e99a52":
["T1_US_FNAL", "T2_CH_CERN", "T2_DE_DESY"]
}
ele['ParentData'] = {"/MY/BLOCK2/NAME#002590494c06": ["T2_DE_DESY"]}
ele['PileupData'] = {"/MY/DATASET/NAME": []}
self.assertEqual(possibleSites(ele), [])
# test element with InputDataset, PileupData and ParentData with no match location
ele['PileupData'] = {"/MY/DATASET/NAME": ["T1_IT_CNAF", "T2_CH_CERN"]}
self.assertEqual(possibleSites(ele), [])
# test element with InputDataset, PileupData and ParentData with valid location
ele['PileupData'] = {"/MY/DATASET/NAME": ["T1_IT_CNAF", "T2_DE_DESY"]}
self.assertEqual(possibleSites(ele), ["T2_DE_DESY"])
def setUp(self):
filePath = os.path.join(
getTestBase(),
"WMCore_t/WorkQueue_t/DataStructs_t/wq_available_elements.json")
with open(filePath, "r") as f:
gqData = json.load(f)
self.gqElements = []
for ele in gqData:
self.gqElements.append(WorkQueueElement(**ele))
def testIdUnique(self):
"""Modifying a relevant parameter varies the id"""
ids = {}
# Vary parameters that affect the work or input data,
# verify each id is unique
for params in itertools.product(RequestNames, TaskNames, Inputs,
Masks, Dbses, Acdcs):
ele = WorkQueueElement(RequestName = params[0], TaskName = params[1],
Inputs = params[2], Mask = params[3],
Dbs = params[4], ACDC = params[5]
)
self.assertFalse(ele.id in ids)
ids[ele.id] = None
def testPossibleSitesLocationFlags(self):
"""
Workqueue element data location check, using the input and PU data location flags
"""
ele = WorkQueueElement(SiteWhitelist=["T1_IT_CNAF", "T2_DE_DESY"])
# test element with InputDataset and no location, but input flag on
ele['Inputs'] = {"/MY/BLOCK/NAME#73e99a52": []}
ele['NoInputUpdate'] = True
self.assertEqual(possibleSites(ele), ["T1_IT_CNAF", "T2_DE_DESY"])
# test element with InputDataset and one match, but input flag on
ele['Inputs'] = {
"/MY/BLOCK/NAME#73e99a52": ["T1_IT_CNAF", "T2_CH_CERN"]
}
self.assertEqual(possibleSites(ele), ["T1_IT_CNAF", "T2_DE_DESY"])
# test element with InputDataset and one match, but pu flag on
ele['NoInputUpdate'] = False
ele['NoPileupUpdate'] = True
self.assertEqual(possibleSites(ele), ["T1_IT_CNAF"])
# test element with InputDataset and one match, but both flags on
ele['NoInputUpdate'] = True
self.assertEqual(possibleSites(ele), ["T1_IT_CNAF", "T2_DE_DESY"])
# test element with InputDataset and ParentData and no location, but both flags on
ele['ParentFlag'] = True
ele['ParentData'] = {"/MY/BLOCK2/NAME#002590494c06": []}
self.assertEqual(possibleSites(ele), ["T1_IT_CNAF", "T2_DE_DESY"])
# test element with InputDataset and ParentData and no location, but input flag on
ele['NoPileupUpdate'] = False
self.assertEqual(possibleSites(ele), ["T1_IT_CNAF", "T2_DE_DESY"])
# test element with InputDataset and ParentData and no location, but pileup flag on
ele['NoInputUpdate'] = False
ele['NoPileupUpdate'] = True
self.assertEqual(possibleSites(ele), [])
# test element with InputDataset, PileupData and ParentData with no location, but pileup flag on
ele['Inputs'] = {
"/MY/BLOCK/NAME#73e99a52":
["T1_US_FNAL", "T2_CH_CERN", "T2_DE_DESY"]
}
ele['ParentData'] = {"/MY/BLOCK2/NAME#002590494c06": ["T2_DE_DESY"]}
ele['PileupData'] = {"/MY/DATASET/NAME": []}
self.assertEqual(possibleSites(ele), ["T2_DE_DESY"])
# test element with InputDataset, PileupData and ParentData with no location, but both flags on
ele['NoInputUpdate'] = True
self.assertEqual(possibleSites(ele), ["T1_IT_CNAF", "T2_DE_DESY"])
# test element with InputDataset, PileupData and ParentData with no location, but input flag on
ele['NoPileupUpdate'] = False
self.assertEqual(possibleSites(ele), [])
def newQueueElement(self, **args):
args.setdefault('Status', 'Available')
args.setdefault('WMSpec', self.wmspec)
args.setdefault('Task', self.initialTask)
args.setdefault('RequestName', self.wmspec.name())
args.setdefault('TaskName', self.initialTask.name())
args.setdefault('Dbs', self.initialTask.dbsUrl())
args.setdefault('SiteWhitelist', self.initialTask.siteWhitelist())
args.setdefault('SiteBlacklist', self.initialTask.siteBlacklist())
args.setdefault('EndPolicy', self.wmspec.endPolicyParameters())
args.setdefault('Priority', self.wmspec.priority())
if not args['Priority']:
args['Priority'] = 0
ele = WorkQueueElement(**args)
for data, sites in ele['Inputs'].items():
if not sites:
raise WorkQueueWMSpecError(
self.wmspec, 'Input data has no locations "%s"' % data)
# catch infinite splitting loops
if len(self.workQueueElements) > self.args.get('maxRequestSize', 1e8):
raise WorkQueueWMSpecError(
self.wmspec, 'Too many elements (%d)' %
self.args.get('MaxRequestElements', 1e8))
self.workQueueElements.append(ele)
def testIdImmutable(self):
"""Id fixed once calculated"""
ele = WorkQueueElement(RequestName='testIdImmutable')
before_id = ele.id
ele['RequestName'] = 'somethingElse'
self.assertEqual(before_id, ele.id)
def testPassesSiteRestriction(self):
"""
Workqueue element site restriction check (same as workRestrictions)
"""
# test element ala MonteCarlo
ele = WorkQueueElement(SiteWhitelist=["T1_IT_CNAF", "T2_DE_DESY"], SiteBlacklist=["T1_US_FNAL"])
self.assertFalse(ele.passesSiteRestriction("T1_US_FNAL"))
self.assertFalse(ele.passesSiteRestriction("T2_CH_CERN"))
self.assertTrue(ele.passesSiteRestriction("T1_IT_CNAF"))
# test element with input dataset
ele['Inputs'] = {"/MY/BLOCK/NAME#73e99a52": []}
self.assertFalse(ele.passesSiteRestriction("T1_US_FNAL"))
self.assertFalse(ele.passesSiteRestriction("T2_CH_CERN"))
self.assertFalse(ele.passesSiteRestriction("T1_IT_CNAF"))
self.assertFalse(ele.passesSiteRestriction("T2_DE_DESY"))
ele['Inputs'] = {"/MY/BLOCK/NAME#73e99a52": ["T1_US_FNAL", "T2_DE_DESY"]}
self.assertFalse(ele.passesSiteRestriction("T1_US_FNAL"))
self.assertFalse(ele.passesSiteRestriction("T1_IT_CNAF"))
self.assertTrue(ele.passesSiteRestriction("T2_DE_DESY"))
# test element with input and parent dataset
ele['Inputs'] = {"/MY/BLOCK/NAME#73e99a52": []}
ele['ParentFlag'] = True
ele['ParentData'] = {"/MY/BLOCK2/NAME#002590494c06": []}
self.assertFalse(ele.passesSiteRestriction("T1_US_FNAL"))
self.assertFalse(ele.passesSiteRestriction("T2_CH_CERN"))
self.assertFalse(ele.passesSiteRestriction("T1_IT_CNAF"))
self.assertFalse(ele.passesSiteRestriction("T2_DE_DESY"))
ele['Inputs'] = {"/MY/BLOCK/NAME#73e99a52": ["T1_US_FNAL", "T2_DE_DESY"]}
self.assertFalse(ele.passesSiteRestriction("T1_US_FNAL"))
self.assertFalse(ele.passesSiteRestriction("T2_DE_DESY"))
ele['Inputs'] = {"/MY/BLOCK/NAME#73e99a52": ["T1_US_FNAL", "T2_DE_DESY"]}
ele['ParentData'] = {"/MY/BLOCK2/NAME#002590494c06": ["T1_IT_CNAF", "T2_CH_CERN", "T2_DE_DESY"]}
self.assertFalse(ele.passesSiteRestriction("T1_US_FNAL"))
self.assertFalse(ele.passesSiteRestriction("T1_IT_CNAF"))
self.assertTrue(ele.passesSiteRestriction("T2_DE_DESY"))
# test element with input, parent and pileup dataset
ele['PileupData'] = {"/MY/DATASET/NAME": []}
self.assertFalse(ele.passesSiteRestriction("T1_US_FNAL"))
self.assertFalse(ele.passesSiteRestriction("T2_CH_CERN"))
self.assertFalse(ele.passesSiteRestriction("T1_IT_CNAF"))
self.assertFalse(ele.passesSiteRestriction("T2_DE_DESY"))
ele['PileupData'] = {"/MY/DATASET/NAME": ["T2_US_Nebraska", "T1_IT_CNAF"]}
self.assertFalse(ele.passesSiteRestriction("T1_IT_CNAF"))
ele['Inputs'] = {"/MY/BLOCK/NAME#73e99a52": ["T1_US_FNAL", "T1_IT_CNAF", "T2_DE_DESY"]}
self.assertFalse(ele.passesSiteRestriction("T1_US_FNAL"))
self.assertTrue(ele.passesSiteRestriction("T1_IT_CNAF"))
self.assertFalse(ele.passesSiteRestriction("T2_DE_DESY"))
def testId(self):
"""Id calculated correctly"""
ele = WorkQueueElement(RequestName='test')
self.assertEqual(ele.id, '8b2e394154f6464f4b2aaf086a45f8c2')
def testPassesSiteRestriction(self):
"""
Workqueue element site restriction check (same as workRestrictions)
"""
# test element ala MonteCarlo
ele = WorkQueueElement(SiteWhitelist=["T1_IT_CNAF", "T2_DE_DESY"],
SiteBlacklist=["T1_US_FNAL"])
self.assertFalse(ele.passesSiteRestriction("T1_US_FNAL"))
self.assertFalse(ele.passesSiteRestriction("T2_CH_CERN"))
self.assertTrue(ele.passesSiteRestriction("T1_IT_CNAF"))
# test element with input dataset
ele['Inputs'] = {"/MY/BLOCK/NAME#73e99a52": []}
self.assertFalse(ele.passesSiteRestriction("T1_US_FNAL"))
self.assertFalse(ele.passesSiteRestriction("T2_CH_CERN"))
self.assertFalse(ele.passesSiteRestriction("T1_IT_CNAF"))
self.assertFalse(ele.passesSiteRestriction("T2_DE_DESY"))
ele['Inputs'] = {
"/MY/BLOCK/NAME#73e99a52": ["T1_US_FNAL", "T2_DE_DESY"]
}
self.assertFalse(ele.passesSiteRestriction("T1_US_FNAL"))
self.assertFalse(ele.passesSiteRestriction("T1_IT_CNAF"))
self.assertTrue(ele.passesSiteRestriction("T2_DE_DESY"))
# test element with input and parent dataset
ele['Inputs'] = {"/MY/BLOCK/NAME#73e99a52": []}
ele['ParentFlag'] = True
ele['ParentData'] = {"/MY/BLOCK2/NAME#002590494c06": []}
self.assertFalse(ele.passesSiteRestriction("T1_US_FNAL"))
self.assertFalse(ele.passesSiteRestriction("T2_CH_CERN"))
self.assertFalse(ele.passesSiteRestriction("T1_IT_CNAF"))
self.assertFalse(ele.passesSiteRestriction("T2_DE_DESY"))
ele['Inputs'] = {
"/MY/BLOCK/NAME#73e99a52": ["T1_US_FNAL", "T2_DE_DESY"]
}
self.assertFalse(ele.passesSiteRestriction("T1_US_FNAL"))
self.assertFalse(ele.passesSiteRestriction("T2_DE_DESY"))
ele['Inputs'] = {
"/MY/BLOCK/NAME#73e99a52": ["T1_US_FNAL", "T2_DE_DESY"]
}
ele['ParentData'] = {
"/MY/BLOCK2/NAME#002590494c06":
["T1_IT_CNAF", "T2_CH_CERN", "T2_DE_DESY"]
}
self.assertFalse(ele.passesSiteRestriction("T1_US_FNAL"))
self.assertFalse(ele.passesSiteRestriction("T1_IT_CNAF"))
self.assertTrue(ele.passesSiteRestriction("T2_DE_DESY"))
# test element with input, parent and pileup dataset
ele['PileupData'] = {"/MY/DATASET/NAME": []}
self.assertFalse(ele.passesSiteRestriction("T1_US_FNAL"))
self.assertFalse(ele.passesSiteRestriction("T2_CH_CERN"))
self.assertFalse(ele.passesSiteRestriction("T1_IT_CNAF"))
self.assertFalse(ele.passesSiteRestriction("T2_DE_DESY"))
ele['PileupData'] = {
"/MY/DATASET/NAME": ["T2_US_Nebraska", "T1_IT_CNAF"]
}
self.assertFalse(ele.passesSiteRestriction("T1_IT_CNAF"))
ele['Inputs'] = {
"/MY/BLOCK/NAME#73e99a52":
["T1_US_FNAL", "T1_IT_CNAF", "T2_DE_DESY"]
}
self.assertFalse(ele.passesSiteRestriction("T1_US_FNAL"))
self.assertTrue(ele.passesSiteRestriction("T1_IT_CNAF"))
self.assertFalse(ele.passesSiteRestriction("T2_DE_DESY"))
