示例#1
0
 def __init__(self, couchDB, id = None, elementParams = None):
     elementParams = elementParams or {}
     WorkQueueElement.__init__(self, **elementParams)
     if id:
         self._id = id
     self._document = Document(id = id)
     self._couch = couchDB
 def testSetId(self):
     """Can override id generation"""
     ele = WorkQueueElement(RequestName='testIdImmutable')
     before_id = ele.id
     ele.id = 'something_new'
     self.assertEqual('something_new', ele.id)
     self.assertNotEqual(before_id, ele.id)
 def __init__(self, couchDB, id=None, elementParams=None):
     elementParams = elementParams or {}
     WorkQueueElement.__init__(self, **elementParams)
     if id:
         self._id = id
     self._document = Document(id=id)
     self._couch = couchDB
 def testSetId(self):
     """Can override id generation"""
     ele = WorkQueueElement(RequestName='testIdImmutable')
     before_id = ele.id
     ele.id = 'something_new'
     self.assertEqual('something_new', ele.id)
     self.assertNotEqual(before_id, ele.id)
示例#5
0
 def testPriority(self):
     """Element priority and ordering handled correctly"""
     element = WorkQueueElement(RequestName = 'backend_test',
                                WMSpec = self.processingSpec,
                                Status = 'Available',
                                Jobs = 10, Priority = 1)
     highprielement = WorkQueueElement(RequestName = 'backend_test_high',
                                       WMSpec = self.processingSpec,
                                       Status = 'Available', Jobs = 10,
                                       Priority = 100)
     element2 = WorkQueueElement(RequestName = 'backend_test_2',
                                 WMSpec = self.processingSpec,
                                 Status = 'Available',
                                 Jobs = 10, Priority = 1)
     lowprielement = WorkQueueElement(RequestName = 'backend_test_low',
                                      WMSpec = self.processingSpec,
                                      Status = 'Available',
                                      Jobs = 10, Priority = 0.1)
     self.backend.insertElements([element])
     self.backend.availableWork({'place' : 1000})
     # timestamp in elements have second coarseness, 2nd element must
     # have a higher timestamp to force it after the 1st
     time.sleep(1)
     self.backend.insertElements([lowprielement, element2, highprielement])
     self.backend.availableWork({'place' : 1000})
     work = self.backend.availableWork({'place' : 1000})
     # order should be high to low, with the standard elements in the order
     # they were queueud
     self.assertEqual([x['RequestName'] for x in work[0]],
                      ['backend_test_high', 'backend_test', 'backend_test_2', 'backend_test_low'])
示例#6
0
 def testNegotiating(self):
     """Workflow only partially split"""
     # one element split and available in workqueue, other not split yet
     parent2 = WQE(Status = 'Negotiating')
     parent2.id = 2
     parents = [self.parent, parent2]
     # throw error as can only compute status when all elements split
     self.assertRaises(RuntimeError, self.policy(), [self.available], parents)
示例#7
0
 def testNegotiating(self):
     """Workflow only partially split"""
     # one element split and available in workqueue, other not split yet
     parent2 = WQE(Status = 'Negotiating')
     parent2.id = 2
     parents = [self.parent, parent2]
     results = self.policy()([self.available], parents)
     self.assertEqual(len(results), 2)
     self.assertEqual(results[0]['Status'], 'Acquired')
     self.assertEqual(results[1]['Status'], 'Negotiating')
示例#8
0
 def testIdIgnoresIrrelevant(self):
     """Id calculation ignores irrelavant variables"""
     ele = WorkQueueElement(RequestName = 'testIdIgnoresIrrelevant')
     this_id = ele.id
     for params2 in itertools.product(Progress, Priority, Teams,
                                      ParentQueueUrl, ParentQueueId):
         ele = WorkQueueElement(RequestName = 'testIdIgnoresIrrelevant',
                                PercentSuccess = params2[0],
                                Priority = params2[1], TeamName = params2[2],
                                ParentQueueUrl = params2[3], ParentQueueId = params2[4],
                                )
         # id not changed by changing irrelvant parameters
         self.assertEqual(ele.id, this_id)
示例#9
0
    def newQueueElement(self, **args):
        # DBS Url may not be available in the initial task
        # but in the pileup data (MC pileup)
        dbsUrl = self.initialTask.dbsUrl()
        if dbsUrl is None and self.pileupData:
            # Get the first DBS found
            dbsUrl = next(iter(self.wmspec.listPileupDatasets()))

        args.setdefault('Status', 'Available')
        args.setdefault('WMSpec', self.wmspec)
        args.setdefault('Task', self.initialTask)
        args.setdefault('RequestName', self.wmspec.name())
        args.setdefault('TaskName', self.initialTask.name())
        args.setdefault('Dbs', dbsUrl)
        args.setdefault('SiteWhitelist', self.initialTask.siteWhitelist())
        args.setdefault('SiteBlacklist', self.initialTask.siteBlacklist())
        args.setdefault('StartPolicy', self.wmspec.startPolicy())
        args.setdefault('EndPolicy', self.wmspec.endPolicyParameters())
        args.setdefault('Priority', self.wmspec.priority())
        args.setdefault('PileupData', self.pileupData)
        if not args['Priority']:
            args['Priority'] = 0
        ele = WorkQueueElement(**args)
        for data, sites in viewitems(ele['Inputs']):
            if not sites:
                raise WorkQueueWMSpecError(
                    self.wmspec, 'Input data has no locations "%s"' % data)

        # catch infinite splitting loops
        if len(self.workQueueElements) > self.args.get('maxRequestSize', 1e8):
            raise WorkQueueWMSpecError(
                self.wmspec, 'Too many elements (%d)' %
                self.args.get('MaxRequestElements', 1e8))
        self.workQueueElements.append(ele)
示例#10
0
    def testPossibleSites(self):
        """
        Workqueue element data location check (same as workRestrictions)
        """
        # test element ala MonteCarlo
        ele = WorkQueueElement(SiteWhitelist=["T1_IT_CNAF", "T2_DE_DESY"])
        self.assertEqual(possibleSites(ele), ["T1_IT_CNAF", "T2_DE_DESY"])

        # test element with InputDataset but no location
        ele['Inputs'] = {"/MY/BLOCK/NAME#73e99a52": []}
        self.assertEqual(possibleSites(ele), [])
        # test element with InputDataset and no match location
        ele['Inputs'] = {
            "/MY/BLOCK/NAME#73e99a52": ["T1_US_FNAL", "T2_CH_CERN"]
        }
        self.assertEqual(possibleSites(ele), [])
        # test element with InputDataset and valid location
        ele['Inputs'] = {
            "/MY/BLOCK/NAME#73e99a52":
            ["T1_US_FNAL", "T2_CH_CERN", "T2_DE_DESY"]
        }
        self.assertEqual(possibleSites(ele), ["T2_DE_DESY"])

        # test element with InputDataset and ParentData with no location
        ele['Inputs'] = {
            "/MY/BLOCK/NAME#73e99a52":
            ["T1_US_FNAL", "T2_CH_CERN", "T2_DE_DESY"]
        }
        ele['ParentFlag'] = True
        ele['ParentData'] = {"/MY/BLOCK2/NAME#002590494c06": []}
        self.assertEqual(possibleSites(ele), [])
        # test element with InputDataset and ParentData with no match location
        ele['ParentData'] = {"/MY/BLOCK2/NAME#002590494c06": ["T1_IT_CNAF"]}
        self.assertEqual(possibleSites(ele), [])
        # test element with InputDataset and ParentData with valid location
        ele['ParentData'] = {
            "/MY/BLOCK2/NAME#002590494c06": ["T1_US_FNAL", "T2_DE_DESY"]
        }
        self.assertEqual(possibleSites(ele), ["T2_DE_DESY"])

        # test element with InputDataset, PileupData and ParentData with no location
        ele['Inputs'] = {
            "/MY/BLOCK/NAME#73e99a52":
            ["T1_US_FNAL", "T2_CH_CERN", "T2_DE_DESY"]
        }
        ele['ParentData'] = {"/MY/BLOCK2/NAME#002590494c06": ["T2_DE_DESY"]}
        ele['PileupData'] = {"/MY/DATASET/NAME": []}
        self.assertEqual(possibleSites(ele), [])
        # test element with InputDataset, PileupData and ParentData with no match location
        ele['PileupData'] = {"/MY/DATASET/NAME": ["T1_IT_CNAF", "T2_CH_CERN"]}
        self.assertEqual(possibleSites(ele), [])
        # test element with InputDataset, PileupData and ParentData with valid location
        ele['PileupData'] = {"/MY/DATASET/NAME": ["T1_IT_CNAF", "T2_DE_DESY"]}
        self.assertEqual(possibleSites(ele), ["T2_DE_DESY"])
示例#11
0
    def setUp(self):

        filePath = os.path.join(
            getTestBase(),
            "WMCore_t/WorkQueue_t/DataStructs_t/wq_available_elements.json")
        with open(filePath, "r") as f:
            gqData = json.load(f)

        self.gqElements = []
        for ele in gqData:
            self.gqElements.append(WorkQueueElement(**ele))
示例#12
0
 def testIdUnique(self):
     """Modifying a relevant parameter varies the id"""
     ids = {}
     # Vary parameters that affect the work or input data,
     # verify each id is unique
     for params in itertools.product(RequestNames, TaskNames, Inputs,
                                     Masks, Dbses, Acdcs):
         ele = WorkQueueElement(RequestName = params[0], TaskName = params[1],
                                Inputs = params[2], Mask = params[3],
                                Dbs = params[4], ACDC = params[5]
                                )
         self.assertFalse(ele.id in ids)
         ids[ele.id] = None
示例#13
0
    def testPossibleSitesLocationFlags(self):
        """
        Workqueue element data location check, using the input and PU data location flags
        """
        ele = WorkQueueElement(SiteWhitelist=["T1_IT_CNAF", "T2_DE_DESY"])

        # test element with InputDataset and no location, but input flag on
        ele['Inputs'] = {"/MY/BLOCK/NAME#73e99a52": []}
        ele['NoInputUpdate'] = True
        self.assertEqual(possibleSites(ele), ["T1_IT_CNAF", "T2_DE_DESY"])
        # test element with InputDataset and one match, but input flag on
        ele['Inputs'] = {
            "/MY/BLOCK/NAME#73e99a52": ["T1_IT_CNAF", "T2_CH_CERN"]
        }
        self.assertEqual(possibleSites(ele), ["T1_IT_CNAF", "T2_DE_DESY"])
        # test element with InputDataset and one match, but pu flag on
        ele['NoInputUpdate'] = False
        ele['NoPileupUpdate'] = True
        self.assertEqual(possibleSites(ele), ["T1_IT_CNAF"])
        # test element with InputDataset and one match, but both flags on
        ele['NoInputUpdate'] = True
        self.assertEqual(possibleSites(ele), ["T1_IT_CNAF", "T2_DE_DESY"])

        # test element with InputDataset and ParentData and no location, but both flags on
        ele['ParentFlag'] = True
        ele['ParentData'] = {"/MY/BLOCK2/NAME#002590494c06": []}
        self.assertEqual(possibleSites(ele), ["T1_IT_CNAF", "T2_DE_DESY"])
        # test element with InputDataset and ParentData and no location, but input flag on
        ele['NoPileupUpdate'] = False
        self.assertEqual(possibleSites(ele), ["T1_IT_CNAF", "T2_DE_DESY"])
        # test element with InputDataset and ParentData and no location, but pileup flag on
        ele['NoInputUpdate'] = False
        ele['NoPileupUpdate'] = True
        self.assertEqual(possibleSites(ele), [])

        # test element with InputDataset, PileupData and ParentData with no location, but pileup flag on
        ele['Inputs'] = {
            "/MY/BLOCK/NAME#73e99a52":
            ["T1_US_FNAL", "T2_CH_CERN", "T2_DE_DESY"]
        }
        ele['ParentData'] = {"/MY/BLOCK2/NAME#002590494c06": ["T2_DE_DESY"]}
        ele['PileupData'] = {"/MY/DATASET/NAME": []}
        self.assertEqual(possibleSites(ele), ["T2_DE_DESY"])
        # test element with InputDataset, PileupData and ParentData with no location, but both flags on
        ele['NoInputUpdate'] = True
        self.assertEqual(possibleSites(ele), ["T1_IT_CNAF", "T2_DE_DESY"])
        # test element with InputDataset, PileupData and ParentData with no location, but input flag on
        ele['NoPileupUpdate'] = False
        self.assertEqual(possibleSites(ele), [])
示例#14
0
 def newQueueElement(self, **args):
     args.setdefault('Status', 'Available')
     args.setdefault('WMSpec', self.wmspec)
     args.setdefault('Task', self.initialTask)
     args.setdefault('RequestName', self.wmspec.name())
     args.setdefault('TaskName', self.initialTask.name())
     args.setdefault('Dbs', self.initialTask.dbsUrl())
     args.setdefault('SiteWhitelist', self.initialTask.siteWhitelist())
     args.setdefault('SiteBlacklist', self.initialTask.siteBlacklist())
     args.setdefault('EndPolicy', self.wmspec.endPolicyParameters())
     args.setdefault('Priority', self.wmspec.priority())
     if not args['Priority']:
         args['Priority'] = 0
     ele = WorkQueueElement(**args)
     for data, sites in ele['Inputs'].items():
         if not sites:
             raise WorkQueueWMSpecError(
                 self.wmspec, 'Input data has no locations "%s"' % data)
     # catch infinite splitting loops
     if len(self.workQueueElements) > self.args.get('maxRequestSize', 1e8):
         raise WorkQueueWMSpecError(
             self.wmspec, 'Too many elements (%d)' %
             self.args.get('MaxRequestElements', 1e8))
     self.workQueueElements.append(ele)
示例#15
0
 def testIdImmutable(self):
     """Id fixed once calculated"""
     ele = WorkQueueElement(RequestName='testIdImmutable')
     before_id = ele.id
     ele['RequestName'] = 'somethingElse'
     self.assertEqual(before_id, ele.id)
    def testPassesSiteRestriction(self):
        """
        Workqueue element site restriction check (same as workRestrictions)
        """
        # test element ala MonteCarlo
        ele = WorkQueueElement(SiteWhitelist=["T1_IT_CNAF", "T2_DE_DESY"], SiteBlacklist=["T1_US_FNAL"])
        self.assertFalse(ele.passesSiteRestriction("T1_US_FNAL"))
        self.assertFalse(ele.passesSiteRestriction("T2_CH_CERN"))
        self.assertTrue(ele.passesSiteRestriction("T1_IT_CNAF"))

        # test element with input dataset
        ele['Inputs'] = {"/MY/BLOCK/NAME#73e99a52": []}
        self.assertFalse(ele.passesSiteRestriction("T1_US_FNAL"))
        self.assertFalse(ele.passesSiteRestriction("T2_CH_CERN"))
        self.assertFalse(ele.passesSiteRestriction("T1_IT_CNAF"))
        self.assertFalse(ele.passesSiteRestriction("T2_DE_DESY"))
        ele['Inputs'] = {"/MY/BLOCK/NAME#73e99a52": ["T1_US_FNAL", "T2_DE_DESY"]}
        self.assertFalse(ele.passesSiteRestriction("T1_US_FNAL"))
        self.assertFalse(ele.passesSiteRestriction("T1_IT_CNAF"))
        self.assertTrue(ele.passesSiteRestriction("T2_DE_DESY"))

        # test element with input and parent dataset
        ele['Inputs'] = {"/MY/BLOCK/NAME#73e99a52": []}
        ele['ParentFlag'] = True
        ele['ParentData'] = {"/MY/BLOCK2/NAME#002590494c06": []}
        self.assertFalse(ele.passesSiteRestriction("T1_US_FNAL"))
        self.assertFalse(ele.passesSiteRestriction("T2_CH_CERN"))
        self.assertFalse(ele.passesSiteRestriction("T1_IT_CNAF"))
        self.assertFalse(ele.passesSiteRestriction("T2_DE_DESY"))
        ele['Inputs'] = {"/MY/BLOCK/NAME#73e99a52": ["T1_US_FNAL", "T2_DE_DESY"]}
        self.assertFalse(ele.passesSiteRestriction("T1_US_FNAL"))
        self.assertFalse(ele.passesSiteRestriction("T2_DE_DESY"))
        ele['Inputs'] = {"/MY/BLOCK/NAME#73e99a52": ["T1_US_FNAL", "T2_DE_DESY"]}
        ele['ParentData'] = {"/MY/BLOCK2/NAME#002590494c06": ["T1_IT_CNAF", "T2_CH_CERN", "T2_DE_DESY"]}
        self.assertFalse(ele.passesSiteRestriction("T1_US_FNAL"))
        self.assertFalse(ele.passesSiteRestriction("T1_IT_CNAF"))
        self.assertTrue(ele.passesSiteRestriction("T2_DE_DESY"))

        # test element with input, parent and pileup dataset
        ele['PileupData'] = {"/MY/DATASET/NAME": []}
        self.assertFalse(ele.passesSiteRestriction("T1_US_FNAL"))
        self.assertFalse(ele.passesSiteRestriction("T2_CH_CERN"))
        self.assertFalse(ele.passesSiteRestriction("T1_IT_CNAF"))
        self.assertFalse(ele.passesSiteRestriction("T2_DE_DESY"))
        ele['PileupData'] = {"/MY/DATASET/NAME": ["T2_US_Nebraska", "T1_IT_CNAF"]}
        self.assertFalse(ele.passesSiteRestriction("T1_IT_CNAF"))
        ele['Inputs'] = {"/MY/BLOCK/NAME#73e99a52": ["T1_US_FNAL", "T1_IT_CNAF", "T2_DE_DESY"]}
        self.assertFalse(ele.passesSiteRestriction("T1_US_FNAL"))
        self.assertTrue(ele.passesSiteRestriction("T1_IT_CNAF"))
        self.assertFalse(ele.passesSiteRestriction("T2_DE_DESY"))
示例#17
0
 def testId(self):
     """Id calculated correctly"""
     ele = WorkQueueElement(RequestName='test')
     self.assertEqual(ele.id, '8b2e394154f6464f4b2aaf086a45f8c2')
示例#18
0
    def testPassesSiteRestriction(self):
        """
        Workqueue element site restriction check (same as workRestrictions)
        """
        # test element ala MonteCarlo
        ele = WorkQueueElement(SiteWhitelist=["T1_IT_CNAF", "T2_DE_DESY"],
                               SiteBlacklist=["T1_US_FNAL"])
        self.assertFalse(ele.passesSiteRestriction("T1_US_FNAL"))
        self.assertFalse(ele.passesSiteRestriction("T2_CH_CERN"))
        self.assertTrue(ele.passesSiteRestriction("T1_IT_CNAF"))

        # test element with input dataset
        ele['Inputs'] = {"/MY/BLOCK/NAME#73e99a52": []}
        self.assertFalse(ele.passesSiteRestriction("T1_US_FNAL"))
        self.assertFalse(ele.passesSiteRestriction("T2_CH_CERN"))
        self.assertFalse(ele.passesSiteRestriction("T1_IT_CNAF"))
        self.assertFalse(ele.passesSiteRestriction("T2_DE_DESY"))
        ele['Inputs'] = {
            "/MY/BLOCK/NAME#73e99a52": ["T1_US_FNAL", "T2_DE_DESY"]
        }
        self.assertFalse(ele.passesSiteRestriction("T1_US_FNAL"))
        self.assertFalse(ele.passesSiteRestriction("T1_IT_CNAF"))
        self.assertTrue(ele.passesSiteRestriction("T2_DE_DESY"))

        # test element with input and parent dataset
        ele['Inputs'] = {"/MY/BLOCK/NAME#73e99a52": []}
        ele['ParentFlag'] = True
        ele['ParentData'] = {"/MY/BLOCK2/NAME#002590494c06": []}
        self.assertFalse(ele.passesSiteRestriction("T1_US_FNAL"))
        self.assertFalse(ele.passesSiteRestriction("T2_CH_CERN"))
        self.assertFalse(ele.passesSiteRestriction("T1_IT_CNAF"))
        self.assertFalse(ele.passesSiteRestriction("T2_DE_DESY"))
        ele['Inputs'] = {
            "/MY/BLOCK/NAME#73e99a52": ["T1_US_FNAL", "T2_DE_DESY"]
        }
        self.assertFalse(ele.passesSiteRestriction("T1_US_FNAL"))
        self.assertFalse(ele.passesSiteRestriction("T2_DE_DESY"))
        ele['Inputs'] = {
            "/MY/BLOCK/NAME#73e99a52": ["T1_US_FNAL", "T2_DE_DESY"]
        }
        ele['ParentData'] = {
            "/MY/BLOCK2/NAME#002590494c06":
            ["T1_IT_CNAF", "T2_CH_CERN", "T2_DE_DESY"]
        }
        self.assertFalse(ele.passesSiteRestriction("T1_US_FNAL"))
        self.assertFalse(ele.passesSiteRestriction("T1_IT_CNAF"))
        self.assertTrue(ele.passesSiteRestriction("T2_DE_DESY"))

        # test element with input, parent and pileup dataset
        ele['PileupData'] = {"/MY/DATASET/NAME": []}
        self.assertFalse(ele.passesSiteRestriction("T1_US_FNAL"))
        self.assertFalse(ele.passesSiteRestriction("T2_CH_CERN"))
        self.assertFalse(ele.passesSiteRestriction("T1_IT_CNAF"))
        self.assertFalse(ele.passesSiteRestriction("T2_DE_DESY"))
        ele['PileupData'] = {
            "/MY/DATASET/NAME": ["T2_US_Nebraska", "T1_IT_CNAF"]
        }
        self.assertFalse(ele.passesSiteRestriction("T1_IT_CNAF"))
        ele['Inputs'] = {
            "/MY/BLOCK/NAME#73e99a52":
            ["T1_US_FNAL", "T1_IT_CNAF", "T2_DE_DESY"]
        }
        self.assertFalse(ele.passesSiteRestriction("T1_US_FNAL"))
        self.assertTrue(ele.passesSiteRestriction("T1_IT_CNAF"))
        self.assertFalse(ele.passesSiteRestriction("T2_DE_DESY"))