def testDuplicateInsertion(self): """Try to insert elements multiple times""" element1 = CouchWorkQueueElement( self.couch_db, elementParams={ 'RequestName': 'backend_test', 'WMSpec': self.processingSpec, 'Status': 'Available', 'Jobs': 10, 'Inputs': { self.processingSpec.listInputDatasets()[0] + '#1': [] } }) element2 = CouchWorkQueueElement( self.couch_db, elementParams={ 'RequestName': 'backend_test', 'WMSpec': self.processingSpec, 'Status': 'Available', 'Jobs': 20, 'Inputs': { self.processingSpec.listInputDatasets()[0] + '#2': [] } }) self.backend.insertElements([element1, element2]) self.backend.insertElements([element1, element2]) # check no duplicates and no conflicts self.assertEqual(len(self.backend.db.allDocs()['rows']), 4) # design doc + workflow + 2 elements self.assertEqual( self.backend.db.loadView('WorkQueue', 'conflicts')['total_rows'], 0)
def testIdFromDbImmutable(self): """Modifying element id algorithm doesn't change existing id's""" ele = CouchWorkQueueElement(self.couch_db, elementParams={'RequestName': 'test'}) ele.save() self.couch_db.commit(timestamp=True) ele2 = CouchWorkQueueElement(self.couch_db, id=ele.id).load() ele2['RequestName'] = 'ThisWouldCauseIdToChange' # id should not change self.assertEqual(ele.id, ele2.id) # save should modify existing element ele2.save() self.couch_db.commit(timestamp=True) self.assertEqual(self.couch_db.info()['doc_count'], 1)
def insertElements(self, units, parent=None): """ Insert element to database @param parent is the parent WorkQueueObject these element's belong to. i.e. a workflow which has been split """ if not units: return # store spec file separately - assume all elements share same spec self.insertWMSpec(units[0]['WMSpec']) newUnitsInserted = [] for unit in units: # cast to couch if not isinstance(unit, CouchWorkQueueElement): unit = CouchWorkQueueElement(self.db, elementParams=dict(unit)) if parent: unit['ParentQueueId'] = parent.id unit['TeamName'] = parent['TeamName'] unit['WMBSUrl'] = parent['WMBSUrl'] if unit._couch.documentExists(unit.id): self.logger.info( 'Element "%s" already exists, skip insertion.' % unit.id) continue else: newUnitsInserted.append(unit) unit.save() unit._couch.commit(all_or_nothing=True) return newUnitsInserted
def testIdSaved(self): """Generated id used as db id""" ele = CouchWorkQueueElement(self.couch_db, elementParams={'RequestName': 'test'}) ele.save() self.couch_db.commit(timestamp=True) self.assertTrue(self.couch_db.documentExists(ele.id)) self.assertEqual(self.couch_db.info()['doc_count'], 1)
def getElements(self, status=None, elementIDs=None, returnIdOnly=False, db=None, loadSpec=False, WorkflowName=None, **elementFilters): """Return elements that match requirements status, elementIDs & filters are 'AND'ed together to filter elements. returnIdOnly causes the element not to be loaded and only the id returned db is used to specify which database to return from loadSpec causes the workflow for each spec to be loaded. WorkflowName may be used in the place of RequestName """ key = [] if not db: db = self.db if elementFilters.get('RequestName') and not WorkflowName: WorkflowName = elementFilters.pop('RequestName') if elementIDs: if elementFilters or status or returnIdOnly: raise ValueError( "Can't specify extra filters (or return id's) when using element id's with getElements()") elements = [CouchWorkQueueElement(db, i).load() for i in elementIDs] else: options = {'include_docs': True, 'filter': elementFilters, 'idOnly': returnIdOnly, 'reduce': False} # filter on workflow or status if possible filterName = 'elementsByWorkflow' if WorkflowName: key.append(WorkflowName) elif status: filterName = 'elementsByStatus' key.append(status) elif elementFilters.get('SubscriptionId'): key.append(elementFilters['SubscriptionId']) filterName = 'elementsBySubscription' # add given params to filters if status: options['filter']['Status'] = status if WorkflowName: options['filter']['RequestName'] = WorkflowName view = db.loadList('WorkQueue', 'filter', filterName, options, key) view = json.loads(view) if returnIdOnly: return view elements = [CouchWorkQueueElement.fromDocument(db, row) for row in view] if loadSpec: specs = {} # cache as may have multiple elements for same spec for ele in elements: if ele['RequestName'] not in specs: wmspec = self.getWMSpec(ele['RequestName']) specs[ele['RequestName']] = wmspec ele['WMSpec'] = specs[ele['RequestName']] del specs return elements
def createWork(self, spec, **kwargs): """Return the Inbox element for this spec. This does not persist it to the database. """ kwargs.update({ 'WMSpec': spec, 'RequestName': spec.name(), 'EndPolicy': spec.endPolicyParameters() }) unit = CouchWorkQueueElement(self.inbox, elementParams=kwargs) unit.id = spec.name() return unit
def testFixElementConflicts(self): """Element conflicts handled""" # list of before and after elements for comparison values = [ # status conflict [[{ 'RequestName': 'arequest', 'Status': 'Available' }, { 'RequestName': 'arequest', 'Status': 'CancelRequested' }], [{ 'RequestName': 'arequest', 'Status': 'CancelRequested' }, { 'RequestName': 'arequest', 'Status': 'CancelRequested' }]], # location conflict - local uses location and global subscriptions [[{ 'RequestName': 'brequest', 'Status': 'CancelRequested', 'Inputs': { 'a': [1] } }, { 'RequestName': 'brequest', 'Status': 'Available', 'Inputs': { 'a': [1, 2] } }], [{ 'RequestName': 'brequest', 'Status': 'CancelRequested', 'Inputs': { 'a': [1, 2] } }, { 'RequestName': 'brequest', 'Status': 'Available', 'Inputs': { 'a': [1, 2] } }]], # status and progress conflict [[{ 'RequestName': 'crequest', 'Status': 'Available', 'PercentComplete': 69 }, { 'RequestName': 'crequest', 'Status': 'CancelRequested' }], [{ 'RequestName': 'crequest', 'Status': 'CancelRequested', 'PercentComplete': 69 }, { 'RequestName': 'crequest', 'Status': 'CancelRequested' }]], # status and subscription conflict [[{ 'RequestName': 'drequest', 'Status': 'Running', 'SubscriptionId': 69 }, { 'RequestName': 'drequest', 'Status': 'CancelRequested' }], [{ 'RequestName': 'drequest', 'Status': 'CancelRequested', 'SubscriptionId': 69 }, { 'RequestName': 'drequest', 'Status': 'CancelRequested' }]], # whitelist conflict [[{ 'RequestName': 'erequest', 'SiteWhitelist': [1, 2] }, { 'RequestName': 'erequest' }], [{ 'RequestName': 'erequest', 'SiteWhitelist': [1, 2] }, { 'RequestName': 'erequest' }]], # black conflict [[{ 'RequestName': 'frequest' }, { 'RequestName': 'frequest', 'SiteBlacklist': [1, 2] }], [{ 'RequestName': 'frequest', 'SiteBlacklist': [1, 2] }, { 'RequestName': 'frequest', 'SiteBlacklist': [1, 2] }]], # priority conflict [[{ 'RequestName': 'grequest' }, { 'RequestName': 'grequest', 'Priority': 69 }], [{ 'RequestName': 'grequest', 'Priority': 69 }, { 'RequestName': 'grequest', 'Priority': 69 }]], ] for before, after in values: before = [ CouchWorkQueueElement(self.couch_db, 1, elementParams=x) for x in before ] # add fake revs [x._document.__setitem__('_rev', "a") for x in before] after = [ CouchWorkQueueElement(self.couch_db, 1, elementParams=x) for x in after ] self.assertEqual(list(fixElementConflicts(*before)), after)