def testDuplicateInsertion(self):
"""Try to insert elements multiple times"""
element1 = CouchWorkQueueElement(
self.couch_db,
elementParams={
'RequestName': 'backend_test',
'WMSpec': self.processingSpec,
'Status': 'Available',
'Jobs': 10,
'Inputs': {
self.processingSpec.listInputDatasets()[0] + '#1': []
}
})
element2 = CouchWorkQueueElement(
self.couch_db,
elementParams={
'RequestName': 'backend_test',
'WMSpec': self.processingSpec,
'Status': 'Available',
'Jobs': 20,
'Inputs': {
self.processingSpec.listInputDatasets()[0] + '#2': []
}
})
self.backend.insertElements([element1, element2])
self.backend.insertElements([element1, element2])
# check no duplicates and no conflicts
self.assertEqual(len(self.backend.db.allDocs()['rows']),
4) # design doc + workflow + 2 elements
self.assertEqual(
self.backend.db.loadView('WorkQueue', 'conflicts')['total_rows'],
0)
def testIdFromDbImmutable(self):
"""Modifying element id algorithm doesn't change existing id's"""
ele = CouchWorkQueueElement(self.couch_db,
elementParams={'RequestName': 'test'})
ele.save()
self.couch_db.commit(timestamp=True)
ele2 = CouchWorkQueueElement(self.couch_db, id=ele.id).load()
ele2['RequestName'] = 'ThisWouldCauseIdToChange'
# id should not change
self.assertEqual(ele.id, ele2.id)
# save should modify existing element
ele2.save()
self.couch_db.commit(timestamp=True)
self.assertEqual(self.couch_db.info()['doc_count'], 1)
def insertElements(self, units, parent=None):
"""
Insert element to database
@param parent is the parent WorkQueueObject these element's belong to.
i.e. a workflow which has been split
"""
if not units:
return
# store spec file separately - assume all elements share same spec
self.insertWMSpec(units[0]['WMSpec'])
newUnitsInserted = []
for unit in units:
# cast to couch
if not isinstance(unit, CouchWorkQueueElement):
unit = CouchWorkQueueElement(self.db, elementParams=dict(unit))
if parent:
unit['ParentQueueId'] = parent.id
unit['TeamName'] = parent['TeamName']
unit['WMBSUrl'] = parent['WMBSUrl']
if unit._couch.documentExists(unit.id):
self.logger.info(
'Element "%s" already exists, skip insertion.' % unit.id)
continue
else:
newUnitsInserted.append(unit)
unit.save()
unit._couch.commit(all_or_nothing=True)
return newUnitsInserted
def testIdSaved(self):
"""Generated id used as db id"""
ele = CouchWorkQueueElement(self.couch_db,
elementParams={'RequestName': 'test'})
ele.save()
self.couch_db.commit(timestamp=True)
self.assertTrue(self.couch_db.documentExists(ele.id))
self.assertEqual(self.couch_db.info()['doc_count'], 1)
def getElements(self, status=None, elementIDs=None, returnIdOnly=False,
db=None, loadSpec=False, WorkflowName=None, **elementFilters):
"""Return elements that match requirements
status, elementIDs & filters are 'AND'ed together to filter elements.
returnIdOnly causes the element not to be loaded and only the id returned
db is used to specify which database to return from
loadSpec causes the workflow for each spec to be loaded.
WorkflowName may be used in the place of RequestName
"""
key = []
if not db:
db = self.db
if elementFilters.get('RequestName') and not WorkflowName:
WorkflowName = elementFilters.pop('RequestName')
if elementIDs:
if elementFilters or status or returnIdOnly:
raise ValueError(
"Can't specify extra filters (or return id's) when using element id's with getElements()")
elements = [CouchWorkQueueElement(db, i).load() for i in elementIDs]
else:
options = {'include_docs': True, 'filter': elementFilters, 'idOnly': returnIdOnly, 'reduce': False}
# filter on workflow or status if possible
filterName = 'elementsByWorkflow'
if WorkflowName:
key.append(WorkflowName)
elif status:
filterName = 'elementsByStatus'
key.append(status)
elif elementFilters.get('SubscriptionId'):
key.append(elementFilters['SubscriptionId'])
filterName = 'elementsBySubscription'
# add given params to filters
if status:
options['filter']['Status'] = status
if WorkflowName:
options['filter']['RequestName'] = WorkflowName
view = db.loadList('WorkQueue', 'filter', filterName, options, key)
view = json.loads(view)
if returnIdOnly:
return view
elements = [CouchWorkQueueElement.fromDocument(db, row) for row in view]
if loadSpec:
specs = {} # cache as may have multiple elements for same spec
for ele in elements:
if ele['RequestName'] not in specs:
wmspec = self.getWMSpec(ele['RequestName'])
specs[ele['RequestName']] = wmspec
ele['WMSpec'] = specs[ele['RequestName']]
del specs
return elements
def createWork(self, spec, **kwargs):
"""Return the Inbox element for this spec.
This does not persist it to the database.
"""
kwargs.update({
'WMSpec': spec,
'RequestName': spec.name(),
'EndPolicy': spec.endPolicyParameters()
})
unit = CouchWorkQueueElement(self.inbox, elementParams=kwargs)
unit.id = spec.name()
return unit
def testFixElementConflicts(self):
"""Element conflicts handled"""
# list of before and after elements for comparison
values = [
# status conflict
[[{
'RequestName': 'arequest',
'Status': 'Available'
}, {
'RequestName': 'arequest',
'Status': 'CancelRequested'
}],
[{
'RequestName': 'arequest',
'Status': 'CancelRequested'
}, {
'RequestName': 'arequest',
'Status': 'CancelRequested'
}]],
# location conflict - local uses location and global subscriptions
[[{
'RequestName': 'brequest',
'Status': 'CancelRequested',
'Inputs': {
'a': [1]
}
}, {
'RequestName': 'brequest',
'Status': 'Available',
'Inputs': {
'a': [1, 2]
}
}],
[{
'RequestName': 'brequest',
'Status': 'CancelRequested',
'Inputs': {
'a': [1, 2]
}
}, {
'RequestName': 'brequest',
'Status': 'Available',
'Inputs': {
'a': [1, 2]
}
}]],
# status and progress conflict
[[{
'RequestName': 'crequest',
'Status': 'Available',
'PercentComplete': 69
}, {
'RequestName': 'crequest',
'Status': 'CancelRequested'
}],
[{
'RequestName': 'crequest',
'Status': 'CancelRequested',
'PercentComplete': 69
}, {
'RequestName': 'crequest',
'Status': 'CancelRequested'
}]],
# status and subscription conflict
[[{
'RequestName': 'drequest',
'Status': 'Running',
'SubscriptionId': 69
}, {
'RequestName': 'drequest',
'Status': 'CancelRequested'
}],
[{
'RequestName': 'drequest',
'Status': 'CancelRequested',
'SubscriptionId': 69
}, {
'RequestName': 'drequest',
'Status': 'CancelRequested'
}]],
# whitelist conflict
[[{
'RequestName': 'erequest',
'SiteWhitelist': [1, 2]
}, {
'RequestName': 'erequest'
}],
[{
'RequestName': 'erequest',
'SiteWhitelist': [1, 2]
}, {
'RequestName': 'erequest'
}]],
# black conflict
[[{
'RequestName': 'frequest'
}, {
'RequestName': 'frequest',
'SiteBlacklist': [1, 2]
}],
[{
'RequestName': 'frequest',
'SiteBlacklist': [1, 2]
}, {
'RequestName': 'frequest',
'SiteBlacklist': [1, 2]
}]],
# priority conflict
[[{
'RequestName': 'grequest'
}, {
'RequestName': 'grequest',
'Priority': 69
}],
[{
'RequestName': 'grequest',
'Priority': 69
}, {
'RequestName': 'grequest',
'Priority': 69
}]],
]
for before, after in values:
before = [
CouchWorkQueueElement(self.couch_db, 1, elementParams=x)
for x in before
]
# add fake revs
[x._document.__setitem__('_rev', "a") for x in before]
after = [
CouchWorkQueueElement(self.couch_db, 1, elementParams=x)
for x in after
]
self.assertEqual(list(fixElementConflicts(*before)), after)
