def performQueueCleanupActions(self, skipWMBS = False):
"""
Apply end policies to determine work status & cleanup finished work
"""
if not self.backend.isAvailable():
self.logger.warning('Backend busy or down: skipping cleanup tasks')
return
self.backend.pullFromParent() # Check we are upto date with inbound changes
self.backend.fixConflicts() # before doing anything fix any conflicts
wf_to_cancel = [] # record what we did for task_activity
finished_elements = []
useWMBS = not skipWMBS and self.params['LocalQueueFlag']
# Get queue elements grouped by their workflow with updated wmbs progress
# Cancel if requested, update locally and remove obsolete elements
for wf, elements in self.status(dictKey = "RequestName", syncWithWMBS = useWMBS).items():
try:
parents = self.backend.getInboxElements(RequestName = wf)
if not parents:
raise RuntimeError, "Parent elements not found for %s" % wf
self.logger.debug("Queue status follows:")
results = endPolicy(elements, parents, self.params['EndPolicySettings'])
for result in results:
self.logger.debug("Request %s, Status %s, Full info: %s" % (result['RequestName'], result['Status'], result))
# check for cancellation requests (affects entire workflow)
if result['Status'] == 'CancelRequested':
canceled = self.cancelWork(elements = elements)
if canceled: # global wont cancel if work in child queue
wf_to_cancel.append(wf)
break
parent = result['ParentQueueElement']
if parent.modified:
self.backend.updateInboxElements(parent.id, **parent.statusMetrics())
if result.inEndState():
self.logger.info("Request %s finished (%s)" % (result['RequestName'], parent.statusMetrics()))
self.backend.deleteElements(*result['Elements'])
finished_elements.extend(result['Elements'])
continue
updated_elements = [x for x in result['Elements'] if x.modified]
for x in updated_elements:
self.logger.debug("Updating progress %s (%s): %s" % (x['RequsetName'], x.id, x.statusMetrics()))
[self.backend.updateElements(x.id, **x.statusMetrics()) for x in result['Elements'] if x.modified]
except Exception, ex:
self.logger.error('Error processing workflow "%s": %s' % (wf, str(ex)))
def performQueueCleanupActions(self, skipWMBS = False):
"""
Apply end policies to determine work status & cleanup finished work
"""
if not self.backend.isAvailable():
self.logger.warning('Backend busy or down: skipping cleanup tasks')
return
self.backend.pullFromParent() # Check we are upto date with inbound changes
if self.params['LocalQueueFlag']:
self.backend.fixConflicts() # before doing anything fix any conflicts
wf_to_cancel = [] # record what we did for task_activity
finished_elements = []
useWMBS = not skipWMBS and self.params['LocalQueueFlag']
# Get queue elements grouped by their workflow with updated wmbs progress
# Cancel if requested, update locally and remove obsolete elements
for wf in self.backend.getWorkflows(includeInbox = True, includeSpecs = True):
try:
elements = self.status(RequestName = wf, syncWithWMBS = useWMBS)
parents = self.backend.getInboxElements(RequestName = wf)
# check for left overs from past work where cleanup needed
if elements and not parents:
self.logger.info("Removing orphaned elements for %s" % wf)
self.backend.deleteElements(*elements)
continue
if not elements and not parents:
self.logger.info("Removing orphaned workflow %s" % wf)
try:
self.backend.db.delete_doc(wf)
except CouchNotFoundError:
pass
continue
self.logger.debug("Queue status follows:")
results = endPolicy(elements, parents, self.params['EndPolicySettings'])
for result in results:
self.logger.debug("Request %s, Status %s, Full info: %s" % (result['RequestName'], result['Status'], result))
# check for cancellation requests (affects entire workflow)
if result['Status'] == 'CancelRequested':
canceled = self.cancelWork(WorkflowName = wf)
if canceled: # global wont cancel if work in child queue
wf_to_cancel.append(wf)
break
elif result['Status'] == 'Negotiating':
self.logger.debug("Waiting for %s to finish splitting" % wf)
continue
parent = result['ParentQueueElement']
if parent.modified:
self.backend.updateInboxElements(parent.id, **parent.statusMetrics())
if result.inEndState():
if elements:
self.logger.info("Request %s finished (%s)" % (result['RequestName'], parent.statusMetrics()))
self.backend.deleteElements(*result['Elements'])
finished_elements.extend(result['Elements'])
else:
self.logger.info('Waiting for parent queue to delete "%s"' % result['RequestName'])
continue
updated_elements = [x for x in result['Elements'] if x.modified]
for x in updated_elements:
self.logger.debug("Updating progress %s (%s): %s" % (x['RequsetName'], x.id, x.statusMetrics()))
if not updated_elements and (float(parent.updatetime) + self.params['stuckElementAlertTime']) < time.time():
self.sendAlert(5, msg = 'Element for %s stuck for 24 hours.' % wf)
[self.backend.updateElements(x.id, **x.statusMetrics()) for x in updated_elements]
except Exception, ex:
self.logger.error('Error processing workflow "%s": %s' % (wf, str(ex)))
def performQueueCleanupActions(self, skipWMBS=False):
"""
Apply end policies to determine work status & cleanup finished work
"""
if not self.backend.isAvailable():
self.logger.warning('Backend busy or down: skipping cleanup tasks')
return
self.backend.pullFromParent(
) # Check we are upto date with inbound changes
if self.params['LocalQueueFlag']:
self.backend.fixConflicts(
) # before doing anything fix any conflicts
wf_to_cancel = [] # record what we did for task_activity
finished_elements = []
useWMBS = not skipWMBS and self.params['LocalQueueFlag']
# Get queue elements grouped by their workflow with updated wmbs progress
# Cancel if requested, update locally and remove obsolete elements
for wf in self.backend.getWorkflows(includeInbox=True,
includeSpecs=True):
try:
elements = self.status(RequestName=wf, syncWithWMBS=useWMBS)
parents = self.backend.getInboxElements(RequestName=wf)
# check for left overs from past work where cleanup needed
if elements and not parents:
self.logger.info("Removing orphaned elements for %s" % wf)
self.backend.deleteElements(*elements)
continue
if not elements and not parents:
self.logger.info("Removing orphaned workflow %s" % wf)
try:
self.backend.db.delete_doc(wf)
except CouchNotFoundError:
pass
continue
self.logger.debug("Queue status follows:")
results = endPolicy(elements, parents,
self.params['EndPolicySettings'])
for result in results:
self.logger.debug(
"Request %s, Status %s, Full info: %s" %
(result['RequestName'], result['Status'], result))
# check for cancellation requests (affects entire workflow)
if result['Status'] == 'CancelRequested':
canceled = self.cancelWork(WorkflowName=wf)
if canceled: # global wont cancel if work in child queue
wf_to_cancel.append(wf)
break
elif result['Status'] == 'Negotiating':
self.logger.debug(
"Waiting for %s to finish splitting" % wf)
continue
parent = result['ParentQueueElement']
if parent.modified:
self.backend.saveElements(parent)
if result.inEndState():
if elements:
self.logger.info("Request %s finished (%s)" %
(result['RequestName'],
parent.statusMetrics()))
self.backend.deleteElements(*result['Elements'])
finished_elements.extend(result['Elements'])
else:
self.logger.info(
'Waiting for parent queue to delete "%s"' %
result['RequestName'])
continue
self.addNewFilesToOpenSubscriptions(*elements)
updated_elements = [
x for x in result['Elements'] if x.modified
]
for x in updated_elements:
self.logger.debug(
"Updating progress %s (%s): %s" %
(x['RequsetName'], x.id, x.statusMetrics()))
if not updated_elements and (
float(parent.updatetime) +
self.params['stuckElementAlertTime']
) < time.time():
self.sendAlert(
5, msg='Element for %s stuck for 24 hours.' % wf)
[
self.backend.updateElements(x.id, **x.statusMetrics())
for x in updated_elements
]
except Exception, ex:
self.logger.error('Error processing workflow "%s": %s' %
(wf, str(ex)))
