def killWorkflows(self, workflows):
"""
_killWorkflows_
Delete all the information in couch and WMBS about the given
workflow, go through all subscriptions and delete one by
one.
The input is a dictionary with workflow names as keys, fully loaded WMWorkloads and
subscriptions lists as values
"""
for workflow in workflows:
logging.info("Deleting workflow %s" % workflow)
try:
#Get the task-workflow ids, sort them by ID,
#higher ID first so we kill
#the leaves of the tree first, root last
workflowsIDs = workflows[workflow]["workflows"].keys()
workflowsIDs.sort(reverse=True)
#Now go through all tasks and load the WMBS workflow objects
wmbsWorkflows = []
for wfID in workflowsIDs:
wmbsWorkflow = Workflow(id=wfID)
wmbsWorkflow.load()
wmbsWorkflows.append(wmbsWorkflow)
#Time to shoot one by one
for wmbsWorkflow in wmbsWorkflows:
if self.uploadPublishInfo:
self.createAndUploadPublish(wmbsWorkflow)
#Load all the associated subscriptions and shoot them one by one
subIDs = workflows[workflow]["workflows"][wmbsWorkflow.id]
for subID in subIDs:
subscription = Subscription(id=subID)
subscription['workflow'] = wmbsWorkflow
subscription.load()
subscription.deleteEverything()
#Check that the workflow is gone
if wmbsWorkflow.exists():
#Something went bad, this workflow
#should be gone by now
msg = "Workflow %s, Task %s was not deleted completely" % (
wmbsWorkflow.name, wmbsWorkflow.task)
raise TaskArchiverPollerException(msg)
#Now delete directories
_, taskDir = getMasterName(startDir=self.jobCacheDir,
workflow=wmbsWorkflow)
logging.info("About to delete work directory %s" % taskDir)
if os.path.exists(taskDir):
if os.path.isdir(taskDir):
shutil.rmtree(taskDir)
else:
# What we think of as a working directory is not a directory
# This should never happen and there is no way we can recover
# from this here. Bail out now and have someone look at things.
msg = "Work directory is not a directory, this should never happen: %s" % taskDir
raise TaskArchiverPollerException(msg)
else:
msg = "Attempted to delete work directory but it was already gone: %s" % taskDir
logging.debug(msg)
spec = workflows[workflow]["spec"]
topTask = spec.getTopLevelTask()[0]
# Now take care of the sandbox
sandbox = getattr(topTask.data.input, 'sandbox', None)
if sandbox:
sandboxDir = os.path.dirname(sandbox)
if os.path.isdir(sandboxDir):
shutil.rmtree(sandboxDir)
logging.debug("Sandbox dir deleted")
else:
logging.error(
"Attempted to delete sandbox dir but it was already gone: %s"
% sandboxDir)
except Exception, ex:
msg = "Critical error while deleting workflow %s\n" % workflow
msg += str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
self.sendAlert(2, msg=msg)
def killWorkflows(self, workflows):
"""
_killWorkflows_
Delete all the information in couch and WMBS about the given
workflow, go through all subscriptions and delete one by
one.
The input is a dictionary with workflow names as keys, fully loaded WMWorkloads and
subscriptions lists as values
"""
for workflow in workflows:
logging.info("Deleting workflow %s" % workflow)
try:
#Get the task-workflow ids, sort them by ID,
#higher ID first so we kill
#the leaves of the tree first, root last
workflowsIDs = workflows[workflow]["workflows"].keys()
workflowsIDs.sort(reverse = True)
#Now go through all tasks and load the WMBS workflow objects
wmbsWorkflows = []
for wfID in workflowsIDs:
wmbsWorkflow = Workflow(id = wfID)
wmbsWorkflow.load()
wmbsWorkflows.append(wmbsWorkflow)
#Time to shoot one by one
for wmbsWorkflow in wmbsWorkflows:
if self.uploadPublishInfo:
self.createAndUploadPublish(wmbsWorkflow)
#Load all the associated subscriptions and shoot them one by one
subIDs = workflows[workflow]["workflows"][wmbsWorkflow.id]
for subID in subIDs:
subscription = Subscription(id = subID)
subscription['workflow'] = wmbsWorkflow
subscription.load()
subscription.deleteEverything()
#Check that the workflow is gone
if wmbsWorkflow.exists():
#Something went bad, this workflow
#should be gone by now
msg = "Workflow %s, Task %s was not deleted completely" % (wmbsWorkflow.name,
wmbsWorkflow.task)
raise TaskArchiverPollerException(msg)
#Now delete directories
_, taskDir = getMasterName(startDir = self.jobCacheDir,
workflow = wmbsWorkflow)
logging.info("About to delete work directory %s" % taskDir)
if os.path.exists(taskDir):
if os.path.isdir(taskDir):
shutil.rmtree(taskDir)
else:
# What we think of as a working directory is not a directory
# This should never happen and there is no way we can recover
# from this here. Bail out now and have someone look at things.
msg = "Work directory is not a directory, this should never happen: %s" % taskDir
raise TaskArchiverPollerException(msg)
else:
msg = "Attempted to delete work directory but it was already gone: %s" % taskDir
logging.debug(msg)
spec = workflows[workflow]["spec"]
topTask = spec.getTopLevelTask()[0]
# Now take care of the sandbox
sandbox = getattr(topTask.data.input, 'sandbox', None)
if sandbox:
sandboxDir = os.path.dirname(sandbox)
if os.path.isdir(sandboxDir):
shutil.rmtree(sandboxDir)
logging.debug("Sandbox dir deleted")
else:
logging.error("Attempted to delete sandbox dir but it was already gone: %s" % sandboxDir)
except Exception, ex:
msg = "Critical error while deleting workflow %s\n" % workflow
msg += str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
self.sendAlert(2, msg = msg)
