def killWorkflows(self, workflows): """ _killWorkflows_ Delete all the information in couch and WMBS about the given workflow, go through all subscriptions and delete one by one. The input is a dictionary with workflow names as keys, fully loaded WMWorkloads and subscriptions lists as values """ for workflow in workflows: logging.info("Deleting workflow %s" % workflow) try: #Get the task-workflow ids, sort them by ID, #higher ID first so we kill #the leaves of the tree first, root last workflowsIDs = workflows[workflow]["workflows"].keys() workflowsIDs.sort(reverse=True) #Now go through all tasks and load the WMBS workflow objects wmbsWorkflows = [] for wfID in workflowsIDs: wmbsWorkflow = Workflow(id=wfID) wmbsWorkflow.load() wmbsWorkflows.append(wmbsWorkflow) #Time to shoot one by one for wmbsWorkflow in wmbsWorkflows: if self.uploadPublishInfo: self.createAndUploadPublish(wmbsWorkflow) #Load all the associated subscriptions and shoot them one by one subIDs = workflows[workflow]["workflows"][wmbsWorkflow.id] for subID in subIDs: subscription = Subscription(id=subID) subscription['workflow'] = wmbsWorkflow subscription.load() subscription.deleteEverything() #Check that the workflow is gone if wmbsWorkflow.exists(): #Something went bad, this workflow #should be gone by now msg = "Workflow %s, Task %s was not deleted completely" % ( wmbsWorkflow.name, wmbsWorkflow.task) raise TaskArchiverPollerException(msg) #Now delete directories _, taskDir = getMasterName(startDir=self.jobCacheDir, workflow=wmbsWorkflow) logging.info("About to delete work directory %s" % taskDir) if os.path.exists(taskDir): if os.path.isdir(taskDir): shutil.rmtree(taskDir) else: # What we think of as a working directory is not a directory # This should never happen and there is no way we can recover # from this here. Bail out now and have someone look at things. msg = "Work directory is not a directory, this should never happen: %s" % taskDir raise TaskArchiverPollerException(msg) else: msg = "Attempted to delete work directory but it was already gone: %s" % taskDir logging.debug(msg) spec = workflows[workflow]["spec"] topTask = spec.getTopLevelTask()[0] # Now take care of the sandbox sandbox = getattr(topTask.data.input, 'sandbox', None) if sandbox: sandboxDir = os.path.dirname(sandbox) if os.path.isdir(sandboxDir): shutil.rmtree(sandboxDir) logging.debug("Sandbox dir deleted") else: logging.error( "Attempted to delete sandbox dir but it was already gone: %s" % sandboxDir) except Exception, ex: msg = "Critical error while deleting workflow %s\n" % workflow msg += str(ex) msg += str(traceback.format_exc()) logging.error(msg) self.sendAlert(2, msg=msg)
def killWorkflows(self, workflows): """ _killWorkflows_ Delete all the information in couch and WMBS about the given workflow, go through all subscriptions and delete one by one. The input is a dictionary with workflow names as keys, fully loaded WMWorkloads and subscriptions lists as values """ for workflow in workflows: logging.info("Deleting workflow %s" % workflow) try: #Get the task-workflow ids, sort them by ID, #higher ID first so we kill #the leaves of the tree first, root last workflowsIDs = workflows[workflow]["workflows"].keys() workflowsIDs.sort(reverse = True) #Now go through all tasks and load the WMBS workflow objects wmbsWorkflows = [] for wfID in workflowsIDs: wmbsWorkflow = Workflow(id = wfID) wmbsWorkflow.load() wmbsWorkflows.append(wmbsWorkflow) #Time to shoot one by one for wmbsWorkflow in wmbsWorkflows: if self.uploadPublishInfo: self.createAndUploadPublish(wmbsWorkflow) #Load all the associated subscriptions and shoot them one by one subIDs = workflows[workflow]["workflows"][wmbsWorkflow.id] for subID in subIDs: subscription = Subscription(id = subID) subscription['workflow'] = wmbsWorkflow subscription.load() subscription.deleteEverything() #Check that the workflow is gone if wmbsWorkflow.exists(): #Something went bad, this workflow #should be gone by now msg = "Workflow %s, Task %s was not deleted completely" % (wmbsWorkflow.name, wmbsWorkflow.task) raise TaskArchiverPollerException(msg) #Now delete directories _, taskDir = getMasterName(startDir = self.jobCacheDir, workflow = wmbsWorkflow) logging.info("About to delete work directory %s" % taskDir) if os.path.exists(taskDir): if os.path.isdir(taskDir): shutil.rmtree(taskDir) else: # What we think of as a working directory is not a directory # This should never happen and there is no way we can recover # from this here. Bail out now and have someone look at things. msg = "Work directory is not a directory, this should never happen: %s" % taskDir raise TaskArchiverPollerException(msg) else: msg = "Attempted to delete work directory but it was already gone: %s" % taskDir logging.debug(msg) spec = workflows[workflow]["spec"] topTask = spec.getTopLevelTask()[0] # Now take care of the sandbox sandbox = getattr(topTask.data.input, 'sandbox', None) if sandbox: sandboxDir = os.path.dirname(sandbox) if os.path.isdir(sandboxDir): shutil.rmtree(sandboxDir) logging.debug("Sandbox dir deleted") else: logging.error("Attempted to delete sandbox dir but it was already gone: %s" % sandboxDir) except Exception, ex: msg = "Critical error while deleting workflow %s\n" % workflow msg += str(ex) msg += str(traceback.format_exc()) logging.error(msg) self.sendAlert(2, msg = msg)