def createAndUploadPublish(self, workflow):
"""
Upload to the UFC a JSON file with all the info needed to publish this dataset later
"""
if self.uploadPublishDir:
workDir = self.uploadPublishDir
else:
workDir, taskDir = getMasterName(startDir=self.jobCacheDir, workflow=workflow)
try:
return uploadPublishWorkflow(self.config, workflow, ufcEndpoint=self.userFileCacheURL, workDir=workDir)
except Exception, ex:
logging.error('Upload failed for workflow: %s' % (workflow))
logging.exception(ex) #Let's print the stacktrace with generic Exception
return False
def createAndUploadPublish(self, workflow):
"""
Upload to the UFC a JSON file with all the info needed to publish this dataset later
"""
if self.uploadPublishDir:
workDir = self.uploadPublishDir
else:
workDir, taskDir = getMasterName(startDir=self.jobCacheDir,
workflow=workflow)
try:
return uploadPublishWorkflow(self.config,
workflow,
ufcEndpoint=self.userFileCacheURL,
workDir=workDir)
except Exception, ex:
logging.error('Upload failed for workflow: %s' % (workflow))
logging.exception(
ex) #Let's print the stacktrace with generic Exception
return False
def killWorkflows(self, workflows):
"""
_killWorkflows_
Delete all the information in couch and WMBS about the given
workflow, go through all subscriptions and delete one by
one.
The input is a dictionary with workflow names as keys, fully loaded WMWorkloads and
subscriptions lists as values
"""
for workflow in workflows:
logging.info("Deleting workflow %s" % workflow)
try:
#Get the task-workflow ids, sort them by ID,
#higher ID first so we kill
#the leaves of the tree first, root last
workflowsIDs = workflows[workflow]["workflows"].keys()
workflowsIDs.sort(reverse=True)
#Now go through all tasks and load the WMBS workflow objects
wmbsWorkflows = []
for wfID in workflowsIDs:
wmbsWorkflow = Workflow(id=wfID)
wmbsWorkflow.load()
wmbsWorkflows.append(wmbsWorkflow)
#Time to shoot one by one
for wmbsWorkflow in wmbsWorkflows:
if self.uploadPublishInfo:
self.createAndUploadPublish(wmbsWorkflow)
#Load all the associated subscriptions and shoot them one by one
subIDs = workflows[workflow]["workflows"][wmbsWorkflow.id]
for subID in subIDs:
subscription = Subscription(id=subID)
subscription['workflow'] = wmbsWorkflow
subscription.load()
subscription.deleteEverything()
#Check that the workflow is gone
if wmbsWorkflow.exists():
#Something went bad, this workflow
#should be gone by now
msg = "Workflow %s, Task %s was not deleted completely" % (
wmbsWorkflow.name, wmbsWorkflow.task)
raise TaskArchiverPollerException(msg)
#Now delete directories
_, taskDir = getMasterName(startDir=self.jobCacheDir,
workflow=wmbsWorkflow)
logging.info("About to delete work directory %s" % taskDir)
if os.path.exists(taskDir):
if os.path.isdir(taskDir):
shutil.rmtree(taskDir)
else:
# What we think of as a working directory is not a directory
# This should never happen and there is no way we can recover
# from this here. Bail out now and have someone look at things.
msg = "Work directory is not a directory, this should never happen: %s" % taskDir
raise TaskArchiverPollerException(msg)
else:
msg = "Attempted to delete work directory but it was already gone: %s" % taskDir
logging.debug(msg)
spec = workflows[workflow]["spec"]
topTask = spec.getTopLevelTask()[0]
# Now take care of the sandbox
sandbox = getattr(topTask.data.input, 'sandbox', None)
if sandbox:
sandboxDir = os.path.dirname(sandbox)
if os.path.isdir(sandboxDir):
shutil.rmtree(sandboxDir)
logging.debug("Sandbox dir deleted")
else:
logging.error(
"Attempted to delete sandbox dir but it was already gone: %s"
% sandboxDir)
except Exception, ex:
msg = "Critical error while deleting workflow %s\n" % workflow
msg += str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
self.sendAlert(2, msg=msg)
def killWorkflows(self, workflows):
"""
_killWorkflows_
Delete all the information in couch and WMBS about the given
workflow, go through all subscriptions and delete one by
one.
The input is a dictionary with workflow names as keys, fully loaded WMWorkloads and
subscriptions lists as values
"""
for workflow in workflows:
logging.info("Deleting workflow %s" % workflow)
try:
#Get the task-workflow ids, sort them by ID,
#higher ID first so we kill
#the leaves of the tree first, root last
workflowsIDs = workflows[workflow]["workflows"].keys()
workflowsIDs.sort(reverse = True)
#Now go through all tasks and load the WMBS workflow objects
wmbsWorkflows = []
for wfID in workflowsIDs:
wmbsWorkflow = Workflow(id = wfID)
wmbsWorkflow.load()
wmbsWorkflows.append(wmbsWorkflow)
#Time to shoot one by one
for wmbsWorkflow in wmbsWorkflows:
if self.uploadPublishInfo:
self.createAndUploadPublish(wmbsWorkflow)
#Load all the associated subscriptions and shoot them one by one
subIDs = workflows[workflow]["workflows"][wmbsWorkflow.id]
for subID in subIDs:
subscription = Subscription(id = subID)
subscription['workflow'] = wmbsWorkflow
subscription.load()
subscription.deleteEverything()
#Check that the workflow is gone
if wmbsWorkflow.exists():
#Something went bad, this workflow
#should be gone by now
msg = "Workflow %s, Task %s was not deleted completely" % (wmbsWorkflow.name,
wmbsWorkflow.task)
raise TaskArchiverPollerException(msg)
#Now delete directories
_, taskDir = getMasterName(startDir = self.jobCacheDir,
workflow = wmbsWorkflow)
logging.info("About to delete work directory %s" % taskDir)
if os.path.exists(taskDir):
if os.path.isdir(taskDir):
shutil.rmtree(taskDir)
else:
# What we think of as a working directory is not a directory
# This should never happen and there is no way we can recover
# from this here. Bail out now and have someone look at things.
msg = "Work directory is not a directory, this should never happen: %s" % taskDir
raise TaskArchiverPollerException(msg)
else:
msg = "Attempted to delete work directory but it was already gone: %s" % taskDir
logging.debug(msg)
spec = workflows[workflow]["spec"]
topTask = spec.getTopLevelTask()[0]
# Now take care of the sandbox
sandbox = getattr(topTask.data.input, 'sandbox', None)
if sandbox:
sandboxDir = os.path.dirname(sandbox)
if os.path.isdir(sandboxDir):
shutil.rmtree(sandboxDir)
logging.debug("Sandbox dir deleted")
else:
logging.error("Attempted to delete sandbox dir but it was already gone: %s" % sandboxDir)
except Exception, ex:
msg = "Critical error while deleting workflow %s\n" % workflow
msg += str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
self.sendAlert(2, msg = msg)
def killSubscriptions(self, doneList):
"""
_killSubscriptions_
Actually dump the subscriptions
"""
for sub in doneList:
logging.info("Deleting subscription %i" % sub['id'])
try:
sub.load()
sub['workflow'].load()
wf = sub['workflow']
if self.uploadPublishInfo:
self.createAndUploadPublish(wf)
sub.deleteEverything()
workflow = sub['workflow']
if workflow.exists():
# Then there are other subscriptions attached
# to the workflow
continue
# If we deleted the workflow, it's time to delete
# the work directories
# Now we have to delete the task area.
workDir, taskDir = getMasterName(startDir = self.jobCacheDir,
workflow = workflow)
logging.info("About to delete work directory %s" % taskDir)
if os.path.isdir(taskDir):
# Remove the taskDir, because we're done
shutil.rmtree(taskDir)
else:
msg = "Attempted to delete work directory but it was already gone: %s" % taskDir
logging.error(msg)
self.sendAlert(1, msg = msg)
# Now check if the workflow is done
if not workflow.countWorkflowsBySpec() == 0:
continue
# If the WMSpec is done, then we have to delete
# the sandbox, and send off the couch summary
# First load the WMSpec
try:
logging.debug("Loading spec to delete sandbox dir for task %s" % workflow.task)
spec = retrieveWMSpec(workflow = workflow)
wmTask = spec.getTaskByPath(workflow.task)
except Exception, ex:
# If this happens, we're well and truly screwed.
# We've passed the deletion point. We can't recover
# Abort this. There will be no couch summary
msg = "Critical error in opening spec after workflow deletion"
msg += "Task: %s" % workflow.task
msg += str(ex)
msg += "There will be NO workflow summary for this task"
raise TaskArchiverPollerException(msg)
# Then pull its info from couch and archive it
self.archiveCouchSummary(workflow = workflow, spec = spec)
self.deleteWorkflowFromCouch(workflowName = workflow.task.split('/')[1])
# Now take care of the sandbox
sandbox = getattr(wmTask.data.input, 'sandbox', None)
if sandbox:
sandboxDir = os.path.dirname(sandbox)
if os.path.isdir(sandboxDir):
shutil.rmtree(sandboxDir)
logging.debug("Sandbox dir deleted")
else:
logging.error("Attempted to delete sandbox dir but it was already gone: %s" % sandboxDir)
except Exception, ex:
msg = "Critical error while deleting subscription %i\n" % sub['id']
msg += str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
self.sendAlert(2, msg = msg)