def parseASOState(self, fp, nodes, statusResult):
""" Parse aso_status and for each job change the job status from 'transferring'
to 'transferred' in case all files in the job have already been successfully
transferred.
fp: file pointer to the ASO status file downloaded from the shedd
nodes: contains the data-structure representing the node state file created by dagman
statusResult: the dictionary it is going to be returned by the status to the client.
we need this to add a warning in case there are jobs missing in the node_state file
"""
transfers = {}
data = json.load(fp)
for docid, result in data['results'].iteritems():
#Oracle has an improved structure in aso_status
if isCouchDBURL(self.asoDBURL):
result = result['value']
else:
result = result[0]
jobid = str(result['jobid'])
if jobid not in nodes:
msg = ("It seems one or more jobs are missing from the node_state file."
" It might be corrupted as a result of a disk failure on the schedd (maybe it is full?)"
" This might be interesting for analysis operation (%s) %" + FEEDBACKMAIL)
statusResult['taskWarningMsg'] = [msg] + statusResult['taskWarningMsg']
if jobid in nodes and nodes[jobid]['State'] == 'transferring':
transfers.setdefault(jobid, {})[docid] = result['state']
return
for jobid in transfers:
## The aso_status file is created/updated by the post-jobs when monitoring the
## transfers, i.e. after all transfer documents for the given job have been
## successfully inserted into the ASO database. Thus, if aso_status contains N
## documents for a given job_id it means there are exactly N files to transfer
## for that job.
if set(transfers[jobid].values()) == set(['done']):
nodes[jobid]['State'] = 'transferred'
def publicationStatus(self, workflow, asourl, asodb, user):
"""Here is what basically the function return, a dict called publicationInfo in the subcalls:
publicationInfo['status']: something like {'publishing': 0, 'publication_failed': 0, 'not_published': 0, 'published': 5}.
Later on goes into dictresult['publication'] before being returned to the client
publicationInfo['status']['error']: String containing the error message if not able to contact couch or oracle
Later on goes into dictresult['publication']['error']
publicationInfo['failure_reasons']: errors of single files (not yet implemented for oracle..)
"""
if isCouchDBURL(asourl):
return self.publicationStatusCouch(workflow, asourl, asodb)
else:
return self.publicationStatusOracle(workflow, user)
def publicationStatusWrapper(self, workflow, asourl, asodb, username, publicationenabled):
publicationInfo = {}
if (publicationenabled == 'T'):
# value is used in the publicationStatus method.
self.isCouchDBURL = isCouchDBURL(asourl)
#let's default asodb to asynctransfer, for old task this is empty!
asodb = asodb or 'asynctransfer'
publicationInfo = self.publicationStatus(workflow, asourl, asodb, username)
self.logger.info("Publication status for workflow %s done", workflow)
else:
publicationInfo['status'] = {'disabled': []}
return publicationInfo
def resubmitPublication(self, asourl, asodb, proxy, taskname):
if isCouchDBURL(asourl):
return self.resubmitCouchPublication(asourl, asodb, proxy, taskname)
else:
return self.resubmitOraclePublication(taskname)
def resubmit(self, workflow, publication, jobids, force, siteblacklist, sitewhitelist, maxjobruntime, maxmemory, numcores, priority, userdn, userproxy):
"""Request to reprocess what the workflow hasn't finished to reprocess.
This needs to create a new workflow in the same campaign
:arg str workflow: a valid workflow name
:arg str list siteblacklist: black list of sites, with CMS name;
:arg str list sitewhitelist: white list of sites, with CMS name;
:arg int whether to resubmit publications or jobs."""
retmsg = "ok"
resubmitWhat = "publications" if publication else "jobs"
self.logger.info("About to resubmit %s for workflow: %s. Getting status first.", resubmitWhat, workflow)
## Get the status of the task/jobs.
statusRes = self.status(workflow, userdn, userproxy)[0]
## Check lifetime of the task and raise ExecutionError if appropriate
self.logger.info("Checking if resubmission is possible: we don't allow resubmission %s days before task expiration date", NUM_DAYS_FOR_RESUBMITDRAIN)
retmsg = checkTaskLifetime(statusRes['submissionTime'])
if retmsg != "ok":
return [{'result': retmsg}]
## Ignore the following options if this is a publication resubmission or if the
## task was never submitted.
if publication or statusRes['status'] == 'SUBMITFAILED':
jobids, force = None, False
siteblacklist, sitewhitelist, maxjobruntime, maxmemory, numcores, priority = None, None, None, None, None, None
## We allow resubmission only if the task status is one of these:
allowedTaskStates = ['SUBMITTED', 'KILLED', 'KILLFAILED', 'RESUBMITFAILED', 'FAILED']
## We allow resubmission of successfully finished jobs if the user explicitly
## gave the job ids and the force option (i.e. the user knows what he/she is
## doing). In that case we have to allow the task to be in COMPLETED status.
## The same is true for publication resubmission.
if (jobids and force) or publication:
allowedTaskStates += ['COMPLETED']
## Allow resubmission of tasks in SUBMITFAILED status if this is not a
## publication resubmission.
if not publication:
allowedTaskStates += ['SUBMITFAILED'] #NB submitfailed goes to NEW, not RESUBMIT
## If the task status is not an allowed one, fail the resubmission.
if statusRes['status'] not in allowedTaskStates:
if statusRes['status'] == 'COMPLETED':
msg = "Task status is COMPLETED."
msg += " To resubmit jobs from a task in status COMPLETED, specify the job ids and use the force option."
msg += " To resubmit publications use the publication option."
else:
msg = "You cannot resubmit %s if the task is in status %s." % (resubmitWhat, statusRes['status'])
raise ExecutionError(msg)
if statusRes['status'] != 'SUBMITFAILED':
## This is the list of job ids that we allow to be resubmitted.
## Note: This list will be empty if statusRes['jobList'] is empty to begin with.
resubmitjobids = []
for jobstatus, jobid in statusRes['jobList']:
if (not publication and jobstatus in self.failedList) or \
(((jobids and force) or publication) and jobstatus in self.successList):
resubmitjobids.append(jobid)
if statusRes['jobList'] and not resubmitjobids:
msg = "There are no %s to resubmit." % (resubmitWhat)
if publication:
msg += " Publications can only be resubmitted for jobs in status %s." % (self.successList)
else:
msg += " Only jobs in status %s can be resubmitted." % (self.failedList)
msg += " Jobs in status %s can also be resubmitted," % (self.successList)
msg += " but only if the jobid is specified and the force option is set."
raise ExecutionError(msg)
## Checks for publication resubmission.
if publication:
if 'publication' not in statusRes or not statusRes['publication']:
msg = "Cannot resubmit publication."
msg += " Unable to retrieve the publication status."
raise ExecutionError(msg)
if 'disabled' in statusRes['publication']:
msg = "Cannot resubmit publication."
msg += " Publication was disabled in the CRAB configuration."
raise ExecutionError(msg)
if 'error' in statusRes['publication']:
msg = "Cannot resubmit publication."
msg += " Error in publication status: %s" % (statusRes['publication']['error'])
raise ExecutionError(msg)
if isCouchDBURL(statusRes['ASOURL']) and statusRes['publication'].get('publication_failed', 0) == 0:
msg = "There are no failed publications to resubmit."
raise ExecutionError(msg)
## Here we can add a check on the publication status of the documents
## corresponding to the job ids in resubmitjobids and jobids. So far the
## publication resubmission will resubmit all the failed publications.
## If the user wants to resubmit a specific set of job ids ...
if jobids:
## ... make the intersection between the "allowed" and "wanted" job ids.
resubmitjobids = list(set(resubmitjobids) & set(jobids))
## Check if all the "wanted" job ids can be resubmitted. If not, fail the resubmission.
if len(resubmitjobids) != len(jobids):
requestedResub = list(set(jobids) - set(resubmitjobids))
msg = "CRAB server refused to resubmit the following jobs: %s." % (str(requestedResub))
msg += " Only jobs in status %s can be resubmitted." % (self.failedList)
msg += " Jobs in status %s can also be resubmitted," % (self.successList)
msg += " but only if the jobid is specified and the force option is set."
raise ExecutionError(msg) #return [{'result': msg}]
if publication:
self.logger.info("Publications to resubmit if failed: %s", resubmitjobids)
else:
self.logger.info("Jobs to resubmit: %s", resubmitjobids)
## If these parameters are not set, give them the same values they had in the
## original task submission.
if (siteblacklist is None) or (sitewhitelist is None) or (maxjobruntime is None) or (maxmemory is None) or (numcores is None) or (priority is None):
## origValues = [orig_siteblacklist, orig_sitewhitelist, orig_maxjobruntime, orig_maxmemory, orig_numcores, orig_priority]
origValues = next(self.api.query(None, None, self.Task.GetResubmitParams_sql, taskname = workflow))
if siteblacklist is None:
siteblacklist = literal_eval(origValues[0])
if sitewhitelist is None:
sitewhitelist = literal_eval(origValues[1])
if maxjobruntime is None:
maxjobruntime = origValues[2]
if maxmemory is None:
maxmemory = origValues[3]
if numcores is None:
numcores = origValues[4]
if priority is None:
priority = origValues[5]
## These are the parameters that we want to writte down in the 'tm_arguments'
## column of the Tasks DB each time a resubmission is done.
## DagmanResubmitter will read these parameters and write them into the task ad.
arguments = {'resubmit_jobids' : resubmitjobids,
'site_blacklist' : siteblacklist,
'site_whitelist' : sitewhitelist,
'maxjobruntime' : maxjobruntime,
'maxmemory' : maxmemory,
'numcores' : numcores,
'priority' : priority,
'resubmit_publication' : publication
}
## Change the 'tm_arguments' column of the Tasks DB for this task to contain the
## above parameters.
self.api.modify(self.Task.SetArgumentsTask_sql, taskname = [workflow], arguments = [str(arguments)])
#TODO states are changed
## Change the status of the task in the Tasks DB to RESUBMIT (or NEW).
if statusRes['status'] == 'SUBMITFAILED':
newstate = ["NEW"]
newcommand = ["SUBMIT"]
else:
newstate = ["NEW"]
newcommand = ["RESUBMIT"]
self.api.modify(self.Task.SetStatusTask_sql, status = newstate, command = newcommand, taskname = [workflow])
return [{'result': retmsg}]
def resubmit2(self, workflow, publication, jobids, siteblacklist, sitewhitelist, maxjobruntime, maxmemory,
numcores, priority, userproxy):
"""Request to reprocess what the workflow hasn't finished to reprocess.
This needs to create a new workflow in the same campaign
"""
retmsg = "ok"
self.logger.info("Getting task ID tuple from DB for task %s", workflow)
row = self.api.query(None, None, self.Task.ID_sql, taskname = workflow)
try:
#just one row is picked up by the previous query
row = self.Task.ID_tuple(*next(row))
except StopIteration:
raise ExecutionError("Impossible to find task %s in the database." % workflow)
submissionTime = getEpochFromDBTime(row.start_time)
self.logger.info("Checking if resubmission is possible: we don't allow resubmission %s days before task expiration date", NUM_DAYS_FOR_RESUBMITDRAIN)
retmsg = checkTaskLifetime(submissionTime)
if retmsg != "ok":
return [{'result': retmsg}]
task_status = row.task_status
task_splitting = row.split_algo
resubmitWhat = "publications" if publication else "jobs"
self.logger.info("About to resubmit %s for workflow: %s.", resubmitWhat, workflow)
## Ignore the following options if this is a publication resubmission or if the
## task was never submitted.
if publication or task_status == 'SUBMITFAILED':
jobids = None
siteblacklist, sitewhitelist, maxjobruntime, maxmemory, numcores, priority = None, None, None, None, None, None
# We only allow resubmission of tasks that are in a final state, listed here:
allowedTaskStates = ['SUBMITTED', 'KILLED', 'KILLFAILED', 'RESUBMITFAILED', 'FAILED']
# Do not resubmit publication for tasks that were not submitted since they don't have any output.
if not publication:
allowedTaskStates += ['SUBMITFAILED'] #NB submitfailed goes to NEW, not RESUBMIT
## If the task status is not an allowed one, fail the resubmission.
if task_status not in allowedTaskStates:
msg = "You cannot resubmit %s if the task is in status %s." % (resubmitWhat, task_status)
raise ExecutionError(msg)
if task_status == 'KILLED' and task_splitting == 'Automatic':
msg = "You cannot resubmit {0} if the task is in status {1} and uses automatic splitting.".format(resubmitWhat, task_status)
raise ExecutionError(msg)
if task_status != 'SUBMITFAILED':
if publication:
## Retrieve publication information.
publicationEnabled = row.publication
asourl = row.asourl
asodb = row.asodb
username = row.username
publicationInfo = self.publicationStatusWrapper(workflow, asourl, asodb, username, publicationEnabled)
if 'status' not in publicationInfo:
msg = "Cannot resubmit publication."
msg += " Unable to retrieve the publication status."
raise ExecutionError(msg)
if 'disabled' in publicationInfo:
msg = "Cannot resubmit publication."
msg += " Publication was disabled in the CRAB configuration."
raise ExecutionError(msg)
if 'error' in publicationInfo:
msg = "Cannot resubmit publication."
msg += " Error in publication status: %s" % (publicationInfo['error'])
raise ExecutionError(msg)
if isCouchDBURL(asourl) and publicationInfo['status'].get('publication_failed', 0) == 0:
msg = "There are no failed publications to resubmit."
raise ExecutionError(msg)
## Here we can add a check on the publication status of the documents
## corresponding to the job ids in resubmitjobids and jobids. So far the
## publication resubmission will resubmit all the failed publications.
self.resubmitPublication(asourl, asodb, userproxy, workflow)
return [{'result': retmsg}]
else:
self.logger.info("Jobs to resubmit: %s", jobids)
## If these parameters are not set, give them the same values they had in the
## original task submission.
if (siteblacklist is None) or (sitewhitelist is None) or (maxjobruntime is None) or (maxmemory is None) or (numcores is None) or (priority is None):
## origValues = [orig_siteblacklist, orig_sitewhitelist, orig_maxjobruntime, orig_maxmemory, orig_numcores, orig_priority]
origValues = next(self.api.query(None, None, self.Task.GetResubmitParams_sql, taskname = workflow))
if siteblacklist is None:
siteblacklist = literal_eval(origValues[0])
if sitewhitelist is None:
sitewhitelist = literal_eval(origValues[1])
if maxjobruntime is None:
maxjobruntime = origValues[2]
if maxmemory is None:
maxmemory = origValues[3]
if numcores is None:
numcores = origValues[4]
if priority is None:
priority = origValues[5]
## These are the parameters that we want to writte down in the 'tm_arguments'
## column of the Tasks DB each time a resubmission is done.
## DagmanResubmitter will read these parameters and write them into the task ad.
arguments = {'resubmit_jobids' : jobids,
'site_blacklist' : siteblacklist,
'site_whitelist' : sitewhitelist,
'maxjobruntime' : maxjobruntime,
'maxmemory' : maxmemory,
'numcores' : numcores,
'priority' : priority,
'resubmit_publication' : publication
}
## Change the 'tm_arguments' column of the Tasks DB for this task to contain the
## above parameters.
self.api.modify(self.Task.SetArgumentsTask_sql, taskname = [workflow], arguments = [str(arguments)])
## Change the status of the task in the Tasks DB to RESUBMIT (or NEW).
if task_status == 'SUBMITFAILED':
newstate = ["NEW"]
newcommand = ["SUBMIT"]
else:
newstate = ["NEW"]
newcommand = ["RESUBMIT"]
self.api.modify(self.Task.SetStatusTask_sql, status = newstate, command = newcommand, taskname = [workflow])
return [{'result': retmsg}]
def resubmitPublication(self, asourl, asodb, proxy, taskname):
if isCouchDBURL(asourl):
return self.resubmitCouchPublication(asourl, asodb, proxy, taskname)
else:
return self.resubmitOraclePublication()
def resubmit(self, workflow, publication, jobids, force, siteblacklist, sitewhitelist, maxjobruntime, maxmemory, numcores, priority, userdn, userproxy):
"""Request to reprocess what the workflow hasn't finished to reprocess.
This needs to create a new workflow in the same campaign
:arg str workflow: a valid workflow name
:arg str list siteblacklist: black list of sites, with CMS name;
:arg str list sitewhitelist: white list of sites, with CMS name;
:arg int whether to resubmit publications or jobs."""
retmsg = "ok"
resubmitWhat = "publications" if publication else "jobs"
self.logger.info("About to resubmit %s for workflow: %s. Getting status first." % (resubmitWhat, workflow))
## Get the status of the task/jobs.
statusRes = self.status(workflow, userdn, userproxy)[0]
## Check lifetime of the task and raise ExecutionError if appropriate
self.logger.info("Checking if resubmission is possible: we don't allow resubmission %s days before task expiration date", NUM_DAYS_FOR_RESUBMITDRAIN)
retmsg = checkTaskLifetime(statusRes['submissionTime'])
if retmsg != "ok":
return [{'result': retmsg}]
## Ignore the following options if this is a publication resubmission or if the
## task was never submitted.
if publication or statusRes['status'] == 'SUBMITFAILED':
jobids, force = None, False
siteblacklist, sitewhitelist, maxjobruntime, maxmemory, numcores, priority = None, None, None, None, None, None
## We allow resubmission only if the task status is one of these:
allowedTaskStates = ['SUBMITTED', 'KILLED', 'KILLFAILED', 'RESUBMITFAILED', 'FAILED']
## We allow resubmission of successfully finished jobs if the user explicitly
## gave the job ids and the force option (i.e. the user knows what he/she is
## doing). In that case we have to allow the task to be in COMPLETED status.
## The same is true for publication resubmission.
if (jobids and force) or publication:
allowedTaskStates += ['COMPLETED']
## Allow resubmission of tasks in SUBMITFAILED status if this is not a
## publication resubmission.
if not publication:
allowedTaskStates += ['SUBMITFAILED'] #NB submitfailed goes to NEW, not RESUBMIT
## If the task status is not an allowed one, fail the resubmission.
if statusRes['status'] not in allowedTaskStates:
if statusRes['status'] == 'COMPLETED':
msg = "Task status is COMPLETED."
msg += " To resubmit jobs from a task in status COMPLETED, specify the job ids and use the force option."
msg += " To resubmit publications use the publication option."
else:
msg = "You cannot resubmit %s if the task is in status %s." % (resubmitWhat, statusRes['status'])
raise ExecutionError(msg)
if statusRes['status'] != 'SUBMITFAILED':
## This is the list of job ids that we allow to be resubmitted.
## Note: This list will be empty if statusRes['jobList'] is empty to begin with.
resubmitjobids = []
for jobstatus, jobid in statusRes['jobList']:
if (not publication and jobstatus in self.failedList) or \
(((jobids and force) or publication) and jobstatus in self.successList):
resubmitjobids.append(jobid)
if statusRes['jobList'] and not resubmitjobids:
msg = "There are no %s to resubmit." % (resubmitWhat)
if publication:
msg += " Publications can only be resubmitted for jobs in status %s." % (self.successList)
else:
msg += " Only jobs in status %s can be resubmitted." % (self.failedList)
msg += " Jobs in status %s can also be resubmitted," % (self.successList)
msg += " but only if the jobid is specified and the force option is set."
raise ExecutionError(msg)
## Checks for publication resubmission.
if publication:
if 'publication' not in statusRes or not statusRes['publication']:
msg = "Cannot resubmit publication."
msg += " Unable to retrieve the publication status."
raise ExecutionError(msg)
if 'disabled' in statusRes['publication']:
msg = "Cannot resubmit publication."
msg += " Publication was disabled in the CRAB configuration."
raise ExecutionError(msg)
if 'error' in statusRes['publication']:
msg = "Cannot resubmit publication."
msg += " Error in publication status: %s" % (statusRes['publication']['error'])
raise ExecutionError(msg)
if isCouchDBURL(statusRes['ASOURL']) and statusRes['publication'].get('publication_failed', 0) == 0:
msg = "There are no failed publications to resubmit."
raise ExecutionError(msg)
## Here we can add a check on the publication status of the documents
## corresponding to the job ids in resubmitjobids and jobids. So far the
## publication resubmission will resubmit all the failed publications.
## If the user wants to resubmit a specific set of job ids ...
if jobids:
## ... make the intersection between the "allowed" and "wanted" job ids.
resubmitjobids = list(set(resubmitjobids) & set(jobids))
## Check if all the "wanted" job ids can be resubmitted. If not, fail the resubmission.
if len(resubmitjobids) != len(jobids):
requestedResub = list(set(jobids) - set(resubmitjobids))
msg = "CRAB server refused to resubmit the following jobs: %s." % (str(requestedResub))
msg += " Only jobs in status %s can be resubmitted." % (self.failedList)
msg += " Jobs in status %s can also be resubmitted," % (self.successList)
msg += " but only if the jobid is specified and the force option is set."
raise ExecutionError(msg) #return [{'result': msg}]
if publication:
self.logger.info("Publications to resubmit if failed: %s" % (resubmitjobids))
else:
self.logger.info("Jobs to resubmit: %s" % (resubmitjobids))
## If these parameters are not set, give them the same values they had in the
## original task submission.
if (siteblacklist is None) or (sitewhitelist is None) or (maxjobruntime is None) or (maxmemory is None) or (numcores is None) or (priority is None):
## origValues = [orig_siteblacklist, orig_sitewhitelist, orig_maxjobruntime, orig_maxmemory, orig_numcores, orig_priority]
origValues = next(self.api.query(None, None, self.Task.GetResubmitParams_sql, taskname = workflow))
if siteblacklist is None:
siteblacklist = literal_eval(origValues[0])
if sitewhitelist is None:
sitewhitelist = literal_eval(origValues[1])
if maxjobruntime is None:
maxjobruntime = origValues[2]
if maxmemory is None:
maxmemory = origValues[3]
if numcores is None:
numcores = origValues[4]
if priority is None:
priority = origValues[5]
## These are the parameters that we want to writte down in the 'tm_arguments'
## column of the Tasks DB each time a resubmission is done.
## DagmanResubmitter will read these parameters and write them into the task ad.
arguments = {'resubmit_jobids' : resubmitjobids,
'site_blacklist' : siteblacklist,
'site_whitelist' : sitewhitelist,
'maxjobruntime' : maxjobruntime,
'maxmemory' : maxmemory,
'numcores' : numcores,
'priority' : priority,
'resubmit_publication' : publication
}
## Change the 'tm_arguments' column of the Tasks DB for this task to contain the
## above parameters.
self.api.modify(self.Task.SetArgumentsTask_sql, taskname = [workflow], arguments = [str(arguments)])
#TODO states are changed
## Change the status of the task in the Tasks DB to RESUBMIT (or NEW).
if statusRes['status'] == 'SUBMITFAILED':
newstate = ["NEW"]
newcommand = ["SUBMIT"]
else:
newstate = ["NEW"]
newcommand = ["RESUBMIT"]
self.api.modify(self.Task.SetStatusTask_sql, status = newstate, command = newcommand, taskname = [workflow])
return [{'result': retmsg}]
def resubmit2(self, workflow, publication, jobids, siteblacklist, sitewhitelist, maxjobruntime, maxmemory,
numcores, priority, userproxy):
"""Request to reprocess what the workflow hasn't finished to reprocess.
This needs to create a new workflow in the same campaign
"""
retmsg = "ok"
self.logger.info("Getting task ID tuple from DB for task %s" % workflow)
row = self.api.query(None, None, self.Task.ID_sql, taskname = workflow)
try:
#just one row is picked up by the previous query
row = self.Task.ID_tuple(*next(row))
except StopIteration:
raise ExecutionError("Impossible to find task %s in the database." % workflow)
submissionTime = getEpochFromDBTime(row.start_time)
self.logger.info("Checking if resubmission is possible: we don't allow resubmission %s days before task expiration date", NUM_DAYS_FOR_RESUBMITDRAIN)
retmsg = checkTaskLifetime(submissionTime)
if retmsg != "ok":
return [{'result': retmsg}]
task_status = row.task_status
resubmitWhat = "publications" if publication else "jobs"
self.logger.info("About to resubmit %s for workflow: %s." % (resubmitWhat, workflow))
## Ignore the following options if this is a publication resubmission or if the
## task was never submitted.
if publication or task_status == 'SUBMITFAILED':
jobids = None
siteblacklist, sitewhitelist, maxjobruntime, maxmemory, numcores, priority = None, None, None, None, None, None
# We only allow resubmission of tasks that are in a final state, listed here:
allowedTaskStates = ['SUBMITTED', 'KILLED', 'KILLFAILED', 'RESUBMITFAILED', 'FAILED']
# Do not resubmit publication for tasks that were not submitted since they don't have any output.
if not publication:
allowedTaskStates += ['SUBMITFAILED'] #NB submitfailed goes to NEW, not RESUBMIT
## If the task status is not an allowed one, fail the resubmission.
if task_status not in allowedTaskStates:
msg = "You cannot resubmit %s if the task is in status %s." % (resubmitWhat, task_status)
raise ExecutionError(msg)
if task_status != 'SUBMITFAILED':
if publication:
## Retrieve publication information.
publicationEnabled = row.publication
asourl = row.asourl
asodb = row.asodb
username = row.username
publicationInfo = self.publicationStatusWrapper(workflow, asourl, asodb, username, publicationEnabled)
if 'status' not in publicationInfo:
msg = "Cannot resubmit publication."
msg += " Unable to retrieve the publication status."
raise ExecutionError(msg)
if 'disabled' in publicationInfo:
msg = "Cannot resubmit publication."
msg += " Publication was disabled in the CRAB configuration."
raise ExecutionError(msg)
if 'error' in publicationInfo:
msg = "Cannot resubmit publication."
msg += " Error in publication status: %s" % (publicationInfo['error'])
raise ExecutionError(msg)
if isCouchDBURL(asourl) and publicationInfo['status'].get('publication_failed', 0) == 0:
msg = "There are no failed publications to resubmit."
raise ExecutionError(msg)
## Here we can add a check on the publication status of the documents
## corresponding to the job ids in resubmitjobids and jobids. So far the
## publication resubmission will resubmit all the failed publications.
self.resubmitPublication(asourl, asodb, userproxy, workflow)
return [{'result': retmsg}]
else:
self.logger.info("Jobs to resubmit: %s" % (jobids))
## If these parameters are not set, give them the same values they had in the
## original task submission.
if (siteblacklist is None) or (sitewhitelist is None) or (maxjobruntime is None) or (maxmemory is None) or (numcores is None) or (priority is None):
## origValues = [orig_siteblacklist, orig_sitewhitelist, orig_maxjobruntime, orig_maxmemory, orig_numcores, orig_priority]
origValues = next(self.api.query(None, None, self.Task.GetResubmitParams_sql, taskname = workflow))
if siteblacklist is None:
siteblacklist = literal_eval(origValues[0])
if sitewhitelist is None:
sitewhitelist = literal_eval(origValues[1])
if maxjobruntime is None:
maxjobruntime = origValues[2]
if maxmemory is None:
maxmemory = origValues[3]
if numcores is None:
numcores = origValues[4]
if priority is None:
priority = origValues[5]
## These are the parameters that we want to writte down in the 'tm_arguments'
## column of the Tasks DB each time a resubmission is done.
## DagmanResubmitter will read these parameters and write them into the task ad.
arguments = {'resubmit_jobids' : jobids,
'site_blacklist' : siteblacklist,
'site_whitelist' : sitewhitelist,
'maxjobruntime' : maxjobruntime,
'maxmemory' : maxmemory,
'numcores' : numcores,
'priority' : priority,
'resubmit_publication' : publication
}
## Change the 'tm_arguments' column of the Tasks DB for this task to contain the
## above parameters.
self.api.modify(self.Task.SetArgumentsTask_sql, taskname = [workflow], arguments = [str(arguments)])
## Change the status of the task in the Tasks DB to RESUBMIT (or NEW).
if task_status == 'SUBMITFAILED':
newstate = ["NEW"]
newcommand = ["SUBMIT"]
else:
newstate = ["NEW"]
newcommand = ["RESUBMIT"]
self.api.modify(self.Task.SetStatusTask_sql, status = newstate, command = newcommand, taskname = [workflow])
return [{'result': retmsg}]
