def submit(name,
job_group,
task_id,
input_sandbox,
output_sandbox,
executable,
site=None,
banned_site=None,
sub_ids=[]):
dirac = Dirac()
submit_result = {'backend_job_ids': {}}
jobInfos = {}
for run in range(int((len(sub_ids) + 99) / 100)):
ids_this_run = [x for x in sub_ids[run * 100:(run + 1) * 100]]
job_names = ['%s.%s' % (name, sub_id) for sub_id in ids_this_run]
j = Job()
j.setName(name)
j.setExecutable(executable)
j.setParameterSequence('JobName', job_names, addToWorkflow=True)
j.setParameterSequence('arguments', ids_this_run, addToWorkflow=True)
if input_sandbox:
j.setInputSandbox(input_sandbox)
if output_sandbox:
j.setOutputSandbox(output_sandbox)
if job_group:
j.setJobGroup(job_group)
if site: # set destination to a certain site; list not allowed
j.setDestination(site)
if banned_site:
j.setBannedSites(banned_site)
result = dirac.submitJob(j)
if not result['OK']:
sys.stdout.write('DIRAC job submit error: %s\n' %
result['Message'])
sys.exit(1)
for sub_id, dirac_id in zip(ids_this_run, result['Value']):
submit_result['backend_job_ids'][sub_id] = dirac_id
jobInfos[dirac_id] = {'SubID': sub_id}
#Register on Task-manager Webapp of IHEPDIRAC
task = RPCClient('WorkloadManagement/TaskManager')
taskInfo = {'TaskName': name, 'JobGroup': job_group, 'JSUB-ID': task_id}
task_result = task.createTask(name, taskInfo, jobInfos)
task_web_id = task_result['Value']
submit_result['backend_task_id'] = task_web_id
return submit_result
class DiracTask:
def __init__(self):
self.__task = RPCClient('WorkloadManagement/TaskManager')
self.reset()
def reset(self):
self.__taskID = 0
self.__taskName = 'UNKNOWN'
self.__taskInfo = {}
self.__jobInfos = {}
self.__jobInfoList = []
self.__jobGroup = 'unknown'
def updateTaskInfo(self, taskInfo):
self.__taskInfo.update(taskInfo)
def appendJobInfo(self, jobInfo):
self.__jobInfoList.append(jobInfo)
def setTaskName(self, taskName):
self.__taskName = taskName
def setJobGroup(self, jobGroup):
self.__jobGroup = jobGroup
def addTaskJob(self, jobID, subID):
self.__jobInfos[jobID] = self.__jobInfoList[subID]
if subID + 1 == len(self.__jobInfoList):
self.__createTask()
def __createTask(self):
result = self.__task.createTask(self.__taskName, self.__taskInfo, self.__jobInfos)
if not result['OK']:
logger.warning('Create task failed: %s' % result['Message'])
return
self.__taskID = result['Value']
print('')
print('The DIRAC task ID of the submitted jobs is : %s' % self.__taskID)
print('The DIRAC task name of the submitted jobs is : %s' % self.__taskName)
print('')
def getTaskID(self):
return self.__taskID
def getTaskName(self):
return self.__taskName
def getAllJobGroups(self):
allJobGroups = []
rpcClient = RPCClient( "WorkloadManagement/JobMonitoring" )
result = rpcClient.getProductionIds()
if result['OK']:
allJobGroups += result['Value']
rpcClient = RPCClient( "Accounting/ReportGenerator" )
result = rpcClient.listUniqueKeyValues( 'Job' )
if result['OK']:
allJobGroups += result['Value']['JobGroup']
return allJobGroups
def getJobGroup(self):
return self.__jobGroup
class TaskClient( object ):
"""
The task manager client
"""
def __init__( self ):
self.__taskManager = RPCClient('WorkloadManagement/TaskManager')
self.__jobManager = RPCClient('WorkloadManagement/JobManager')
self.__jobMonitor = RPCClient('WorkloadManagement/JobMonitoring')
def createTask( self, taskName, taskInfo, jobInfos ):
return self.__taskManager.createTask( taskName, taskInfo, jobInfos )
def getTaskCount( self, condDict ):
return self.__taskManager.getTaskCount( condDict )
def listTask( self, condDict, limit, offset, orderAttribute ):
return self.__taskManager.getTasks( condDict, limit, offset, orderAttribute, 1 )
def renameTask( self, taskID, newName ):
return self.__taskManager.renameTask( taskID, newName )
def getTaskOwners( self ):
return self.__taskManager.getTaskOwners()
def getTaskOwnerGroups( self ):
return self.__taskManager.getTaskOwnerGroups()
def getTask( self, taskID ):
return self.__taskManager.getTask( taskID, 1 )
def getTaskProgress( self, taskID ):
return self.__taskManager.getTaskProgress( taskID )
def getTaskInfo( self, taskID ):
return self.__taskManager.getTaskInfo( taskID )
def getTaskJobs( self, taskID ):
return self.__taskManager.getTaskJobs( taskID )
def getTaskJobsStatistics( self, taskID, statusType ):
return self.__taskManager.getTaskJobsStatistics( taskID, statusType )
def getTaskHistories( self, taskID ):
return self.__taskManager.getTaskHistories( taskID )
def getJobs( self, jobIDs, outFields ):
return self.__taskManager.getJobs( jobIDs, outFields )
def rescheduleTask( self, taskID, jobStatus=[] ):
return self.__manageTask( taskID, self.__jobManager.rescheduleJob, jobStatus )
def deleteTask( self, taskID, jobStatus=[] ):
result = self.__manageTask( taskID, self.__jobManager.deleteJob, jobStatus )
if not result['OK']:
return result
jobIDs = result['Value']['JobID']
result = self.__taskManager.removeTask( taskID )
if not result['OK']:
return S_ERROR( 'Remove task error: %s' % result['Message'] )
return S_OK( {'TaskID': taskID, 'JobID': jobIDs} )
def __manageTask( self, taskID, action, jobStatus=[] ):
result = self.__taskManager.getTaskJobs( taskID )
if not result['OK']:
return S_ERROR( 'Get task jobs error: %s' % result['Message'] )
jobIDs = result['Value']
if jobStatus:
result = self.__jobMonitor.getJobs( { 'JobID': jobIDs, 'Status': jobStatus } )
if not result['OK']:
return S_ERROR( 'Get jobs of status %s error: %s' % (status, result['Message']) )
jobIDs = result['Value']
if jobIDs:
result = action( jobIDs )
if not result['OK']:
return S_ERROR( 'Manage jobs error (%s): %s' % (action.__name__, result['Message']) )
return S_OK( {'TaskID': taskID, 'JobID': jobIDs} )
class TaskClient(object):
"""
The task manager client
"""
def __init__(self):
self.__taskManager = RPCClient('WorkloadManagement/TaskManager')
self.__jobManager = RPCClient('WorkloadManagement/JobManager')
self.__jobMonitor = RPCClient('WorkloadManagement/JobMonitoring')
def createTask(self, taskName, taskInfo, jobInfos):
return self.__taskManager.createTask(taskName, taskInfo, jobInfos)
def getTaskCount(self, condDict):
return self.__taskManager.getTaskCount(condDict)
def listTask(self, condDict, limit, offset, orderAttribute):
return self.__taskManager.getTasks(condDict, limit, offset,
orderAttribute, 1)
def renameTask(self, taskID, newName):
return self.__taskManager.renameTask(taskID, newName)
def getTaskOwners(self):
return self.__taskManager.getTaskOwners()
def getTaskOwnerGroups(self):
return self.__taskManager.getTaskOwnerGroups()
def getTask(self, taskID):
return self.__taskManager.getTask(taskID, 1)
def getTaskProgress(self, taskID):
return self.__taskManager.getTaskProgress(taskID)
def getTaskInfo(self, taskID):
return self.__taskManager.getTaskInfo(taskID)
def getTaskJobs(self, taskID):
return self.__taskManager.getTaskJobs(taskID)
def getTaskJobsStatistics(self, taskID, statusType):
return self.__taskManager.getTaskJobsStatistics(taskID, statusType)
def getTaskHistories(self, taskID):
return self.__taskManager.getTaskHistories(taskID)
def getJobs(self, jobIDs, outFields):
return self.__taskManager.getJobs(jobIDs, outFields)
def rescheduleTask(self, taskID, jobStatus=[]):
return self.__manageTask(taskID, self.__jobManager.rescheduleJob,
jobStatus)
def deleteTask(self, taskID, jobStatus=[]):
result = self.__manageTask(taskID, self.__jobManager.deleteJob,
jobStatus)
if not result['OK']:
return result
jobIDs = result['Value']['JobID']
result = self.__taskManager.removeTask(taskID)
if not result['OK']:
return S_ERROR('Remove task error: %s' % result['Message'])
return S_OK({'TaskID': taskID, 'JobID': jobIDs})
def __manageTask(self, taskID, action, jobStatus=[]):
result = self.__taskManager.getTaskJobs(taskID)
if not result['OK']:
return S_ERROR('Get task jobs error: %s' % result['Message'])
jobIDs = result['Value']
if jobStatus:
result = self.__jobMonitor.getJobs({
'JobID': jobIDs,
'Status': jobStatus
})
if not result['OK']:
return S_ERROR('Get jobs of status %s error: %s' %
(status, result['Message']))
jobIDs = result['Value']
if jobIDs:
result = action(jobIDs)
if not result['OK']:
return S_ERROR('Manage jobs error (%s): %s' %
(action.__name__, result['Message']))
return S_OK({'TaskID': taskID, 'JobID': jobIDs})