Esempio n. 1
0
def submit(name,
           job_group,
           task_id,
           input_sandbox,
           output_sandbox,
           executable,
           site=None,
           banned_site=None,
           sub_ids=[]):
    dirac = Dirac()

    submit_result = {'backend_job_ids': {}}
    jobInfos = {}

    for run in range(int((len(sub_ids) + 99) / 100)):
        ids_this_run = [x for x in sub_ids[run * 100:(run + 1) * 100]]
        job_names = ['%s.%s' % (name, sub_id) for sub_id in ids_this_run]
        j = Job()
        j.setName(name)
        j.setExecutable(executable)

        j.setParameterSequence('JobName', job_names, addToWorkflow=True)
        j.setParameterSequence('arguments', ids_this_run, addToWorkflow=True)

        if input_sandbox:
            j.setInputSandbox(input_sandbox)
        if output_sandbox:
            j.setOutputSandbox(output_sandbox)

        if job_group:
            j.setJobGroup(job_group)
        if site:  # set destination to a certain site; list not allowed
            j.setDestination(site)

        if banned_site:
            j.setBannedSites(banned_site)

        result = dirac.submitJob(j)

        if not result['OK']:
            sys.stdout.write('DIRAC job submit error: %s\n' %
                             result['Message'])
            sys.exit(1)

        for sub_id, dirac_id in zip(ids_this_run, result['Value']):
            submit_result['backend_job_ids'][sub_id] = dirac_id
            jobInfos[dirac_id] = {'SubID': sub_id}

    #Register on Task-manager Webapp of IHEPDIRAC
    task = RPCClient('WorkloadManagement/TaskManager')
    taskInfo = {'TaskName': name, 'JobGroup': job_group, 'JSUB-ID': task_id}
    task_result = task.createTask(name, taskInfo, jobInfos)
    task_web_id = task_result['Value']
    submit_result['backend_task_id'] = task_web_id

    return submit_result
Esempio n. 2
0
class DiracTask:

    def __init__(self):
        self.__task = RPCClient('WorkloadManagement/TaskManager')
        self.reset()

    def reset(self):
        self.__taskID = 0
        self.__taskName = 'UNKNOWN'
        self.__taskInfo = {}
        self.__jobInfos = {}
        self.__jobInfoList = []
        self.__jobGroup = 'unknown'

    def updateTaskInfo(self, taskInfo):
        self.__taskInfo.update(taskInfo)

    def appendJobInfo(self, jobInfo):
        self.__jobInfoList.append(jobInfo)

    def setTaskName(self, taskName):
        self.__taskName = taskName

    def setJobGroup(self, jobGroup):
        self.__jobGroup = jobGroup

    def addTaskJob(self, jobID, subID):
        self.__jobInfos[jobID] = self.__jobInfoList[subID]
        if subID + 1 == len(self.__jobInfoList):
            self.__createTask()

    def __createTask(self):
        result = self.__task.createTask(self.__taskName, self.__taskInfo, self.__jobInfos)
        if not result['OK']:
            logger.warning('Create task failed: %s' % result['Message'])
            return
        self.__taskID = result['Value']
        print('')
        print('The DIRAC task ID   of the submitted jobs is : %s' % self.__taskID)
        print('The DIRAC task name of the submitted jobs is : %s' % self.__taskName)
        print('')

    def getTaskID(self):
        return self.__taskID

    def getTaskName(self):
        return self.__taskName

    def getAllJobGroups(self):
        allJobGroups = []

        rpcClient = RPCClient( "WorkloadManagement/JobMonitoring" )
        result = rpcClient.getProductionIds()
        if result['OK']:
            allJobGroups += result['Value']

        rpcClient = RPCClient( "Accounting/ReportGenerator" )
        result = rpcClient.listUniqueKeyValues( 'Job' )
        if result['OK']:
            allJobGroups += result['Value']['JobGroup']

        return allJobGroups

    def getJobGroup(self):
        return self.__jobGroup
Esempio n. 3
0
class TaskClient( object ):
  """
      The task manager client
  """

  def __init__( self ):
    self.__taskManager = RPCClient('WorkloadManagement/TaskManager')
    self.__jobManager = RPCClient('WorkloadManagement/JobManager')
    self.__jobMonitor = RPCClient('WorkloadManagement/JobMonitoring')


  def createTask( self, taskName, taskInfo, jobInfos ):
    return self.__taskManager.createTask( taskName, taskInfo, jobInfos )


  def getTaskCount( self, condDict ):
    return self.__taskManager.getTaskCount( condDict )

  def listTask( self, condDict, limit, offset, orderAttribute ):
    return self.__taskManager.getTasks( condDict, limit, offset, orderAttribute, 1 )


  def renameTask( self, taskID, newName ):
    return self.__taskManager.renameTask( taskID, newName )

  def getTaskOwners( self ):
    return self.__taskManager.getTaskOwners()

  def getTaskOwnerGroups( self ):
    return self.__taskManager.getTaskOwnerGroups()

  def getTask( self, taskID ):
    return self.__taskManager.getTask( taskID, 1 )

  def getTaskProgress( self, taskID ):
    return self.__taskManager.getTaskProgress( taskID )

  def getTaskInfo( self, taskID ):
    return self.__taskManager.getTaskInfo( taskID )

  def getTaskJobs( self, taskID ):
    return self.__taskManager.getTaskJobs( taskID )

  def getTaskJobsStatistics( self, taskID, statusType ):
    return self.__taskManager.getTaskJobsStatistics( taskID, statusType )

  def getTaskHistories( self, taskID ):
    return self.__taskManager.getTaskHistories( taskID )


  def getJobs( self, jobIDs, outFields ):
    return self.__taskManager.getJobs( jobIDs, outFields )


  def rescheduleTask( self, taskID, jobStatus=[] ):
    return self.__manageTask( taskID, self.__jobManager.rescheduleJob, jobStatus )


  def deleteTask( self, taskID, jobStatus=[] ):
    result = self.__manageTask( taskID, self.__jobManager.deleteJob, jobStatus )
    if not result['OK']:
      return result
    jobIDs = result['Value']['JobID']

    result = self.__taskManager.removeTask( taskID )
    if not result['OK']:
      return S_ERROR( 'Remove task error: %s' % result['Message'] )

    return S_OK( {'TaskID': taskID, 'JobID': jobIDs} )


  def __manageTask( self, taskID, action, jobStatus=[] ):
    result = self.__taskManager.getTaskJobs( taskID )
    if not result['OK']:
      return S_ERROR( 'Get task jobs error: %s' % result['Message'] )
    jobIDs = result['Value']

    if jobStatus:
      result = self.__jobMonitor.getJobs( { 'JobID': jobIDs, 'Status': jobStatus } )
      if not result['OK']:
        return S_ERROR( 'Get jobs of status %s error: %s' % (status, result['Message']) )
      jobIDs = result['Value']

    if jobIDs:
      result = action( jobIDs )
      if not result['OK']:
        return S_ERROR( 'Manage jobs error (%s): %s' % (action.__name__, result['Message']) )

    return S_OK( {'TaskID': taskID, 'JobID': jobIDs} )
Esempio n. 4
0
class TaskClient(object):
    """
      The task manager client
  """
    def __init__(self):
        self.__taskManager = RPCClient('WorkloadManagement/TaskManager')
        self.__jobManager = RPCClient('WorkloadManagement/JobManager')
        self.__jobMonitor = RPCClient('WorkloadManagement/JobMonitoring')

    def createTask(self, taskName, taskInfo, jobInfos):
        return self.__taskManager.createTask(taskName, taskInfo, jobInfos)

    def getTaskCount(self, condDict):
        return self.__taskManager.getTaskCount(condDict)

    def listTask(self, condDict, limit, offset, orderAttribute):
        return self.__taskManager.getTasks(condDict, limit, offset,
                                           orderAttribute, 1)

    def renameTask(self, taskID, newName):
        return self.__taskManager.renameTask(taskID, newName)

    def getTaskOwners(self):
        return self.__taskManager.getTaskOwners()

    def getTaskOwnerGroups(self):
        return self.__taskManager.getTaskOwnerGroups()

    def getTask(self, taskID):
        return self.__taskManager.getTask(taskID, 1)

    def getTaskProgress(self, taskID):
        return self.__taskManager.getTaskProgress(taskID)

    def getTaskInfo(self, taskID):
        return self.__taskManager.getTaskInfo(taskID)

    def getTaskJobs(self, taskID):
        return self.__taskManager.getTaskJobs(taskID)

    def getTaskJobsStatistics(self, taskID, statusType):
        return self.__taskManager.getTaskJobsStatistics(taskID, statusType)

    def getTaskHistories(self, taskID):
        return self.__taskManager.getTaskHistories(taskID)

    def getJobs(self, jobIDs, outFields):
        return self.__taskManager.getJobs(jobIDs, outFields)

    def rescheduleTask(self, taskID, jobStatus=[]):
        return self.__manageTask(taskID, self.__jobManager.rescheduleJob,
                                 jobStatus)

    def deleteTask(self, taskID, jobStatus=[]):
        result = self.__manageTask(taskID, self.__jobManager.deleteJob,
                                   jobStatus)
        if not result['OK']:
            return result
        jobIDs = result['Value']['JobID']

        result = self.__taskManager.removeTask(taskID)
        if not result['OK']:
            return S_ERROR('Remove task error: %s' % result['Message'])

        return S_OK({'TaskID': taskID, 'JobID': jobIDs})

    def __manageTask(self, taskID, action, jobStatus=[]):
        result = self.__taskManager.getTaskJobs(taskID)
        if not result['OK']:
            return S_ERROR('Get task jobs error: %s' % result['Message'])
        jobIDs = result['Value']

        if jobStatus:
            result = self.__jobMonitor.getJobs({
                'JobID': jobIDs,
                'Status': jobStatus
            })
            if not result['OK']:
                return S_ERROR('Get jobs of status %s error: %s' %
                               (status, result['Message']))
            jobIDs = result['Value']

        if jobIDs:
            result = action(jobIDs)
            if not result['OK']:
                return S_ERROR('Manage jobs error (%s): %s' %
                               (action.__name__, result['Message']))

        return S_OK({'TaskID': taskID, 'JobID': jobIDs})