def createJobs_(self, jobsL, isNew=True): """ Fill crab DB with the jobs filed """ task = self.getTask() jobs = [] for id in jobsL: parameters = {} parameters['jobId'] = int(id) parameters['taskId'] = 1 parameters['name'] = task['name'] + '_' + 'job' + str(id) job = Job(parameters) jobs.append(job) common.bossSession.getRunningInstance(job) job.runningJob['status'] = 'C' ## added to support second step creation ## maybe it is not needed. TO CLARIFY if isNew: task.addJobs(jobs) else: task.appendJobs(jobs) try: common.bossSession.updateDB( task ) except Exception, e : raise CrabException('Error updating task '+str(traceback.format_exc()))
def removeTask(self, task): """ remove task, jobs and their running instances from db """ # db connect if self.db is None: self.connect() # remove jobs in db with non relational checks if self.bossLiteDB.database == "SQLite": # load running jobs rjob = RunningJob({'taskId': task['id']}) rjobList = self.db.select(rjob) for rjob in rjobList: rjob.remove(self.db) # load jobs job = Job({'taskId': task['id']}) jobList = self.db.select(job) for job in jobList: job.remove(self.db) # remove task task.remove(self.db) self.bossLiteDB.commit() task = None return task
def removeTask( self, task ): """ remove task, jobs and their running instances from db """ # db connect if self.db is None : self.connect() # remove jobs in db with non relational checks if self.bossLiteDB.database == "SQLite": # load running jobs rjob = RunningJob( { 'taskId' : task['id'] } ) rjobList = self.db.select( rjob) for rjob in rjobList : rjob.remove( self.db ) # load jobs job = Job( { 'taskId' : task['id'] } ) jobList = self.db.select( job) for job in jobList : job.remove( self.db ) # remove task task.remove( self.db ) self.bossLiteDB.commit() task = None return task
def createJobs_(self, jobsL, isNew=True): """ Fill crab DB with the jobs filed """ task = self.getTask() jobs = [] for id in jobsL: parameters = {} parameters['jobId'] = int(id) parameters['taskId'] = 1 parameters['name'] = task['name'] + '_' + 'job' + str(id) job = Job(parameters) jobs.append(job) common.bossSession.getRunningInstance(job) job.runningJob['status'] = 'C' ## added to support second step creation ## maybe it is not needed. TO CLARIFY if isNew: task.addJobs(jobs) else: task.appendJobs(jobs) try: common.bossSession.updateDB(task) except Exception, e: raise CrabException('Error updating task ' + str(traceback.format_exc()))
def testTask(self): """ __testTask__ """ try: if self.taskId is not None: self.load(self.taskId, self.jobRange) else: self.task = self.bossSession.loadTaskByName(self.taskName, deep=False) self.load(self.task, self.jobRange) print "Task loaded..." except BossLiteError, e: print "Task not found... declaring" taskParams = { 'name': self.taskName, 'globalSandbox': '/etc/redhat-release' } self.task = Task(taskParams) print self.task parameters = { 'executable': '/bin/echo', 'arguments': 'ciao', 'standardError': 'err.txt', 'standardOutput': 'out.txt', 'outputFiles': ['out.txt'] } # 'outputFiles' : ['err.txt', 'out.txt', '.BrokerInfo']} jobs = [] for jobId in range(1, 51): parameters['name'] = 'job' + str(jobId) job = Job(parameters) self.bossSession.getNewRunningInstance(job) jobs.append(job) parameters['arguments'] = 'ciao2' for jobId in range(51, 101): parameters['name'] = 'job' + str(jobId) job = Job(parameters) self.bossSession.getNewRunningInstance(job) jobs.append(job) self.task.addJobs(jobs) self.bossSession.saveTask(self.task) for job in self.task.jobs: print job['jobId'], job['taskId'], job['submissionNumber'], if job.runningJob is not None: print job.runningJob['jobId'], \ job.runningJob['taskId'], \ job.runningJob['submission']
def loadJobsByRunningAttr( self, runningAttrs=None, jobAttributes=None, all=False, strict=True, limit=None, offset=None, jobList=None ) : """ retrieve job information from db for job whose running instance match attributes """ # db connect if self.db is None : self.connect() # creating running jobs if runningAttrs is None : run = RunningJob() else: run = RunningJob( runningAttrs ) # creating jobs if jobAttributes is None : job = Job() else : job = Job( jobAttributes ) # if requested, with a right join is possible to fill a running # instance where missing if all : jType = 'right' else : jType = '' # load bunch of jobs? if jobList is None or jobList == [] : inList = None else : inList = {'jobId' : jobList} # load job from db jMap = { 'jobId' : 'jobId', 'taskId' : 'taskId', 'submissionNumber' : 'submission' } options = { 'strict' : strict, 'jType' : jType, 'limit' : limit, 'offset' : offset, 'inList' : inList } runJobList = self.db.selectJoin( job, run, \ jMap=jMap , \ options=options ) # recall jobs for job, runningJob in runJobList : job.setRunningInstance( runningJob ) # return return [key[0] for key in runJobList]
def declareNewJobs(self, xml, task, proxyFile=None): """ register job related informations in the db """ taskInfo, jobInfos, rjAttrs = self.deserialize(xml) jobRange = range(len(task.getJobs()) + 1) jobRange.remove(0) # reconstruct jobs and fill the data jobs = [] for jI in jobInfos: # check if jobs is new if int(jI['jobId']) not in jobRange: job = Job(jI) job['taskId'] = task['id'] subn = int(job['submissionNumber']) if subn > 0: job['submissionNumber'] = subn - 1 else: job['submissionNumber'] = subn #jobs.append(job) task.appendJob(job) for job in task.jobs: if int(job['jobId']) not in jobRange: attrs = rjAttrs[str(job['name'])] self.getRunningInstance(job, attrs) self.updateDB(job) # self.updateDB( task ) # all done return task
def declare(self, xml, proxyFile=None): """ register job related informations in the db """ taskInfo, jobInfos, rjAttrs = self.deserialize(xml) # reconstruct task task = Task(taskInfo) task['user_proxy'] = proxyFile self.saveTask(task) # reconstruct jobs and fill the data jobs = [] for jI in jobInfos: job = Job(jI) subn = int(job['submissionNumber']) if subn > 0: job['submissionNumber'] = subn - 1 else: job['submissionNumber'] = subn jobs.append(job) task.addJobs(jobs) for job in task.jobs: attrs = rjAttrs[str(job['name'])] self.getRunningInstance(job, attrs) self.updateDB(job) # self.updateDB( task ) # all done return task
def loadJob(self, taskId, jobId): """ retrieve job information from db using task and job id """ # db connect if self.db is None: self.connect() # creating job jobAttributes = {'taskId': taskId, "jobId": jobId} job = Job(jobAttributes) # load job from db job.load(self.db) return job
def loadJob( self, taskId, jobId ) : """ retrieve job information from db using task and job id """ # db connect if self.db is None : self.connect() # creating job jobAttributes = { 'taskId' : taskId, "jobId" : jobId} job = Job( jobAttributes ) # load job from db job.load(self.db) return job
def loadJobDistinct(self, taskId, distinctAttr, jobAttributes=None): """ retrieve job templates with distinct job attribute """ # db connect if self.db is None: self.connect() # creating job if jobAttributes is None: job = Job() else: job = Job(jobAttributes) job['taskId'] = taskId # load job from db jobList = self.db.selectDistinct(job, distinctAttr) return jobList
def loadJobsByTimestamp( self, more, less, runningAttrs=None, jobAttributes=None) : """ retrieve job information from db for job whose running instance match attributes """ # db connect if self.db is None : self.connect() # creating running jobs if runningAttrs is None : run = RunningJob() else: run = RunningJob( runningAttrs ) # creating jobs if jobAttributes is None : job = Job() else : job = Job( jobAttributes ) # load job from db jMap = { 'jobId' : 'jobId', 'taskId' : 'taskId', 'submissionNumber' : 'submission' } runJobList = self.db.selectJoin( job, run, \ jMap=jMap, \ less=less, \ more=more ) # recall jobs for job, runningJob in runJobList : job.setRunningInstance( runningJob ) # return return [key[0] for key in runJobList]
def loadJobsByAttr(self, jobAttributes): """ retrieve job information from db for job matching attributes """ # db connect if self.db is None: self.connect() # creating jobs job = Job(jobAttributes) # load job from db jobList = self.db.select(job) return jobList
def loadJobDistAttr(self, taskId, value_1, value_2, alist): """ retrieve job distinct job attribute """ # db connect if self.db is None: self.connect() # creating job jobAttributes = {'taskId': taskId} job = Job(jobAttributes) # load job from db jobList = self.db.distinctAttr(job, value_1, value_2, alist) return jobList
def loadJobsByTimestamp(self, more, less, runningAttrs=None, jobAttributes=None): """ retrieve job information from db for job whose running instance match attributes """ # db connect if self.db is None: self.connect() # creating running jobs if runningAttrs is None: run = RunningJob() else: run = RunningJob(runningAttrs) # creating jobs if jobAttributes is None: job = Job() else: job = Job(jobAttributes) # load job from db jMap = { 'jobId': 'jobId', 'taskId': 'taskId', 'submissionNumber': 'submission' } runJobList = self.db.selectJoin( job, run, \ jMap=jMap, \ less=less, \ more=more ) # recall jobs for job, runningJob in runJobList: job.setRunningInstance(runningJob) # return return [key[0] for key in runJobList]
" template specified: %s" % self) if len(objects) > 1: raise TaskError("Multiple task instances corresponds to the" + \ " template specified: %s" % self) # copy fields for key in self.fields: self.data[key] = objects[0][key] # get job objects if requested if deep: # create basic template if not provided if self.jobs == []: template = Job({'taskId' : self.data['id']}) # select single job as template checking proper specification else: if len(self.jobs) != 1: raise TaskError("Multiple job instances specified as" + \ " templates in load task operation for task: %s" % self) # update template for matching template = self.jobs[0] template['taskId'] = self['id'] template['jobId'] = None # get jobs self.jobs = db.select(template)
def loadJobsByRunningAttr(self, runningAttrs=None, jobAttributes=None, all=False, strict=True, limit=None, offset=None, jobList=None): """ retrieve job information from db for job whose running instance match attributes """ # db connect if self.db is None: self.connect() # creating running jobs if runningAttrs is None: run = RunningJob() else: run = RunningJob(runningAttrs) # creating jobs if jobAttributes is None: job = Job() else: job = Job(jobAttributes) # if requested, with a right join is possible to fill a running # instance where missing if all: jType = 'right' else: jType = '' # load bunch of jobs? if jobList is None or jobList == []: inList = None else: inList = {'jobId': jobList} # load job from db jMap = { 'jobId': 'jobId', 'taskId': 'taskId', 'submissionNumber': 'submission' } options = { 'strict': strict, 'jType': jType, 'limit': limit, 'offset': offset, 'inList': inList } runJobList = self.db.selectJoin( job, run, \ jMap=jMap , \ options=options ) # recall jobs for job, runningJob in runJobList: job.setRunningInstance(runningJob) # return return [key[0] for key in runJobList]