def createJobs_(self, jobsL, isNew=True):
"""
Fill crab DB with the jobs filed
"""
task = self.getTask()
jobs = []
for id in jobsL:
parameters = {}
parameters['jobId'] = int(id)
parameters['taskId'] = 1
parameters['name'] = task['name'] + '_' + 'job' + str(id)
job = Job(parameters)
jobs.append(job)
common.bossSession.getRunningInstance(job)
job.runningJob['status'] = 'C'
## added to support second step creation
## maybe it is not needed. TO CLARIFY
if isNew:
task.addJobs(jobs)
else:
task.appendJobs(jobs)
try:
common.bossSession.updateDB( task )
except Exception, e :
raise CrabException('Error updating task '+str(traceback.format_exc()))
def removeTask(self, task):
"""
remove task, jobs and their running instances from db
"""
# db connect
if self.db is None:
self.connect()
# remove jobs in db with non relational checks
if self.bossLiteDB.database == "SQLite":
# load running jobs
rjob = RunningJob({'taskId': task['id']})
rjobList = self.db.select(rjob)
for rjob in rjobList:
rjob.remove(self.db)
# load jobs
job = Job({'taskId': task['id']})
jobList = self.db.select(job)
for job in jobList:
job.remove(self.db)
# remove task
task.remove(self.db)
self.bossLiteDB.commit()
task = None
return task
def removeTask( self, task ):
"""
remove task, jobs and their running instances from db
"""
# db connect
if self.db is None :
self.connect()
# remove jobs in db with non relational checks
if self.bossLiteDB.database == "SQLite":
# load running jobs
rjob = RunningJob( { 'taskId' : task['id'] } )
rjobList = self.db.select( rjob)
for rjob in rjobList :
rjob.remove( self.db )
# load jobs
job = Job( { 'taskId' : task['id'] } )
jobList = self.db.select( job)
for job in jobList :
job.remove( self.db )
# remove task
task.remove( self.db )
self.bossLiteDB.commit()
task = None
return task
def createJobs_(self, jobsL, isNew=True):
"""
Fill crab DB with the jobs filed
"""
task = self.getTask()
jobs = []
for id in jobsL:
parameters = {}
parameters['jobId'] = int(id)
parameters['taskId'] = 1
parameters['name'] = task['name'] + '_' + 'job' + str(id)
job = Job(parameters)
jobs.append(job)
common.bossSession.getRunningInstance(job)
job.runningJob['status'] = 'C'
## added to support second step creation
## maybe it is not needed. TO CLARIFY
if isNew:
task.addJobs(jobs)
else:
task.appendJobs(jobs)
try:
common.bossSession.updateDB(task)
except Exception, e:
raise CrabException('Error updating task ' +
str(traceback.format_exc()))
def testTask(self):
"""
__testTask__
"""
try:
if self.taskId is not None:
self.load(self.taskId, self.jobRange)
else:
self.task = self.bossSession.loadTaskByName(self.taskName,
deep=False)
self.load(self.task, self.jobRange)
print "Task loaded..."
except BossLiteError, e:
print "Task not found... declaring"
taskParams = {
'name': self.taskName,
'globalSandbox': '/etc/redhat-release'
}
self.task = Task(taskParams)
print self.task
parameters = {
'executable': '/bin/echo',
'arguments': 'ciao',
'standardError': 'err.txt',
'standardOutput': 'out.txt',
'outputFiles': ['out.txt']
}
# 'outputFiles' : ['err.txt', 'out.txt', '.BrokerInfo']}
jobs = []
for jobId in range(1, 51):
parameters['name'] = 'job' + str(jobId)
job = Job(parameters)
self.bossSession.getNewRunningInstance(job)
jobs.append(job)
parameters['arguments'] = 'ciao2'
for jobId in range(51, 101):
parameters['name'] = 'job' + str(jobId)
job = Job(parameters)
self.bossSession.getNewRunningInstance(job)
jobs.append(job)
self.task.addJobs(jobs)
self.bossSession.saveTask(self.task)
for job in self.task.jobs:
print job['jobId'], job['taskId'], job['submissionNumber'],
if job.runningJob is not None:
print job.runningJob['jobId'], \
job.runningJob['taskId'], \
job.runningJob['submission']
def loadJobsByRunningAttr( self, runningAttrs=None, jobAttributes=None, all=False, strict=True, limit=None, offset=None, jobList=None ) :
"""
retrieve job information from db for job
whose running instance match attributes
"""
# db connect
if self.db is None :
self.connect()
# creating running jobs
if runningAttrs is None :
run = RunningJob()
else:
run = RunningJob( runningAttrs )
# creating jobs
if jobAttributes is None :
job = Job()
else :
job = Job( jobAttributes )
# if requested, with a right join is possible to fill a running
# instance where missing
if all :
jType = 'right'
else :
jType = ''
# load bunch of jobs?
if jobList is None or jobList == [] :
inList = None
else :
inList = {'jobId' : jobList}
# load job from db
jMap = { 'jobId' : 'jobId',
'taskId' : 'taskId',
'submissionNumber' : 'submission' }
options = { 'strict' : strict,
'jType' : jType,
'limit' : limit,
'offset' : offset,
'inList' : inList }
runJobList = self.db.selectJoin( job, run, \
jMap=jMap , \
options=options )
# recall jobs
for job, runningJob in runJobList :
job.setRunningInstance( runningJob )
# return
return [key[0] for key in runJobList]
def declareNewJobs(self, xml, task, proxyFile=None):
"""
register job related informations in the db
"""
taskInfo, jobInfos, rjAttrs = self.deserialize(xml)
jobRange = range(len(task.getJobs()) + 1)
jobRange.remove(0)
# reconstruct jobs and fill the data
jobs = []
for jI in jobInfos:
# check if jobs is new
if int(jI['jobId']) not in jobRange:
job = Job(jI)
job['taskId'] = task['id']
subn = int(job['submissionNumber'])
if subn > 0:
job['submissionNumber'] = subn - 1
else:
job['submissionNumber'] = subn
#jobs.append(job)
task.appendJob(job)
for job in task.jobs:
if int(job['jobId']) not in jobRange:
attrs = rjAttrs[str(job['name'])]
self.getRunningInstance(job, attrs)
self.updateDB(job)
# self.updateDB( task )
# all done
return task
def declare(self, xml, proxyFile=None):
"""
register job related informations in the db
"""
taskInfo, jobInfos, rjAttrs = self.deserialize(xml)
# reconstruct task
task = Task(taskInfo)
task['user_proxy'] = proxyFile
self.saveTask(task)
# reconstruct jobs and fill the data
jobs = []
for jI in jobInfos:
job = Job(jI)
subn = int(job['submissionNumber'])
if subn > 0:
job['submissionNumber'] = subn - 1
else:
job['submissionNumber'] = subn
jobs.append(job)
task.addJobs(jobs)
for job in task.jobs:
attrs = rjAttrs[str(job['name'])]
self.getRunningInstance(job, attrs)
self.updateDB(job)
# self.updateDB( task )
# all done
return task
def loadJob(self, taskId, jobId):
"""
retrieve job information from db using task and job id
"""
# db connect
if self.db is None:
self.connect()
# creating job
jobAttributes = {'taskId': taskId, "jobId": jobId}
job = Job(jobAttributes)
# load job from db
job.load(self.db)
return job
def loadJob( self, taskId, jobId ) :
"""
retrieve job information from db using task and job id
"""
# db connect
if self.db is None :
self.connect()
# creating job
jobAttributes = { 'taskId' : taskId, "jobId" : jobId}
job = Job( jobAttributes )
# load job from db
job.load(self.db)
return job
def loadJobDistinct(self, taskId, distinctAttr, jobAttributes=None):
"""
retrieve job templates with distinct job attribute
"""
# db connect
if self.db is None:
self.connect()
# creating job
if jobAttributes is None:
job = Job()
else:
job = Job(jobAttributes)
job['taskId'] = taskId
# load job from db
jobList = self.db.selectDistinct(job, distinctAttr)
return jobList
def loadJobsByTimestamp( self, more, less, runningAttrs=None, jobAttributes=None) :
"""
retrieve job information from db for job
whose running instance match attributes
"""
# db connect
if self.db is None :
self.connect()
# creating running jobs
if runningAttrs is None :
run = RunningJob()
else:
run = RunningJob( runningAttrs )
# creating jobs
if jobAttributes is None :
job = Job()
else :
job = Job( jobAttributes )
# load job from db
jMap = { 'jobId' : 'jobId',
'taskId' : 'taskId',
'submissionNumber' : 'submission' }
runJobList = self.db.selectJoin( job, run, \
jMap=jMap, \
less=less, \
more=more )
# recall jobs
for job, runningJob in runJobList :
job.setRunningInstance( runningJob )
# return
return [key[0] for key in runJobList]
def loadJobsByAttr(self, jobAttributes):
"""
retrieve job information from db for job matching attributes
"""
# db connect
if self.db is None:
self.connect()
# creating jobs
job = Job(jobAttributes)
# load job from db
jobList = self.db.select(job)
return jobList
def loadJobDistAttr(self, taskId, value_1, value_2, alist):
"""
retrieve job distinct job attribute
"""
# db connect
if self.db is None:
self.connect()
# creating job
jobAttributes = {'taskId': taskId}
job = Job(jobAttributes)
# load job from db
jobList = self.db.distinctAttr(job, value_1, value_2, alist)
return jobList
def loadJobsByTimestamp(self,
more,
less,
runningAttrs=None,
jobAttributes=None):
"""
retrieve job information from db for job
whose running instance match attributes
"""
# db connect
if self.db is None:
self.connect()
# creating running jobs
if runningAttrs is None:
run = RunningJob()
else:
run = RunningJob(runningAttrs)
# creating jobs
if jobAttributes is None:
job = Job()
else:
job = Job(jobAttributes)
# load job from db
jMap = {
'jobId': 'jobId',
'taskId': 'taskId',
'submissionNumber': 'submission'
}
runJobList = self.db.selectJoin( job, run, \
jMap=jMap, \
less=less, \
more=more )
# recall jobs
for job, runningJob in runJobList:
job.setRunningInstance(runningJob)
# return
return [key[0] for key in runJobList]
" template specified: %s" % self)
if len(objects) > 1:
raise TaskError("Multiple task instances corresponds to the" + \
" template specified: %s" % self)
# copy fields
for key in self.fields:
self.data[key] = objects[0][key]
# get job objects if requested
if deep:
# create basic template if not provided
if self.jobs == []:
template = Job({'taskId' : self.data['id']})
# select single job as template checking proper specification
else:
if len(self.jobs) != 1:
raise TaskError("Multiple job instances specified as" + \
" templates in load task operation for task: %s" % self)
# update template for matching
template = self.jobs[0]
template['taskId'] = self['id']
template['jobId'] = None
# get jobs
self.jobs = db.select(template)
def loadJobsByRunningAttr(self,
runningAttrs=None,
jobAttributes=None,
all=False,
strict=True,
limit=None,
offset=None,
jobList=None):
"""
retrieve job information from db for job
whose running instance match attributes
"""
# db connect
if self.db is None:
self.connect()
# creating running jobs
if runningAttrs is None:
run = RunningJob()
else:
run = RunningJob(runningAttrs)
# creating jobs
if jobAttributes is None:
job = Job()
else:
job = Job(jobAttributes)
# if requested, with a right join is possible to fill a running
# instance where missing
if all:
jType = 'right'
else:
jType = ''
# load bunch of jobs?
if jobList is None or jobList == []:
inList = None
else:
inList = {'jobId': jobList}
# load job from db
jMap = {
'jobId': 'jobId',
'taskId': 'taskId',
'submissionNumber': 'submission'
}
options = {
'strict': strict,
'jType': jType,
'limit': limit,
'offset': offset,
'inList': inList
}
runJobList = self.db.selectJoin( job, run, \
jMap=jMap , \
options=options )
# recall jobs
for job, runningJob in runJobList:
job.setRunningInstance(runningJob)
# return
return [key[0] for key in runJobList]
