def declare(self, xml, proxyFile=None):
"""
register job related informations in the db
"""
taskInfo, jobInfos, rjAttrs = self.deserialize(xml)
# reconstruct task
task = Task(taskInfo)
task['user_proxy'] = proxyFile
self.saveTask(task)
# reconstruct jobs and fill the data
jobs = []
for jI in jobInfos:
job = Job(jI)
subn = int(job['submissionNumber'])
if subn > 0:
job['submissionNumber'] = subn - 1
else:
job['submissionNumber'] = subn
jobs.append(job)
task.addJobs(jobs)
for job in task.jobs:
attrs = rjAttrs[str(job['name'])]
self.getRunningInstance(job, attrs)
self.updateDB(job)
# self.updateDB( task )
# all done
return task
def loadTask(self, taskId, jobRange='all', deep=True):
"""
retrieve task information from db using task id
the jobs loading can be tuned using jobRange:
- None : no jobs are loaded
- 'all' : all jobs are loaded
- list/range : only selected jobs are loaded
"""
# db connect
if self.db is None:
self.connect()
# create template for task
task = Task()
task['id'] = int(taskId)
task.load(self.db, False)
# load jobs
# # backward compatible 'deep' parameter handling
if jobRange is not None and deep != False:
self.load(task, jobRange)
return task
def loadTask( self, taskId, jobRange='all', deep=True ) :
"""
retrieve task information from db using task id
the jobs loading can be tuned using jobRange:
- None : no jobs are loaded
- 'all' : all jobs are loaded
- list/range : only selected jobs are loaded
"""
# db connect
if self.db is None :
self.connect()
# create template for task
task = Task()
task['id'] = int(taskId)
task.load(self.db, False)
# load jobs
# # backward compatible 'deep' parameter handling
if jobRange is not None and deep != False :
self.load( task, jobRange )
return task
def declare( self, xml, proxyFile=None ) :
"""
register job related informations in the db
"""
taskInfo, jobInfos, rjAttrs = self.deserialize( xml )
# reconstruct task
task = Task( taskInfo )
task['user_proxy'] = proxyFile
self.saveTask( task )
# reconstruct jobs and fill the data
jobs = []
for jI in jobInfos:
job = Job( jI )
subn = int( job['submissionNumber'] )
if subn > 0 :
job['submissionNumber'] = subn - 1
else :
job['submissionNumber'] = subn
jobs.append(job)
task.addJobs(jobs)
for job in task.jobs:
attrs = rjAttrs[ str(job['name']) ]
self.getRunningInstance( job, attrs )
self.updateDB( job )
# self.updateDB( task )
# all done
return task
def testTask(self):
"""
__testTask__
"""
try:
if self.taskId is not None:
self.load(self.taskId, self.jobRange)
else:
self.task = self.bossSession.loadTaskByName(self.taskName,
deep=False)
self.load(self.task, self.jobRange)
print "Task loaded..."
except BossLiteError, e:
print "Task not found... declaring"
taskParams = {
'name': self.taskName,
'globalSandbox': '/etc/redhat-release'
}
self.task = Task(taskParams)
print self.task
parameters = {
'executable': '/bin/echo',
'arguments': 'ciao',
'standardError': 'err.txt',
'standardOutput': 'out.txt',
'outputFiles': ['out.txt']
}
# 'outputFiles' : ['err.txt', 'out.txt', '.BrokerInfo']}
jobs = []
for jobId in range(1, 51):
parameters['name'] = 'job' + str(jobId)
job = Job(parameters)
self.bossSession.getNewRunningInstance(job)
jobs.append(job)
parameters['arguments'] = 'ciao2'
for jobId in range(51, 101):
parameters['name'] = 'job' + str(jobId)
job = Job(parameters)
self.bossSession.getNewRunningInstance(job)
jobs.append(job)
self.task.addJobs(jobs)
self.bossSession.saveTask(self.task)
for job in self.task.jobs:
print job['jobId'], job['taskId'], job['submissionNumber'],
if job.runningJob is not None:
print job.runningJob['jobId'], \
job.runningJob['taskId'], \
job.runningJob['submission']
def loadTaskByName(self, name, jobRange='all', deep=True):
"""
retrieve task information from db for task 'name'
"""
# db connect
if self.db is None:
self.connect()
# create template for task and load
task = Task()
task['name'] = name
task.load(self.db, False)
# load jobs
# # backward compatible 'deep' parameter handling
if jobRange is not None and deep != False:
self.load(task, jobRange)
return task
def loadTaskByName( self, name, jobRange='all', deep=True ) :
"""
retrieve task information from db for task 'name'
"""
# db connect
if self.db is None :
self.connect()
# create template for task and load
task = Task()
task['name'] = name
task.load(self.db, False)
# load jobs
# # backward compatible 'deep' parameter handling
if jobRange is not None and deep != False :
self.load( task, jobRange )
return task
def testTask( self ) :
"""
__testTask__
"""
try:
if self.taskId is not None :
self.load( self.taskId, self.jobRange)
else :
self.task = self.bossSession.loadTaskByName( self.taskName,
deep=False)
self.load( self.task, self.jobRange)
print "Task loaded..."
except BossLiteError, e:
print "Task not found... declaring"
taskParams = {'name' : self.taskName,
'globalSandbox' : '/etc/redhat-release' }
self.task = Task( taskParams )
print self.task
parameters = {'executable' : '/bin/echo',
'arguments' : 'ciao',
'standardError' : 'err.txt',
'standardOutput' : 'out.txt',
'outputFiles' : ['out.txt']}
# 'outputFiles' : ['err.txt', 'out.txt', '.BrokerInfo']}
jobs = []
for jobId in range(1, 51):
parameters['name'] = 'job' + str(jobId)
job = Job(parameters)
self.bossSession.getNewRunningInstance(job)
jobs.append(job)
parameters['arguments'] = 'ciao2'
for jobId in range(51, 101):
parameters['name'] = 'job' + str(jobId)
job = Job(parameters)
self.bossSession.getNewRunningInstance(job)
jobs.append(job)
self.task.addJobs(jobs)
self.bossSession.saveTask( self.task )
for job in self.task.jobs :
print job['jobId'], job['taskId'], job['submissionNumber'],
if job.runningJob is not None :
print job.runningJob['jobId'], \
job.runningJob['taskId'], \
job.runningJob['submission']
def loadTasksByUser(self, user, deep=True):
"""
retrieve task information from db for task owned by user
"""
# db connect
if self.db is None:
self.connect()
# create template for task
task = Task()
task['user'] = user
# load task
taskList = self.db.select(task, deep)
return taskList
def createTask_(self, optsToSave):
"""
Task declaration
with the first coniguration stuff
"""
opt = {}
if optsToSave.get('server_mode', 0) == 1:
opt['serverName'] = optsToSave['server_name']
opt['name'] = getUserName() + '_' + string.split(
common.work_space.topDir(),
'/')[-2] + '_' + common.work_space.task_uuid()
task = Task(opt)
try:
common.bossSession.saveTask(task)
except Exception, e:
raise CrabException('Error creating task ' +
str(traceback.format_exc()))
def loadTasksByProxy(self, name, deep=True):
"""
retrieve task information from db for all tasks
with user proxy set to name
"""
# db connect
if self.db is None:
self.connect()
# create template for task
task = Task()
task['user_proxy'] = name
# load task
taskList = self.db.select(task, deep)
return taskList
def load(self,
task,
jobRange="all",
jobAttributes=None,
runningAttrs=None,
strict=True,
limit=None,
offset=None):
"""
retrieve information from db for:
- jobRange can be of the form:
'a,b:c,d,e'
['a',b','c']
'all'
None (no jobs to be loaded
In some way these should be the option to build the query.
Maybe, same options should be used also in
loadSubmitted, loadCreated, loadEnded, loadFailed
Takes the highest submission number for each job
"""
# db connect
if self.db is None:
self.connect()
# already loaded task?
if not isinstance(task, Task):
task = Task({'id': task})
task.load(self.db, False)
elif jobRange == 'all' and task.jobs != []:
jobRange = None
# simple case: no jobs loading request
if jobRange is None:
return task
# defining default
if jobAttributes is None:
jobAttributes = {}
# evaluate job list
jobList = None
if jobRange is not None and jobRange != 'all':
# identify jobRange
if type(jobRange) == list:
jobList = jobRange
else:
jobList = parseRange(jobRange)
# if there are loaded jobs, load just missing
if task.jobs != []:
s = [str(job['jobId']) for job in task.jobs]
jobList = [str(x) for x in jobList if str(x) not in s]
# no jobs to be loaded?
if jobList == []:
return task
elif jobList is not None:
jobList.sort()
# load
jobAttributes['taskId'] = int(task['id'])
jobs = self.loadJobsByRunningAttr( runningAttrs, \
jobAttributes, \
strict=strict, \
limit=limit, offset=offset,\
jobList=jobList )
task.appendJobs(jobs)
return task
def load( self, task, jobRange="all", jobAttributes=None, runningAttrs=None, strict=True, limit=None, offset=None ) :
"""
retrieve information from db for:
- jobRange can be of the form:
'a,b:c,d,e'
['a',b','c']
'all'
None (no jobs to be loaded
In some way these should be the option to build the query.
Maybe, same options should be used also in
loadSubmitted, loadCreated, loadEnded, loadFailed
Takes the highest submission number for each job
"""
# db connect
if self.db is None :
self.connect()
# already loaded task?
if not isinstance( task, Task ) :
task = Task({'id' : task})
task.load(self.db, False)
elif jobRange == 'all' and task.jobs != []:
jobRange = None
# simple case: no jobs loading request
if jobRange is None:
return task
# defining default
if jobAttributes is None :
jobAttributes = {}
# evaluate job list
jobList = None
if jobRange is not None and jobRange != 'all':
# identify jobRange
if type( jobRange ) == list :
jobList = jobRange
else :
jobList = parseRange( jobRange )
# if there are loaded jobs, load just missing
if task.jobs != []:
s = [ str(job['jobId']) for job in task.jobs ]
jobList = [str(x) for x in jobList if str(x) not in s]
# no jobs to be loaded?
if jobList == [] :
return task
elif jobList is not None:
jobList.sort()
# load
jobAttributes['taskId'] = int( task['id'] )
jobs = self.loadJobsByRunningAttr( runningAttrs, \
jobAttributes, \
strict=strict, \
limit=limit, offset=offset,\
jobList=jobList )
task.appendJobs( jobs )
return task
class TaskAPITests(object):
"""
TaskAPITests for the Job class
"""
##########################################################################
def __init__(self, dbtype, installDb=False):
"""
__init__
"""
self.bossSession = None
self.schedSession = None
self.task = None
self.jobRange = 'all'
self.outdir = None
self.taskId = None
self.taskName = 'test_task'
# read configuration
self.database = dbType
self.dbConfig = dbConfig
self.schedulerConfig = schedulerConfig
if dbtype.lower() == 'sqlite':
self.bossSession = self._SqLiteSession(installDb)
elif dbtype.lower() == 'mysql':
self.bossSession = self._MySqlSession(installDb)
else:
print "bad db choice: '%s', allowed only 'SQLite' or 'MySQL'" % \
dbtype
sys.exit()
##########################################################################
def _SqLiteSession(self, installDb):
"""
__sqLiteSession__
"""
if self.bossSession is not None:
return
# BossLiteApi session
self.database = "SQLite"
self.bossSession = BossLiteAPI(self.database, self.dbConfig)
# db installed?
try:
if installDb:
self.bossSession.bossLiteDB.installDB(
'$PRODCOMMON_ROOT/lib/ProdCommon/BossLite/DbObjects/setupDatabase-sqlite.sql'
)
self.bossSession.bossLiteDB.reset()
except:
pass
return self.bossSession
##########################################################################
def _MySqlSession(self, installDb):
"""
__mySqlSession__
"""
if self.bossSession is not None:
return
self.database = "MySQL"
# BossLiteApi session
self.bossSession = BossLiteAPI(self.database, self.dbConfig)
# db installed?
try:
if installDb:
self.bossSession.bossLiteDB.installDB(
'$PRODCOMMON_ROOT/lib/ProdCommon/BossLite/DbObjects/setupDatabase.sql'
)
except:
pass
return self.bossSession
##########################################################################
def schedulerSession(self):
"""
__schedulerSession__
"""
if self.schedSession is not None:
return self.schedSession
self.schedSession = BossLiteAPISched( self.bossSession, \
self.schedulerConfig, \
self.task )
return self.schedSession
##########################################################################
def testTask(self):
"""
__testTask__
"""
try:
if self.taskId is not None:
self.load(self.taskId, self.jobRange)
else:
self.task = self.bossSession.loadTaskByName(self.taskName,
deep=False)
self.load(self.task, self.jobRange)
print "Task loaded..."
except BossLiteError, e:
print "Task not found... declaring"
taskParams = {
'name': self.taskName,
'globalSandbox': '/etc/redhat-release'
}
self.task = Task(taskParams)
print self.task
parameters = {
'executable': '/bin/echo',
'arguments': 'ciao',
'standardError': 'err.txt',
'standardOutput': 'out.txt',
'outputFiles': ['out.txt']
}
# 'outputFiles' : ['err.txt', 'out.txt', '.BrokerInfo']}
jobs = []
for jobId in range(1, 51):
parameters['name'] = 'job' + str(jobId)
job = Job(parameters)
self.bossSession.getNewRunningInstance(job)
jobs.append(job)
parameters['arguments'] = 'ciao2'
for jobId in range(51, 101):
parameters['name'] = 'job' + str(jobId)
job = Job(parameters)
self.bossSession.getNewRunningInstance(job)
jobs.append(job)
self.task.addJobs(jobs)
self.bossSession.saveTask(self.task)
for job in self.task.jobs:
print job['jobId'], job['taskId'], job['submissionNumber'],
if job.runningJob is not None:
print job.runningJob['jobId'], \
job.runningJob['taskId'], \
job.runningJob['submission']
return self.task
class TaskAPITests(object):
"""
TaskAPITests for the Job class
"""
##########################################################################
def __init__(self, dbtype, installDb=False):
"""
__init__
"""
self.bossSession = None
self.schedSession = None
self.task = None
self.jobRange = 'all'
self.outdir = None
self.taskId = None
self.taskName = 'test_task'
# read configuration
self.database = dbType
self.dbConfig = dbConfig
self.schedulerConfig = schedulerConfig
if dbtype.lower() == 'sqlite' :
self.bossSession = self._SqLiteSession(installDb)
elif dbtype.lower() == 'mysql' :
self.bossSession = self._MySqlSession(installDb)
else :
print "bad db choice: '%s', allowed only 'SQLite' or 'MySQL'" % \
dbtype
sys.exit()
##########################################################################
def _SqLiteSession(self, installDb):
"""
__sqLiteSession__
"""
if self.bossSession is not None :
return
# BossLiteApi session
self.database = "SQLite"
self.bossSession = BossLiteAPI( self.database, self.dbConfig )
# db installed?
try:
if installDb :
self.bossSession.bossLiteDB.installDB(
'$PRODCOMMON_ROOT/lib/ProdCommon/BossLite/DbObjects/setupDatabase-sqlite.sql'
)
self.bossSession.bossLiteDB.reset()
except:
pass
return self.bossSession
##########################################################################
def _MySqlSession(self, installDb):
"""
__mySqlSession__
"""
if self.bossSession is not None :
return
self.database = "MySQL"
# BossLiteApi session
self.bossSession = BossLiteAPI( self.database, self.dbConfig )
# db installed?
try:
if installDb :
self.bossSession.bossLiteDB.installDB(
'$PRODCOMMON_ROOT/lib/ProdCommon/BossLite/DbObjects/setupDatabase.sql'
)
except:
pass
return self.bossSession
##########################################################################
def schedulerSession(self):
"""
__schedulerSession__
"""
if self.schedSession is not None:
return self.schedSession
self.schedSession = BossLiteAPISched( self.bossSession, \
self.schedulerConfig, \
self.task )
return self.schedSession
##########################################################################
def testTask( self ) :
"""
__testTask__
"""
try:
if self.taskId is not None :
self.load( self.taskId, self.jobRange)
else :
self.task = self.bossSession.loadTaskByName( self.taskName,
deep=False)
self.load( self.task, self.jobRange)
print "Task loaded..."
except BossLiteError, e:
print "Task not found... declaring"
taskParams = {'name' : self.taskName,
'globalSandbox' : '/etc/redhat-release' }
self.task = Task( taskParams )
print self.task
parameters = {'executable' : '/bin/echo',
'arguments' : 'ciao',
'standardError' : 'err.txt',
'standardOutput' : 'out.txt',
'outputFiles' : ['out.txt']}
# 'outputFiles' : ['err.txt', 'out.txt', '.BrokerInfo']}
jobs = []
for jobId in range(1, 51):
parameters['name'] = 'job' + str(jobId)
job = Job(parameters)
self.bossSession.getNewRunningInstance(job)
jobs.append(job)
parameters['arguments'] = 'ciao2'
for jobId in range(51, 101):
parameters['name'] = 'job' + str(jobId)
job = Job(parameters)
self.bossSession.getNewRunningInstance(job)
jobs.append(job)
self.task.addJobs(jobs)
self.bossSession.saveTask( self.task )
for job in self.task.jobs :
print job['jobId'], job['taskId'], job['submissionNumber'],
if job.runningJob is not None :
print job.runningJob['jobId'], \
job.runningJob['taskId'], \
job.runningJob['submission']
return self.task
