def recreateOutputOperations(cls, pool):
"""
__recreateOutputOperations__
recreate interrupted get output operations. Two types of operations
can be in interrupted states: 1) operations that have been
requested but never performed, and 2) operations that have been
performed but never processed (meaning that output was retrieved
but no messages were sent based on it).
"""
logging.debug("Recreating interrupted operations")
try :
# open database
bossLiteSession = \
BossLiteAPI('MySQL', pool=cls.params['sessionPool'])
# get interrupted operations
jobs = bossLiteSession.loadJobsByRunningAttr(
{ 'processStatus' : 'in_progress' } )
jobs.extend(
bossLiteSession.loadJobsByRunningAttr(
{ 'processStatus' : 'output_retrieved' } )
)
except DbError, msg:
logging.error('Error updating DB : %s ' % str(msg))
def loadDB(self):
dbname = common.work_space.shareDir() + 'crabDB'
dbConfig = {'dbName': dbname}
try:
common.bossSession = BossLiteAPI(self.db_type, dbConfig)
except Exception, e:
raise CrabException('Istantiate DB Session : ' + str(e))
def __init__(self, schedType, **args):
""" __init__ """
self.host = socket.getfqdn()
self.schedType = schedType
self.bossTask = None
self.scheduler = None
print dbConfig
self.bossLiteSession = BossLiteAPI('MySQL', dbConfig)
def __init__(self, schedType, **args):
""" __init__ """
self.schedType = schedType
self.bossTask = None
self.scheduler = None
self.counters = ['pending', 'submitted', 'waiting', 'ready', \
'scheduled', 'running', 'cleared', 'created', 'other']
print dbConfig
self.bossLiteSession = BossLiteAPI('MySQL', dbConfig, makePool=True)
self.sessionPool = self.bossLiteSession.bossLiteDB.getPool()
self.db = TrackingDB( self.bossLiteSession.bossLiteDB )
def _MySqlSession(self, installDb):
"""
__mySqlSession__
"""
if self.bossSession is not None:
return
self.database = "MySQL"
# BossLiteApi session
self.bossSession = BossLiteAPI(self.database, self.dbConfig)
# db installed?
try:
if installDb:
self.bossSession.bossLiteDB.installDB(
'$PRODCOMMON_ROOT/lib/ProdCommon/BossLite/DbObjects/setupDatabase.sql'
)
except:
pass
return self.bossSession
def __init__(self, logger, workerAttributes):
# Worker Properties
self.tInit = time.time()
self.log = logger
self.configs = workerAttributes
# derived attributes
self.blDBsession = BossLiteAPI('MySQL', \
pool=self.configs['blWorkerPool'])
# Load DB queries
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package = "WMCore.WMBS", \
logger = myThread.logger, \
dbinterface = myThread.dbi)
self.queries = self.daoFactory(classname = "Jobs.ChangeState")
def delegateWmsProxy( wms, config ) :
"""
_delegateWmsProxy_
"""
# // build scheduler session, which also checks proxy validity
# // an exception raised will stop the submission
try:
bossLiteSession = BossLiteAPI('MySQL', dbConfig)
schedulerConfig = { 'name' : 'SchedulerGLiteAPI',
'config' : config,
'service' : wms }
schedSession = BossLiteAPISched( bossLiteSession, schedulerConfig )
schedSession.getSchedulerInterface().delegateProxy()
except BossLiteError, err:
logging.error( "Failed to retrieve scheduler session : %s" % str(err) )
def _MySqlSession(self, installDb):
"""
__mySqlSession__
"""
if self.bossSession is not None :
return
self.database = "MySQL"
# BossLiteApi session
self.bossSession = BossLiteAPI( self.database, self.dbConfig )
# db installed?
try:
if installDb :
self.bossSession.bossLiteDB.installDB(
'$PRODCOMMON_ROOT/lib/ProdCommon/BossLite/DbObjects/setupDatabase.sql'
)
except:
pass
return self.bossSession
class CrabJobCreatorWorker:
"""
_CrabJobCreatorWorker_
"""
def __init__(self, logger, workerAttributes):
# Worker Properties
self.tInit = time.time()
self.log = logger
self.configs = workerAttributes
# derived attributes
self.blDBsession = BossLiteAPI('MySQL', \
pool=self.configs['blWorkerPool'])
# Load DB queries
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package = "WMCore.WMBS", \
logger = myThread.logger, \
dbinterface = myThread.dbi)
self.queries = self.daoFactory(classname = "Jobs.ChangeState")
def updateState(self, payload, status):
"""
Udpate the status of job in payload to status
"""
self.log.info("CrabJobCreatorWorker initialized with payload \
%s"%payload)
import re
r = re.compile("BossJob_(\d+)_(\d+)/")
m = r.search(payload)
if (m):
taskId, jobId = m.groups()
else:
self.log.info("CrabJobCreatorWorkerFailed to parse %s \
and update job status to %s" %(payload,status))
return
# Parse payload to obtain taskId and jobId
self.log.info("--->>> taskId = " + str(taskId))
self.log.info("--->>> jobId = " + str(jobId))
task = self.blDBsession.load(taskId, jobId)
self.log.info("--->>> wmbs job id %s" %task.jobs[0]["wmbsJobId"])
wrapperReturnCode=str(task.jobs[0].runningJob['wrapperReturnCode'])
applicationReturnCode=str(task.jobs[0].runningJob['applicationReturnCode'])
self.log.info("--->>> wrapperReturnCode = " + str(wrapperReturnCode))
self.log.info("--->>> applicationReturnCode = " + str(applicationReturnCode))
# Consider jobs with wrapperReturnCode=0 and applicationReturnCode=0 as success jobs
if int(wrapperReturnCode) == 0 or int(applicationReturnCode) == 0:
status = 'success'
else:
status = 'jobfailed'
if not task.jobs[0]["wmbsJobId"] :
self.log.info("--->>> jobId %s doesn't have wmbsJobId %s" \
%(str(jobId),task.jobs[0]["wmbsJobId"]))
return
# Changment state work
jobObj = Job(id = task.jobs[0]["wmbsJobId"])
if jobObj.exists() == False:
self.log.info("--->>> wmbs job id %s doesn't exists" %task.jobs[0]["wmbsJobId"])
else:
jobObj.load()
jobObj.changeState(status)
self.queries.execute(jobs = [jobObj])
jobObj["outcome"] = status
jobObj.save()
self.log.info("CrabJobCreatorWorker update state to %s of wmbsJob \
%s bl_job %s task %s" %(status, task.jobs[0]["wmbsJobId"], jobId, taskId))
self.log.info("CrabJobCreatorWorker finished")
return
class PilotBossSubmitter:
def __init__(self, schedType, **args):
""" __init__ """
self.host = socket.getfqdn()
self.schedType = schedType
self.bossTask = None
self.scheduler = None
print dbConfig
self.bossLiteSession = BossLiteAPI('MySQL', dbConfig)
def submitPilot(self, taskName, exePath, exe, inpSandbox):
""" __submitPilot__ """
if ( self.schedType == 'LSF' ):
#schedConfig = {'user_proxy' : '', 'service' : '', 'config' : '' }
schedConfig = {'cpCmd': 'rfcp', 'rfioSer': '%s:' % self.host}
self.scheduler = Scheduler.Scheduler('SchedulerLsf', schedConfig)
#self.scheduler = SchedulerLsf(cpCmd='rfcp', rfioSer='vocms13.cern.ch:')
inpSandbox = 'data/khawar/prototype/Pilot.tar'
exe = 'pilotbossSub.sh'
exePath = 'data/khawar/prototype/work'
self.lsfPilotTask(taskName, exe, exePath, inpSandbox)
def lsfPilotTask(self, taskName, exe, exePath, inpSandbox):
"""
__lsfPilotTask__
declares Task and Job objects but don't store
in the db
"""
outdir = time.strftime('%Y%m%d_%H%M%S')
lsfLogDir = os.path.join( '/afs/cern.ch/user/k/khawar/scratch2/khawar/logs', outdir )
#task object
try:
os.mkdir(lsfLogDir)
self.bossTask = Task()
self.bossTask['name'] = taskName
self.bossTask['globalSandbox']= exePath+'/'+exe+','+ inpSandbox
self.bossTask['jobType'] = 'Processing'
self.bossTask['outputDirectory']=lsfLogDir
self.bossLiteSession.saveTask( self.bossTask )
except BossLiteError, ex:
self.failedSubmission = 'pilotboss'
raise JSException(str(ex), FailureList = self.failedSubmission)
#job object
try:
job = Job()
job['name'] = 'pilot'
job['arguments'] = 'pilot'
#job['inputFiles'] = 'data/khawar/prototype/Pilot.tar'
job['standardOutput'] = 'pilot.log'
job['standardError'] = 'piloterr.log'
job['executable'] = exe
job['outputFiles'] = [ 'pilot.log', \
'pilot.tgz', \
'FrameworkJobReport.xml' ]
self.bossLiteSession.getNewRunningInstance( job )
self.bossTask.addJob( job )
self.bossLiteSession.updateDB( self.bossTask )
#get queue information from config
requirements = '-q 8nh80'
#map, taskId, queue = self.scheduler.submitJob(job, self.bossTask, requirements)
print self.scheduler.submit(self.bossTask, requirements)
#print map
#print 'taskId: %s'%taskId
#print 'queue: %s'%queue
except:
print 'error: %s' % sys.exc_info()[0]
print '%s:%s' % (sys.exc_info()[1], sys.exc_info()[2])
traceback.print_exc(file=sys.stdout)
raise Exception('pilotsubmissionfailed','pilot submission failed')
class PilotBossMonitor(MonitorInterface):
"""
_PilotBossMonitor_
"""
def __init__(self, schedType, **args):
""" __init__ """
self.schedType = schedType
self.bossTask = None
self.scheduler = None
self.counters = ['pending', 'submitted', 'waiting', 'ready', \
'scheduled', 'running', 'cleared', 'created', 'other']
print dbConfig
self.bossLiteSession = BossLiteAPI('MySQL', dbConfig, makePool=True)
self.sessionPool = self.bossLiteSession.bossLiteDB.getPool()
self.db = TrackingDB( self.bossLiteSession.bossLiteDB )
def monitorPilot(self, site, tqStateApi):
""" __monitorPilot__
This fnction will submit pilot jobs using
the selected mechanism
"""
if ( self.schedType == 'LSF' ):
#schedConfig = {'user_proxy' : '', 'service' : '', 'config' : '' }
schedConfig = {'cpCmd': 'rfcp', 'rfioSer': '%s:' % self.host}
#self.scheduler = Scheduler.Scheduler('SchedulerLsf', schedConfig)
poller = PABossLitePoll()
poller()
total = poller.pilotJobs
#start the logic here
def getGroups(self, group):
tasks = self.db.getGroupTasks(group)
for taskId in tasks :
print taskId
def getStatistic(self):
"""
__getStatistics__
Poll the BOSS DB for a summary of the job status
"""
# summary of the jobs in the DB
result = self.db.getJobsStatistic()
if result is not None:
counter = {}
for ctr in self.counters:
counter[ctr] = 0
for pair in result :
status, count = pair
if status == 'E':
continue
elif status == 'R' :
counter['running'] = count
elif status == 'I':
counter['pending'] = count
elif status == 'SW' :
counter['waiting'] = count
elif status == 'SR':
counter['ready'] = count
elif status == 'SS':
counter['scheduled'] = count
elif status == 'SU':
counter['submitted'] = count
elif status == 'SE':
counter['cleared'] = count
elif status == 'C':
counter['created'] = count
else:
counter['other'] += count
# display counters
for ctr, value in counter.iteritems():
print(ctr + " jobs : " + str(value))
print("....................")
return result
def pollJobs(self, runningAttrs, processStatus, skipStatus=None ):
"""
__pollJobs__
basic structure for jobs polling
"""
offset = 0
loop = True
while loop :
logging.debug("Max jobs to be loaded %s:%s " % \
(str( offset ), str( offset + self.jobLimit) ) )
self.newJobs = self.bossLiteSession.loadJobsByRunningAttr(
runningAttrs=runningAttrs, \
limit=self.jobLimit, offset=offset
)
logging.info("Polled jobs : " + str( len(self.newJobs) ) )
# exit if no more jobs to query
if self.newJobs == [] :
loop = False
break
else :
offset += self.jobLimit
try:
self.db.processBulkUpdate( self.newJobs, processStatus, \
skipStatus )
logging.info( "Changed status to %s for %s loaded jobs" \
% ( processStatus, str( len(self.newJobs) ) ) )
except BossLiteError, err:
logging.error(
"Failed handling %s loaded jobs, waiting next round: %s" \
% ( processStatus, str( err ) ) )
continue
def doWork(cls, job):
"""
__doWork__
get the output of the job specified.
return job when successful,
None if the job does not need/allow further processing
*** thread safe ***
"""
try:
ret = None
logging.debug("%s: Getting output" % cls.fullId( job ) )
# open database
bossLiteSession = \
BossLiteAPI('MySQL', pool=cls.params['sessionPool'])
# instantiate JobHandling object
jHParams = deepcopy(cls.params['jobHandlingParams'])
jHParams['bossLiteSession'] = bossLiteSession
jobHandling = JobHandling( jHParams )
# verify the status
status = job.runningJob['processStatus']
# a variable to check if the output already retrieved
skipRetrieval = False
# output retrieved before, then recover interrupted operation
if status == 'output_retrieved':
logging.warning("%s: Enqueuing previous ouput" % \
cls.fullId( job ) )
skipRetrieval = True
# non expected status, abandon processing for job
elif status != 'in_progress' :
logging.error("%s: Cannot get output, status is %s" % \
(cls.fullId( job ), status) )
return
# inconsistent status
if status == 'in_progress' and job.runningJob['closed'] == 'Y':
logging.warning(
"%s in status %s: Enqueuing previous output" % \
(cls.fullId( job ), status) )
job.runningJob['processStatus'] = 'output_retrieved'
bossLiteSession.updateDB( job )
skipRetrieval = True
logging.debug("%s: Processing output" % cls.fullId( job ) )
if skipRetrieval :
# job failed: perform postMortem operations and notify failure
if job.runningJob['status'] in cls.failureCodes:
ret = jobHandling.performErrorProcessing(job)
else:
ret = jobHandling.performOutputProcessing(job)
else :
ret = cls.action( bossLiteSession, job, jobHandling )
logging.debug("%s : Processing output finished" % \
cls.fullId( job ) )
# update status
if ret is not None:
job.runningJob['processStatus'] = 'processed'
job.runningJob['closed'] = 'Y'
# Fabio # possible fix for SubSuccess bug
job.runningJob['status'] = 'E'
bossLiteSession.updateDB( job )
##
else:
# allow job to be reprocessed
cls.recoverStatus( job, bossLiteSession )
# if the job has not to be reprocessed
if int( job.runningJob['getOutputRetry'] ) >= \
int( cls.params['maxGetOutputAttempts'] ) :
logging.error( "%s: LAST ATTEMPT RETRIEVAL FAILED!!!" % \
cls.fullId( job ) )
# set as failed
job.runningJob['status'] = 'A'
job.runningJob['processStatus'] = 'failed'
job.runningJob['statusScheduler'] = 'Abandoned'
job.runningJob['statusReason'] = \
'GetOutput failed %s times' \
% cls.params['maxGetOutputAttempts']
# perform update
bossLiteSession.updateDB( job )
return ret
# thread has failed because of a Bossite problem
except BossLiteError, err:
# allow job to be reprocessed
cls.recoverStatus( job, bossLiteSession )
# show error message
logging.error( "%s failed to process output : %s" % \
( cls.fullId( job ), str(err) ) )
class TaskAPITests(object):
"""
TaskAPITests for the Job class
"""
##########################################################################
def __init__(self, dbtype, installDb=False):
"""
__init__
"""
self.bossSession = None
self.schedSession = None
self.task = None
self.jobRange = 'all'
self.outdir = None
self.taskId = None
self.taskName = 'test_task'
# read configuration
self.database = dbType
self.dbConfig = dbConfig
self.schedulerConfig = schedulerConfig
if dbtype.lower() == 'sqlite':
self.bossSession = self._SqLiteSession(installDb)
elif dbtype.lower() == 'mysql':
self.bossSession = self._MySqlSession(installDb)
else:
print "bad db choice: '%s', allowed only 'SQLite' or 'MySQL'" % \
dbtype
sys.exit()
##########################################################################
def _SqLiteSession(self, installDb):
"""
__sqLiteSession__
"""
if self.bossSession is not None:
return
# BossLiteApi session
self.database = "SQLite"
self.bossSession = BossLiteAPI(self.database, self.dbConfig)
# db installed?
try:
if installDb:
self.bossSession.bossLiteDB.installDB(
'$PRODCOMMON_ROOT/lib/ProdCommon/BossLite/DbObjects/setupDatabase-sqlite.sql'
)
self.bossSession.bossLiteDB.reset()
except:
pass
return self.bossSession
##########################################################################
def _MySqlSession(self, installDb):
"""
__mySqlSession__
"""
if self.bossSession is not None:
return
self.database = "MySQL"
# BossLiteApi session
self.bossSession = BossLiteAPI(self.database, self.dbConfig)
# db installed?
try:
if installDb:
self.bossSession.bossLiteDB.installDB(
'$PRODCOMMON_ROOT/lib/ProdCommon/BossLite/DbObjects/setupDatabase.sql'
)
except:
pass
return self.bossSession
##########################################################################
def schedulerSession(self):
"""
__schedulerSession__
"""
if self.schedSession is not None:
return self.schedSession
self.schedSession = BossLiteAPISched( self.bossSession, \
self.schedulerConfig, \
self.task )
return self.schedSession
##########################################################################
def testTask(self):
"""
__testTask__
"""
try:
if self.taskId is not None:
self.load(self.taskId, self.jobRange)
else:
self.task = self.bossSession.loadTaskByName(self.taskName,
deep=False)
self.load(self.task, self.jobRange)
print "Task loaded..."
except BossLiteError, e:
print "Task not found... declaring"
taskParams = {
'name': self.taskName,
'globalSandbox': '/etc/redhat-release'
}
self.task = Task(taskParams)
print self.task
parameters = {
'executable': '/bin/echo',
'arguments': 'ciao',
'standardError': 'err.txt',
'standardOutput': 'out.txt',
'outputFiles': ['out.txt']
}
# 'outputFiles' : ['err.txt', 'out.txt', '.BrokerInfo']}
jobs = []
for jobId in range(1, 51):
parameters['name'] = 'job' + str(jobId)
job = Job(parameters)
self.bossSession.getNewRunningInstance(job)
jobs.append(job)
parameters['arguments'] = 'ciao2'
for jobId in range(51, 101):
parameters['name'] = 'job' + str(jobId)
job = Job(parameters)
self.bossSession.getNewRunningInstance(job)
jobs.append(job)
self.task.addJobs(jobs)
self.bossSession.saveTask(self.task)
for job in self.task.jobs:
print job['jobId'], job['taskId'], job['submissionNumber'],
if job.runningJob is not None:
print job.runningJob['jobId'], \
job.runningJob['taskId'], \
job.runningJob['submission']
return self.task
def __call__(self, schedType):
self.schedType = schedType
self.bossLiteSession = BossLiteAPI('MySQL', dbConfig)
class PilotBossSubmitter:
def __init__(self):
""" __init__ """
self.host = socket.getfqdn()
self.schedType = None
self.bossTask = None
self.scheduler = None
myThread = threading.currentThread()
self.logger = myThread.logger
#self.logger.debug( dbConfig )
#self.bossLiteSession = BossLiteAPI('MySQL', dbConfig)
def __call__(self, schedType):
self.schedType = schedType
self.bossLiteSession = BossLiteAPI('MySQL', dbConfig)
def taskCount (self):
"""
__taskCount__
count the tasks in the table.
helps in making unique name for tasks
"""
sql="select count(id) from bl_task"
bossLiteDB = BossLiteDB( 'MySQL', dbConfig )
taskTotal = bossLiteDB.selectOne( sql )
return taskTotal
def submitPilot(self, taskName, exe, exePath, inpSandbox, bulkSize=1):
""" __submitPilot__
This fnction will submit pilot jobs using
the selected mechanism
"""
if ( self.schedType == 'LSF' ):
#schedConfig = {'user_proxy' : '', 'service' : '', 'config' : '' }
#schedConfig = {'cpCmd': 'rfcp', 'rfioSer': '%s:' % self.host}
schedConfig = {'cpCmd': 'rfcp', 'rfioSer': '%s:' % self.host}
self.scheduler = Scheduler.Scheduler('SchedulerLsf', schedConfig)
self.lsfPilotTask(taskName, exe, exePath, inpSandbox, bulkSize)
def lsfPilotTask(self, taskName, exe, exePath, inpSandbox, bulkSize):
"""
__lsfPilotTask__
declares Task and Job objects but don't store
in the db
"""
#load LSF configurations
lsfConfig = loadLSFConfig()
if ( lsfConfig is None):
logging.debug('LSFConfig is None: returning back the call ')
return
outdir = time.strftime('%Y%m%d_%H%M%S')
#got logDir information from the config
logDir = lsfConfig['logDir']
lsfLogDir = os.path.join(logDir, outdir )
taskCount = self.taskCount()
taskName = "%s_%s" % (taskName, taskCount + 1)
#task object
try:
os.mkdir(lsfLogDir)
self.bossTask = Task()
self.bossTask['name'] = taskName
self.bossTask['jobType'] = 'PilotJob'
self.bossTask['globalSandbox'] = exePath+'/'+exe+','+ inpSandbox
self.bossTask['outputDirectory'] = lsfLogDir
self.bossLiteSession.saveTask( self.bossTask )
except BossLiteError, ex:
self.failedSubmission = 'pilotboss'
raise JSException(str(ex), FailureList = self.failedSubmission)
#job object
try:
for j in range(0, bulkSize):
job = Job()
job['name'] = '%s_pilot'%j
#these arguments are passed to the submission script
# ANTO: job must output to its working dir
# LSF will copy result back to logDir
# Get std.out and std.err
# job['standardOutput'] = '%s/pilot_%s.log'%(lsfLogDir, j)
# job['standardOutput'] = 'pilot_%s.log' % (j)
job['standardOutput'] = '%s_std.out' % j
# job['standardError'] = '%s/piloterr_%s.log'%(lsfLogDir, j)
# job['standardError'] = 'piloterr_%s.log' % (j)
job['standardError'] = '%s_std.err' % j
job['executable'] = exe
# job['outputFiles'] = [ '%s/pilot_%s.log'%(lsfLogDir, j), \
# '%s/pilot_%s.tgz'%(lsfLogDir, j)]
# job['outputFiles'] = [ 'pilot_%s.log' % (j), \
# 'pilot_%s.tgz' % (j)]
job['outputFiles'] = [ '*std.out', '*std.err']
# job['outputDirectory'] = lsfLogDir + '/%s' % (j)
self.bossLiteSession.getNewRunningInstance( job )
# job.runningJob['outputDirectory'] = lsfLogDir + '/%s' % (j)
# os.mkdir(job.runningJob['outputDirectory'])
# END ANTO
self.bossTask.addJob( job )
self.bossLiteSession.updateDB( self.bossTask )
lsfQueue = lsfConfig['Queue']
lsfRsrReq = lsfConfig['Resource']
requirements = '-q %s -g %s' % (lsfQueue, LSFConfiguration.getGroup() )
requirements += ' -J %s' % taskName
if ( lsfRsrReq is not None or lsfRsrReq != ""):
requirements += " -R \"%s\"" % lsfRsrReq
self.logger.debug( 'Now Submitting %s pilotjobs through BossInterface' % bulkSize)
output = self.scheduler.submit(self.bossTask, requirements)
self.logger.debug(output)
self.bossLiteSession.updateDB( self.bossTask )
except:
self.logger.debug( 'error: %s' % sys.exc_info()[0])
self.logger.debug( '%s:%s' % (sys.exc_info()[1], sys.exc_info()[2]) )
traceback.print_exc(file=sys.stdout)
raise Exception('pilotsubmissionfailed','pilot submission failed')
class TaskAPITests(object):
"""
TaskAPITests for the Job class
"""
##########################################################################
def __init__(self, dbtype, installDb=False):
"""
__init__
"""
self.bossSession = None
self.schedSession = None
self.task = None
self.jobRange = 'all'
self.outdir = None
self.taskId = None
self.taskName = 'test_task'
# read configuration
self.database = dbType
self.dbConfig = dbConfig
self.schedulerConfig = schedulerConfig
if dbtype.lower() == 'sqlite' :
self.bossSession = self._SqLiteSession(installDb)
elif dbtype.lower() == 'mysql' :
self.bossSession = self._MySqlSession(installDb)
else :
print "bad db choice: '%s', allowed only 'SQLite' or 'MySQL'" % \
dbtype
sys.exit()
##########################################################################
def _SqLiteSession(self, installDb):
"""
__sqLiteSession__
"""
if self.bossSession is not None :
return
# BossLiteApi session
self.database = "SQLite"
self.bossSession = BossLiteAPI( self.database, self.dbConfig )
# db installed?
try:
if installDb :
self.bossSession.bossLiteDB.installDB(
'$PRODCOMMON_ROOT/lib/ProdCommon/BossLite/DbObjects/setupDatabase-sqlite.sql'
)
self.bossSession.bossLiteDB.reset()
except:
pass
return self.bossSession
##########################################################################
def _MySqlSession(self, installDb):
"""
__mySqlSession__
"""
if self.bossSession is not None :
return
self.database = "MySQL"
# BossLiteApi session
self.bossSession = BossLiteAPI( self.database, self.dbConfig )
# db installed?
try:
if installDb :
self.bossSession.bossLiteDB.installDB(
'$PRODCOMMON_ROOT/lib/ProdCommon/BossLite/DbObjects/setupDatabase.sql'
)
except:
pass
return self.bossSession
##########################################################################
def schedulerSession(self):
"""
__schedulerSession__
"""
if self.schedSession is not None:
return self.schedSession
self.schedSession = BossLiteAPISched( self.bossSession, \
self.schedulerConfig, \
self.task )
return self.schedSession
##########################################################################
def testTask( self ) :
"""
__testTask__
"""
try:
if self.taskId is not None :
self.load( self.taskId, self.jobRange)
else :
self.task = self.bossSession.loadTaskByName( self.taskName,
deep=False)
self.load( self.task, self.jobRange)
print "Task loaded..."
except BossLiteError, e:
print "Task not found... declaring"
taskParams = {'name' : self.taskName,
'globalSandbox' : '/etc/redhat-release' }
self.task = Task( taskParams )
print self.task
parameters = {'executable' : '/bin/echo',
'arguments' : 'ciao',
'standardError' : 'err.txt',
'standardOutput' : 'out.txt',
'outputFiles' : ['out.txt']}
# 'outputFiles' : ['err.txt', 'out.txt', '.BrokerInfo']}
jobs = []
for jobId in range(1, 51):
parameters['name'] = 'job' + str(jobId)
job = Job(parameters)
self.bossSession.getNewRunningInstance(job)
jobs.append(job)
parameters['arguments'] = 'ciao2'
for jobId in range(51, 101):
parameters['name'] = 'job' + str(jobId)
job = Job(parameters)
self.bossSession.getNewRunningInstance(job)
jobs.append(job)
self.task.addJobs(jobs)
self.bossSession.saveTask( self.task )
for job in self.task.jobs :
print job['jobId'], job['taskId'], job['submissionNumber'],
if job.runningJob is not None :
print job.runningJob['jobId'], \
job.runningJob['taskId'], \
job.runningJob['submission']
return self.task