def __init__(self, database, dbConfig=None, pool=None, makePool=False):
"""
initialize the API instance
- database can be both MySQl or SQLite
- dbConfig can be a dictionary with the format
{'dbName':'BossLiteDB',
'host':'localhost',
'user':'******',
'passwd':'BossLitePass',
'socketFileLocation':'/var/run/mysql/mysql.sock',
'portNr':'',
'refreshPeriod' : 4*3600 ,
'maxConnectionAttempts' : 5,
'dbWaitingTime' : 10
}
Passing only dbConfig, a SafeSession will be used for db connection.
Passing a pool or setting makePool, a pool of SafeSession (SafePool)
will be used, enabling a better multithread usage
"""
if database == "WMCore":
from ProdCommon.BossLite.API.BossLiteDBWMCore import BossLiteDBWMCore
self.bossLiteDB = BossLiteDBWMCore(database, dbConfig=dbConfig)
elif pool is None and makePool == False:
self.bossLiteDB = BossLiteDB(database, dbConfig=dbConfig)
else:
from ProdCommon.BossLite.API.BossLitePoolDB import BossLitePoolDB
self.bossLiteDB = BossLitePoolDB( database, dbConfig=dbConfig, \
pool=pool )
self.db = None
def __init__(self, database, dbConfig=None, pool=None, makePool=False):
"""
initialize the API instance
- database can be both MySQl or SQLite
- dbConfig can be a dictionary with the format
{'dbName':'BossLiteDB',
'host':'localhost',
'user':'******',
'passwd':'BossLitePass',
'socketFileLocation':'/var/run/mysql/mysql.sock',
'portNr':'',
'refreshPeriod' : 4*3600 ,
'maxConnectionAttempts' : 5,
'dbWaitingTime' : 10
}
Passing only dbConfig, a SafeSession will be used for db connection.
Passing a pool or setting makePool, a pool of SafeSession (SafePool)
will be used, enabling a better multithread usage
"""
if database == "WMCore":
from ProdCommon.BossLite.API.BossLiteDBWMCore import BossLiteDBWMCore
self.bossLiteDB = BossLiteDBWMCore( database, dbConfig=dbConfig )
elif pool is None and makePool == False:
self.bossLiteDB = BossLiteDB( database, dbConfig=dbConfig )
else :
from ProdCommon.BossLite.API.BossLitePoolDB import BossLitePoolDB
self.bossLiteDB = BossLitePoolDB( database, dbConfig=dbConfig, \
pool=pool )
self.db = None
def removeFinishedJobs(cls, group):
"""
remove all finished jobs from a specific group.
jobs assigned: all jobs in the group
"""
try:
session = BossLitePoolDB( "MySQL", pool=cls.params['sessionPool'] )
db = TrackingDB( session )
joblist = db.getAssociatedJobs()
# in case of empty results
if joblist is None:
logging.debug(
"No finished jobs to be removed from query queues" )
return
for pair in joblist:
db.removeFromCheck( group, pair[0], pair[1], )
#logging.debug(
# "Removing jobs for group " + str(group) \
# + " with BOSS id " + str( pair[0] ) + '.' \
# + str( pair[1] )\
# )
logging.debug("Removed jobs from group %s" % str(group) )
session.close()
del( joblist )
except BossLiteError, ex:
logging.error( 'Failed to remove jobs from queues: %s ' % ex )
def addNewJobs(cls):
"""
include new jobs in the set of jobs to be watched for.
jobs assigned: all new jobs.
"""
try:
session = BossLitePoolDB( "MySQL", pool=cls.params['sessionPool'] )
db = TrackingDB( session )
joblist = db.getUnassociatedJobs()
# in case of empty results
if joblist is None:
logging.debug( "No new jobs to be added in query queues")
return
for pair in joblist:
db.addForCheck( pair[0], pair[1] )
#logging.debug(\
# "Adding jobs to queue with BOSS id "\
# + str( pair[0] ) + '.' + str( pair[1] )\
# )
session.close()
del( joblist )
except BossLiteError, ex:
logging.error( 'Failed to remove jobs from queues: %s ' % ex )
class BossLiteAPI(object):
"""
High level API class for DBObjets.
It allows load/operate/update jobs and taks using just id ranges
"""
def __init__(self, database, dbConfig=None, pool=None, makePool=False):
"""
initialize the API instance
- database can be both MySQl or SQLite
- dbConfig can be a dictionary with the format
{'dbName':'BossLiteDB',
'host':'localhost',
'user':'******',
'passwd':'BossLitePass',
'socketFileLocation':'/var/run/mysql/mysql.sock',
'portNr':'',
'refreshPeriod' : 4*3600 ,
'maxConnectionAttempts' : 5,
'dbWaitingTime' : 10
}
Passing only dbConfig, a SafeSession will be used for db connection.
Passing a pool or setting makePool, a pool of SafeSession (SafePool)
will be used, enabling a better multithread usage
"""
if database == "WMCore":
from ProdCommon.BossLite.API.BossLiteDBWMCore import BossLiteDBWMCore
self.bossLiteDB = BossLiteDBWMCore(database, dbConfig=dbConfig)
elif pool is None and makePool == False:
self.bossLiteDB = BossLiteDB(database, dbConfig=dbConfig)
else:
from ProdCommon.BossLite.API.BossLitePoolDB import BossLitePoolDB
self.bossLiteDB = BossLitePoolDB( database, dbConfig=dbConfig, \
pool=pool )
self.db = None
##########################################################################
def connect(self):
"""
recreate a session and db access
"""
# open Db and create a db access
self.bossLiteDB.connect()
self.db = TrackingDB(self.bossLiteDB)
##########################################################################
def updateDB(self, obj):
"""
update any object table in the DB
works for tasks, jobs, runningJobs
"""
# db connect
if self.db is None:
self.connect()
# update
obj.update(self.db)
self.bossLiteDB.commit()
##########################################################################
def updateRunningInstances(self, task, notSkipClosed=True):
"""
update runningInstances of a task in the DB
"""
# db connect
if self.db is None:
self.connect()
# update
for job in task.jobs:
# update only loaded instances!
if job.runningJob is not None:
job.updateRunningInstance(self.db, notSkipClosed)
self.bossLiteDB.commit()
##########################################################################
def declare(self, xml, proxyFile=None):
"""
register job related informations in the db
"""
taskInfo, jobInfos, rjAttrs = self.deserialize(xml)
# reconstruct task
task = Task(taskInfo)
task['user_proxy'] = proxyFile
self.saveTask(task)
# reconstruct jobs and fill the data
jobs = []
for jI in jobInfos:
job = Job(jI)
subn = int(job['submissionNumber'])
if subn > 0:
job['submissionNumber'] = subn - 1
else:
job['submissionNumber'] = subn
jobs.append(job)
task.addJobs(jobs)
for job in task.jobs:
attrs = rjAttrs[str(job['name'])]
self.getRunningInstance(job, attrs)
self.updateDB(job)
# self.updateDB( task )
# all done
return task
def declareNewJobs(self, xml, task, proxyFile=None):
"""
register job related informations in the db
"""
taskInfo, jobInfos, rjAttrs = self.deserialize(xml)
jobRange = range(len(task.getJobs()) + 1)
jobRange.remove(0)
# reconstruct jobs and fill the data
jobs = []
for jI in jobInfos:
# check if jobs is new
if int(jI['jobId']) not in jobRange:
job = Job(jI)
job['taskId'] = task['id']
subn = int(job['submissionNumber'])
if subn > 0:
job['submissionNumber'] = subn - 1
else:
job['submissionNumber'] = subn
#jobs.append(job)
task.appendJob(job)
for job in task.jobs:
if int(job['jobId']) not in jobRange:
attrs = rjAttrs[str(job['name'])]
self.getRunningInstance(job, attrs)
self.updateDB(job)
# self.updateDB( task )
# all done
return task
##########################################################################
def saveTask(self, task):
"""
register task related informations in the db
"""
# db connect
if self.db is None:
self.connect()
# save task
try:
task.updateInternalData()
task.save(self.db)
self.bossLiteDB.commit()
except TaskError, err:
if str(err).find( 'column name is not unique') == -1 and \
str(err).find( 'Duplicate entry') == -1 and \
task['id'] is not None :
self.removeTask(task)
task = None
raise
return task
class BossLiteAPI(object):
"""
High level API class for DBObjets.
It allows load/operate/update jobs and taks using just id ranges
"""
def __init__(self, database, dbConfig=None, pool=None, makePool=False):
"""
initialize the API instance
- database can be both MySQl or SQLite
- dbConfig can be a dictionary with the format
{'dbName':'BossLiteDB',
'host':'localhost',
'user':'******',
'passwd':'BossLitePass',
'socketFileLocation':'/var/run/mysql/mysql.sock',
'portNr':'',
'refreshPeriod' : 4*3600 ,
'maxConnectionAttempts' : 5,
'dbWaitingTime' : 10
}
Passing only dbConfig, a SafeSession will be used for db connection.
Passing a pool or setting makePool, a pool of SafeSession (SafePool)
will be used, enabling a better multithread usage
"""
if database == "WMCore":
from ProdCommon.BossLite.API.BossLiteDBWMCore import BossLiteDBWMCore
self.bossLiteDB = BossLiteDBWMCore( database, dbConfig=dbConfig )
elif pool is None and makePool == False:
self.bossLiteDB = BossLiteDB( database, dbConfig=dbConfig )
else :
from ProdCommon.BossLite.API.BossLitePoolDB import BossLitePoolDB
self.bossLiteDB = BossLitePoolDB( database, dbConfig=dbConfig, \
pool=pool )
self.db = None
##########################################################################
def connect ( self ) :
"""
recreate a session and db access
"""
# open Db and create a db access
self.bossLiteDB.connect()
self.db = TrackingDB(self.bossLiteDB)
##########################################################################
def updateDB( self, obj ) :
"""
update any object table in the DB
works for tasks, jobs, runningJobs
"""
# db connect
if self.db is None :
self.connect()
# update
obj.update(self.db)
self.bossLiteDB.commit()
##########################################################################
def updateRunningInstances( self, task, notSkipClosed=True ) :
"""
update runningInstances of a task in the DB
"""
# db connect
if self.db is None :
self.connect()
# update
for job in task.jobs:
# update only loaded instances!
if job.runningJob is not None:
job.updateRunningInstance(self.db, notSkipClosed)
self.bossLiteDB.commit()
##########################################################################
def declare( self, xml, proxyFile=None ) :
"""
register job related informations in the db
"""
taskInfo, jobInfos, rjAttrs = self.deserialize( xml )
# reconstruct task
task = Task( taskInfo )
task['user_proxy'] = proxyFile
self.saveTask( task )
# reconstruct jobs and fill the data
jobs = []
for jI in jobInfos:
job = Job( jI )
subn = int( job['submissionNumber'] )
if subn > 0 :
job['submissionNumber'] = subn - 1
else :
job['submissionNumber'] = subn
jobs.append(job)
task.addJobs(jobs)
for job in task.jobs:
attrs = rjAttrs[ str(job['name']) ]
self.getRunningInstance( job, attrs )
self.updateDB( job )
# self.updateDB( task )
# all done
return task
def declareNewJobs( self, xml, task, proxyFile=None ) :
"""
register job related informations in the db
"""
taskInfo, jobInfos, rjAttrs = self.deserialize( xml )
jobRange=range(len(task.getJobs())+1)
jobRange.remove(0)
# reconstruct jobs and fill the data
jobs = []
for jI in jobInfos:
# check if jobs is new
if int(jI['jobId']) not in jobRange:
job = Job( jI )
job['taskId']=task['id']
subn = int( job['submissionNumber'] )
if subn > 0 :
job['submissionNumber'] = subn - 1
else :
job['submissionNumber'] = subn
#jobs.append(job)
task.appendJob(job)
for job in task.jobs:
if int(job['jobId']) not in jobRange:
attrs = rjAttrs[ str(job['name']) ]
self.getRunningInstance( job, attrs )
self.updateDB( job )
# self.updateDB( task )
# all done
return task
##########################################################################
def saveTask( self, task ):
"""
register task related informations in the db
"""
# db connect
if self.db is None :
self.connect()
# save task
try :
task.updateInternalData()
task.save(self.db)
self.bossLiteDB.commit()
except TaskError, err:
if str(err).find( 'column name is not unique') == -1 and \
str(err).find( 'Duplicate entry') == -1 and \
task['id'] is not None :
self.removeTask( task )
task = None
raise
return task
