def __init__(self, database, dbConfig=None, pool=None, makePool=False): """ initialize the API instance - database can be both MySQl or SQLite - dbConfig can be a dictionary with the format {'dbName':'BossLiteDB', 'host':'localhost', 'user':'******', 'passwd':'BossLitePass', 'socketFileLocation':'/var/run/mysql/mysql.sock', 'portNr':'', 'refreshPeriod' : 4*3600 , 'maxConnectionAttempts' : 5, 'dbWaitingTime' : 10 } Passing only dbConfig, a SafeSession will be used for db connection. Passing a pool or setting makePool, a pool of SafeSession (SafePool) will be used, enabling a better multithread usage """ if database == "WMCore": from ProdCommon.BossLite.API.BossLiteDBWMCore import BossLiteDBWMCore self.bossLiteDB = BossLiteDBWMCore(database, dbConfig=dbConfig) elif pool is None and makePool == False: self.bossLiteDB = BossLiteDB(database, dbConfig=dbConfig) else: from ProdCommon.BossLite.API.BossLitePoolDB import BossLitePoolDB self.bossLiteDB = BossLitePoolDB( database, dbConfig=dbConfig, \ pool=pool ) self.db = None
def __init__(self, database, dbConfig=None, pool=None, makePool=False): """ initialize the API instance - database can be both MySQl or SQLite - dbConfig can be a dictionary with the format {'dbName':'BossLiteDB', 'host':'localhost', 'user':'******', 'passwd':'BossLitePass', 'socketFileLocation':'/var/run/mysql/mysql.sock', 'portNr':'', 'refreshPeriod' : 4*3600 , 'maxConnectionAttempts' : 5, 'dbWaitingTime' : 10 } Passing only dbConfig, a SafeSession will be used for db connection. Passing a pool or setting makePool, a pool of SafeSession (SafePool) will be used, enabling a better multithread usage """ if database == "WMCore": from ProdCommon.BossLite.API.BossLiteDBWMCore import BossLiteDBWMCore self.bossLiteDB = BossLiteDBWMCore( database, dbConfig=dbConfig ) elif pool is None and makePool == False: self.bossLiteDB = BossLiteDB( database, dbConfig=dbConfig ) else : from ProdCommon.BossLite.API.BossLitePoolDB import BossLitePoolDB self.bossLiteDB = BossLitePoolDB( database, dbConfig=dbConfig, \ pool=pool ) self.db = None
def removeFinishedJobs(cls, group): """ remove all finished jobs from a specific group. jobs assigned: all jobs in the group """ try: session = BossLitePoolDB( "MySQL", pool=cls.params['sessionPool'] ) db = TrackingDB( session ) joblist = db.getAssociatedJobs() # in case of empty results if joblist is None: logging.debug( "No finished jobs to be removed from query queues" ) return for pair in joblist: db.removeFromCheck( group, pair[0], pair[1], ) #logging.debug( # "Removing jobs for group " + str(group) \ # + " with BOSS id " + str( pair[0] ) + '.' \ # + str( pair[1] )\ # ) logging.debug("Removed jobs from group %s" % str(group) ) session.close() del( joblist ) except BossLiteError, ex: logging.error( 'Failed to remove jobs from queues: %s ' % ex )
def addNewJobs(cls): """ include new jobs in the set of jobs to be watched for. jobs assigned: all new jobs. """ try: session = BossLitePoolDB( "MySQL", pool=cls.params['sessionPool'] ) db = TrackingDB( session ) joblist = db.getUnassociatedJobs() # in case of empty results if joblist is None: logging.debug( "No new jobs to be added in query queues") return for pair in joblist: db.addForCheck( pair[0], pair[1] ) #logging.debug(\ # "Adding jobs to queue with BOSS id "\ # + str( pair[0] ) + '.' + str( pair[1] )\ # ) session.close() del( joblist ) except BossLiteError, ex: logging.error( 'Failed to remove jobs from queues: %s ' % ex )
class BossLiteAPI(object): """ High level API class for DBObjets. It allows load/operate/update jobs and taks using just id ranges """ def __init__(self, database, dbConfig=None, pool=None, makePool=False): """ initialize the API instance - database can be both MySQl or SQLite - dbConfig can be a dictionary with the format {'dbName':'BossLiteDB', 'host':'localhost', 'user':'******', 'passwd':'BossLitePass', 'socketFileLocation':'/var/run/mysql/mysql.sock', 'portNr':'', 'refreshPeriod' : 4*3600 , 'maxConnectionAttempts' : 5, 'dbWaitingTime' : 10 } Passing only dbConfig, a SafeSession will be used for db connection. Passing a pool or setting makePool, a pool of SafeSession (SafePool) will be used, enabling a better multithread usage """ if database == "WMCore": from ProdCommon.BossLite.API.BossLiteDBWMCore import BossLiteDBWMCore self.bossLiteDB = BossLiteDBWMCore(database, dbConfig=dbConfig) elif pool is None and makePool == False: self.bossLiteDB = BossLiteDB(database, dbConfig=dbConfig) else: from ProdCommon.BossLite.API.BossLitePoolDB import BossLitePoolDB self.bossLiteDB = BossLitePoolDB( database, dbConfig=dbConfig, \ pool=pool ) self.db = None ########################################################################## def connect(self): """ recreate a session and db access """ # open Db and create a db access self.bossLiteDB.connect() self.db = TrackingDB(self.bossLiteDB) ########################################################################## def updateDB(self, obj): """ update any object table in the DB works for tasks, jobs, runningJobs """ # db connect if self.db is None: self.connect() # update obj.update(self.db) self.bossLiteDB.commit() ########################################################################## def updateRunningInstances(self, task, notSkipClosed=True): """ update runningInstances of a task in the DB """ # db connect if self.db is None: self.connect() # update for job in task.jobs: # update only loaded instances! if job.runningJob is not None: job.updateRunningInstance(self.db, notSkipClosed) self.bossLiteDB.commit() ########################################################################## def declare(self, xml, proxyFile=None): """ register job related informations in the db """ taskInfo, jobInfos, rjAttrs = self.deserialize(xml) # reconstruct task task = Task(taskInfo) task['user_proxy'] = proxyFile self.saveTask(task) # reconstruct jobs and fill the data jobs = [] for jI in jobInfos: job = Job(jI) subn = int(job['submissionNumber']) if subn > 0: job['submissionNumber'] = subn - 1 else: job['submissionNumber'] = subn jobs.append(job) task.addJobs(jobs) for job in task.jobs: attrs = rjAttrs[str(job['name'])] self.getRunningInstance(job, attrs) self.updateDB(job) # self.updateDB( task ) # all done return task def declareNewJobs(self, xml, task, proxyFile=None): """ register job related informations in the db """ taskInfo, jobInfos, rjAttrs = self.deserialize(xml) jobRange = range(len(task.getJobs()) + 1) jobRange.remove(0) # reconstruct jobs and fill the data jobs = [] for jI in jobInfos: # check if jobs is new if int(jI['jobId']) not in jobRange: job = Job(jI) job['taskId'] = task['id'] subn = int(job['submissionNumber']) if subn > 0: job['submissionNumber'] = subn - 1 else: job['submissionNumber'] = subn #jobs.append(job) task.appendJob(job) for job in task.jobs: if int(job['jobId']) not in jobRange: attrs = rjAttrs[str(job['name'])] self.getRunningInstance(job, attrs) self.updateDB(job) # self.updateDB( task ) # all done return task ########################################################################## def saveTask(self, task): """ register task related informations in the db """ # db connect if self.db is None: self.connect() # save task try: task.updateInternalData() task.save(self.db) self.bossLiteDB.commit() except TaskError, err: if str(err).find( 'column name is not unique') == -1 and \ str(err).find( 'Duplicate entry') == -1 and \ task['id'] is not None : self.removeTask(task) task = None raise return task
class BossLiteAPI(object): """ High level API class for DBObjets. It allows load/operate/update jobs and taks using just id ranges """ def __init__(self, database, dbConfig=None, pool=None, makePool=False): """ initialize the API instance - database can be both MySQl or SQLite - dbConfig can be a dictionary with the format {'dbName':'BossLiteDB', 'host':'localhost', 'user':'******', 'passwd':'BossLitePass', 'socketFileLocation':'/var/run/mysql/mysql.sock', 'portNr':'', 'refreshPeriod' : 4*3600 , 'maxConnectionAttempts' : 5, 'dbWaitingTime' : 10 } Passing only dbConfig, a SafeSession will be used for db connection. Passing a pool or setting makePool, a pool of SafeSession (SafePool) will be used, enabling a better multithread usage """ if database == "WMCore": from ProdCommon.BossLite.API.BossLiteDBWMCore import BossLiteDBWMCore self.bossLiteDB = BossLiteDBWMCore( database, dbConfig=dbConfig ) elif pool is None and makePool == False: self.bossLiteDB = BossLiteDB( database, dbConfig=dbConfig ) else : from ProdCommon.BossLite.API.BossLitePoolDB import BossLitePoolDB self.bossLiteDB = BossLitePoolDB( database, dbConfig=dbConfig, \ pool=pool ) self.db = None ########################################################################## def connect ( self ) : """ recreate a session and db access """ # open Db and create a db access self.bossLiteDB.connect() self.db = TrackingDB(self.bossLiteDB) ########################################################################## def updateDB( self, obj ) : """ update any object table in the DB works for tasks, jobs, runningJobs """ # db connect if self.db is None : self.connect() # update obj.update(self.db) self.bossLiteDB.commit() ########################################################################## def updateRunningInstances( self, task, notSkipClosed=True ) : """ update runningInstances of a task in the DB """ # db connect if self.db is None : self.connect() # update for job in task.jobs: # update only loaded instances! if job.runningJob is not None: job.updateRunningInstance(self.db, notSkipClosed) self.bossLiteDB.commit() ########################################################################## def declare( self, xml, proxyFile=None ) : """ register job related informations in the db """ taskInfo, jobInfos, rjAttrs = self.deserialize( xml ) # reconstruct task task = Task( taskInfo ) task['user_proxy'] = proxyFile self.saveTask( task ) # reconstruct jobs and fill the data jobs = [] for jI in jobInfos: job = Job( jI ) subn = int( job['submissionNumber'] ) if subn > 0 : job['submissionNumber'] = subn - 1 else : job['submissionNumber'] = subn jobs.append(job) task.addJobs(jobs) for job in task.jobs: attrs = rjAttrs[ str(job['name']) ] self.getRunningInstance( job, attrs ) self.updateDB( job ) # self.updateDB( task ) # all done return task def declareNewJobs( self, xml, task, proxyFile=None ) : """ register job related informations in the db """ taskInfo, jobInfos, rjAttrs = self.deserialize( xml ) jobRange=range(len(task.getJobs())+1) jobRange.remove(0) # reconstruct jobs and fill the data jobs = [] for jI in jobInfos: # check if jobs is new if int(jI['jobId']) not in jobRange: job = Job( jI ) job['taskId']=task['id'] subn = int( job['submissionNumber'] ) if subn > 0 : job['submissionNumber'] = subn - 1 else : job['submissionNumber'] = subn #jobs.append(job) task.appendJob(job) for job in task.jobs: if int(job['jobId']) not in jobRange: attrs = rjAttrs[ str(job['name']) ] self.getRunningInstance( job, attrs ) self.updateDB( job ) # self.updateDB( task ) # all done return task ########################################################################## def saveTask( self, task ): """ register task related informations in the db """ # db connect if self.db is None : self.connect() # save task try : task.updateInternalData() task.save(self.db) self.bossLiteDB.commit() except TaskError, err: if str(err).find( 'column name is not unique') == -1 and \ str(err).find( 'Duplicate entry') == -1 and \ task['id'] is not None : self.removeTask( task ) task = None raise return task