def __call__(self, destination=None):
"""
_operator()_
Query BossLite Here....
"""
#sqlStr1 = \
"""
select count(bl_job.id) from bl_job,bl_runningjob where bl_runningjob.job_id=bl_job.job_id and bl_runningjob.task_id=bl_job.task_id and bl_job.name like '%pilot%' and bl_runningjob.status not in ('E','C','A','SD') and bl_runningjob.closed='N';
"""
sqlStr1 = ''
if ( destination!=None and destination.find('cern.ch')>0 ):
sqlStr1 = \
"""select count(bl_runningjob.id) from bl_task,bl_runningjob where bl_runningjob.task_id=bl_task.id and bl_task.job_type='PilotJob' and bl_runningjob.status not in ('E','C','A','SD') and bl_runningjob.closed='N';
"""
elif ( destination != None):
sqlStr1 = \
"""select count(bl_runningjob.id) from bl_task,bl_runningjob where bl_runningjob.task_id=bl_task.id and bl_task.job_type='PilotJob' and bl_runningjob.destination='%s' and bl_runningjob.status not in ('E','C','A','SD') and bl_runningjob.closed='N'; """ % destination
print sqlStr1
bossLiteDB = BossLiteDB( 'MySQL', dbConfig )
self.pilotJobs = bossLiteDB.selectOne( sqlStr1 )
print self.pilotJobs
def bossAdmin(sql):
bossLiteDB = BossLiteDB( 'MySQL', dbConfig )
answ = bossLiteDB.select( sql )
if answ is None :
return []
return [ line for line in answ ]
def activeSiteJobs():
## returns a dictionary with ce_queue regexp and number of active jobs.
## portnumbers are not available, hope they are unique per ce
bossLiteDB = BossLiteDB( 'MySQL', dbConfig )
ce_act = {}
act_re = re.compile(r"\s*(?P<dest_ce>\S+)\s+(?P<dest_queue>\S+)\s+(?P<count>\d+)")
found_scheds = all_sched()
for sched in found_scheds:
## select the names for states that are considered waiting for this scheduler
if waiting_name.has_key(sched):
wnames = waiting_name[sched]
else:
wnames = waiting_name['default']
## select the names for states that are considered running for this scheduler
if running_name.has_key(sched):
rnames = running_name[sched]
else:
rnames = running_name['default']
## get the number of waiting jobs
sql = \
"select destination,count(*) from bl_runningjob where scheduler='"\
+ sched + "' and "
for name in wnames:
sql += 'status_scheduler = \''+name+'\' or '
sql = re.sub(' or $', '', sql)
sql += 'group by destination;'
answ = bossLiteDB.select( sql )
for ce in answ:
if act_re.search(ce[0]):
reggy = ce[0]
if not ce_act.has_key(reggy):
ce_act[reggy] = {'Idle':0, 'Running':0}
## prepare regexp for ce matching
ce_act[reggy]['Idle'] += int(ce[1])
## get the number of running jobs
sql = \
"select destination,count(*) from bl_runningjob where scheduler='"\
+ sched + "' and "
for name in rnames:
sql += 'status_scheduler = \'' + name + '\' or '
sql = re.sub(' or $', '', sql)
sql += 'group by destination;'
answ = bossLiteDB.select( sql )
for ce in answ:
if act_re.search(ce[0]):
reggy = ce[0]
if not ce_act.has_key(reggy):
ce_act[reggy] = {'Idle':0, 'Running':0}
## prepare regexp for ce matching
ce_act[reggy]['Running'] += int(ce[1])
return ce_act
def all_sched():
sql = 'select scheduler from bl_runningjob group by scheduler where scheduler is not NULL;'
bossLiteDB = BossLiteDB( 'MySQL', dbConfig )
answ = bossLiteDB.select( sql )
if answ is None :
return []
return [ line[0] for line in answ ]
def taskCount (self):
"""
__taskCount__
count the tasks in the table.
helps in making unique name for tasks
"""
sql="select count(id) from bl_task"
bossLiteDB = BossLiteDB( 'MySQL', dbConfig )
taskTotal = bossLiteDB.selectOne( sql )
return taskTotal
def activeJobs():
## returns a simple dictionary with total number of jobs, running and waiting
bossLiteDB = BossLiteDB( 'MySQL', dbConfig )
act_re = re.compile(r"\s*(?P<count>\d+)")
ce_act = {'Idle':0, 'Running':0}
found_scheds = all_sched()
for sched in found_scheds:
## select the names for states that are considered waiting for this scheduler
if waiting_name.has_key(sched):
wnames = waiting_name[sched]
else:
wnames = waiting_name['default']
## select the names for states that are considered running for this scheduler
if running_name.has_key(sched):
rnames = running_name[sched]
else:
rnames = running_name['default']
## get the number of waiting jobs
sql = "select count(*) from bl_runningjob where scheduler='" \
+ sched + "' and "
for name in wnames:
sql += 'status_scheduler = \'' + name + '\' or '
sql = re.sub(' or $', '', sql)
sql += 'group by destination;'
bossLiteDB = BossLiteDB( 'MySQL', dbConfig )
answ = bossLiteDB.select( sql )
for count in answ :
if act_re.search(count[0]):
ce_act['Idle'] += int(act_re.search(count[0]).group('count'))
## get the number of running jobs
sql = "select count(*) from bl_runningjob where scheduler='" \
+ sched + "' and "
for name in rnames:
sql += 'status_scheduler = \'' + name + '\' or '
sql = re.sub(' or $', '', sql)
sql += 'group by destination;'
answ = bossLiteDB.select( sql )
for count in answ :
if act_re.search(count[0]):
ce_act['Running'] += int(act_re.search(count[0]).group('count'))
return ce_act
def __init__(self, database, dbConfig=None, pool=None, makePool=False):
"""
initialize the API instance
- database can be both MySQl or SQLite
- dbConfig can be a dictionary with the format
{'dbName':'BossLiteDB',
'host':'localhost',
'user':'******',
'passwd':'BossLitePass',
'socketFileLocation':'/var/run/mysql/mysql.sock',
'portNr':'',
'refreshPeriod' : 4*3600 ,
'maxConnectionAttempts' : 5,
'dbWaitingTime' : 10
}
Passing only dbConfig, a SafeSession will be used for db connection.
Passing a pool or setting makePool, a pool of SafeSession (SafePool)
will be used, enabling a better multithread usage
"""
if database == "WMCore":
from ProdCommon.BossLite.API.BossLiteDBWMCore import BossLiteDBWMCore
self.bossLiteDB = BossLiteDBWMCore(database, dbConfig=dbConfig)
elif pool is None and makePool == False:
self.bossLiteDB = BossLiteDB(database, dbConfig=dbConfig)
else:
from ProdCommon.BossLite.API.BossLitePoolDB import BossLitePoolDB
self.bossLiteDB = BossLitePoolDB( database, dbConfig=dbConfig, \
pool=pool )
self.db = None
def __call__(self):
"""
_operator()_
Query BossLite Here....
"""
### config = loadProdAgentConfiguration()
### BOSSconfig = config.getConfig("BOSS")
### bossCfgDir = BOSSconfig['configDir']
sqlStr1 = \
"""
select count(bl_job.id) from bl_job,bl_runningjob where bl_runningjob.job_id=bl_job.job_id and bl_runningjob.task_id=bl_job.task_id and bl_job.name not like '%merge%' and bl_job.name not like '%CleanUp%' and bl_job.name not like '%LogCollect%' and bl_runningjob.status not in ('E','C','A','SD') and bl_runningjob.closed='N';
"""
sqlStr2 = \
"""
select count(bl_job.id) from bl_job,bl_runningjob where bl_runningjob.job_id=bl_job.job_id and bl_runningjob.task_id=bl_job.task_id and bl_job.name like '%merge%' and bl_runningjob.status not in ('E','C','A','SD') and bl_runningjob.closed='N';
"""
sqlStr3 = \
"""
select count(bl_job.id) from bl_job,bl_runningjob where bl_runningjob.job_id=bl_job.id and bl_runningjob.task_id=bl_job.task_id and bl_job.name like '%CleanUp%' and bl_runningjob.status not in ('E','C','A','SD') and bl_runningjob.closed='N';
"""
sqlStr4 = \
"""
select count(bl_job.id) from bl_job,bl_runningjob where bl_runningjob.job_id=bl_job.id and bl_runningjob.task_id=bl_job.task_id and bl_job.name like '%LogCollect%' and bl_runningjob.status not in ('E','C','A','SD') and bl_runningjob.closed='N';
"""
### processingout=commands.getoutput("bossAdmin SQL -query \"%s\" -c %s"%(sqlStr1,bossCfgDir))
### numProcessing=long(processingout.strip().split('\n')[1])
###
### mergeout=commands.getoutput("bossAdmin SQL -query \"%s\" -c %s"%(sqlStr2,bossCfgDir))
### numMerge=long(mergeout.strip().split('\n')[1])
###
### cleanout=commands.getoutput("bossAdmin SQL -query \"%s\" -c %s"%(sqlStr3,bossCfgDir))
### numClean=long(cleanout.strip().split('\n')[1])
bossLiteDB = BossLiteDB( 'MySQL', dbConfig )
numProcessing = bossLiteDB.selectOne( sqlStr1 )
numMerge = bossLiteDB.selectOne( sqlStr2 )
numClean = bossLiteDB.selectOne( sqlStr3 )
numCollect = bossLiteDB.selectOne( sqlStr4 )
#
#BOSSdbConfig = dbConfig
#BOSSdbConfig['dbName'] = "%s_BOSS"%(dbConfig['dbName'],)
#
#Session.set_database(BOSSdbConfig)
#Session.connect()
#Session.start_transaction()
#
#Session.execute(sqlStr1)
#numProcessing = Session.fetchone()[0]
#Session.execute(sqlStr2)
#numMerge = Session.fetchone()[0]
#Session.close_all()
total = numProcessing + numMerge
self['Total'] = total
self['Processing'] = numProcessing
self['Merge'] = numMerge
self['CleanUp'] = numClean
self['LogCollect'] = numCollect
return
#!/usr/bin/env python
from ProdCommon.BossLite.API.BossLiteDB import BossLiteDB
from ProdCommon.BossLite.Monitoring.Monitoring import Monitoring
from ProdAgentDB.Config import defaultConfig as dbConfig
bossSession = BossLiteDB('MySQL', dbConfig)
dbClass = Monitoring(bossSession)
print 'Exit codes per site'
print dbClass.exitCodes()
print ''
print 'Active status'
print dbClass.activeStatus('days', 365, 1228384986)
print ''
print 'Destination'
print dbClass.destination('hours', 12)