def testUnbalancedJobSchedulerIter(self):
"""
testUnbalancedJobSchedulerIter(self):
With an unbalanced load on the processors, each processor still gets the same number of hits
"""
global me
numProcessors = 5
loopCount = 3
dbtestutil.fillProcessorTable(self.connection.cursor(),numProcessors)
self.connection.commit()
m = monitor.Monitor(me.config)
dbCon,dbCur = m.getDatabaseConnectionPair()
try:
dbtestutil.addSomeJobs(dbCur,{1:12},logger=me.logger)
iter = m.unbalancedJobSchedulerIter(dbCur)
num = 0
hits = dict(((1+x,0) for x in range (numProcessors)))
for id in iter:
num += 1
hits[int(id)] += 1
if num >= loopCount*numProcessors: break
for i in range(numProcessors):
assert hits[i+1] == loopCount, 'expected %d for processor %d, but got %d'%(loopCount,i+1,hits[i+1])
finally:
m.databaseConnectionPool.cleanup()
def testSetPriority_Jobs():
global me
cursor = me.connection.cursor()
try:
dbtu.fillProcessorTable(cursor,3,stamp=dt.datetime(2008,3,4,5,6,7,tzinfo=UTC))
cursor.execute("SELECT id FROM processors")
me.connection.commit()
counts = dict((x[0],x[0]) for x in cursor.fetchall())
dbtu.addSomeJobs(cursor,counts,logger)
cursor.execute("SELECT id FROM jobs")
me.connection.commit()
jobIds = [x[0] for x in cursor.fetchall()]
half = len(jobIds)/2
expectPri = jobIds[:half]
expectNon = jobIds[half:]
dbtu.setPriority(cursor,expectPri)
cursor.execute("SELECT id FROM jobs WHERE priority > 0 ORDER BY id")
gotPri = [x[0] for x in cursor.fetchall()]
cursor.execute("SELECT id FROM jobs WHERE priority = 0 ORDER BY id")
gotNon = [x[0] for x in cursor.fetchall()]
me.connection.commit()
assert expectPri == gotPri
assert expectNon == gotNon
finally:
jt = db_schema.JobsTable(logger)
jt.drop(cursor)
jt.create(cursor)
pt = db_schema.ProcessorsTable(logger)
pt.drop(cursor)
pt.create(cursor)
me.connection.commit()
def testJobScheduleIter_StartUnbalanced(self):
"""
testJobScheduleIter_StartUnbalanced(self):
Assure that an unbalanced start eventually produces balanced result
"""
numProcessors = 5
dbtestutil.fillProcessorTable(self.connection.cursor(),numProcessors)
self.connection.commit()
m = monitor.Monitor(me.config)
dbCon,dbCur = m.getDatabaseConnectionPair()
try:
dbtestutil.addSomeJobs(dbCur,dict([(1+x,1+x) for x in range(numProcessors)]),logger=me.logger)
iter = m.jobSchedulerIter(dbCur)
num = 0
hits = dict(((1+x,0) for x in range (numProcessors)))
for id in iter:
num += 1
hits[int(id)] += 1
me.logger.debug('HIT on %d: %d'%(id,hits[id]))
if num >= 3*numProcessors: break
for i in range(numProcessors):
assert hits[i+1] == 5 - i, 'Expected num hits to be count down sequence from 5 to 1, but at idx %d, got %d'%(i+1,hits[i+1])
me.logger.debug('ONE: At index %d, got count %d'%(i+1, hits[i+1]))
finally:
m.databaseConnectionPool.cleanup()
def testAddSomeJobs():
global me
cursor = me.connection.cursor()
cursor.execute("SELECT id from processors")
me.connection.commit()
jdMap = {1:1,2:2,3:3,4:0}
xdMap = {1:set(),2:set(),3:set(),4:set()}
gdMap = {1:set(),2:set(),3:set(),4:set()}
data = _makeJobDetails(jdMap)
for d in data:
xdMap[d[2]].add(d)
try:
dbtu.fillProcessorTable(cursor,3,logger=logger)
cursor.execute("SELECT id from processors")
me.connection.commit()
addedJobs = dbtu.addSomeJobs(cursor,jdMap)
me.connection.commit()
assert data == addedJobs
cursor.execute("SELECT pathname,uuid,owner FROM jobs ORDER BY OWNER ASC")
me.connection.commit()
data2 = cursor.fetchall()
assert len(data) == len(data2)
for d in data2:
gdMap[d[2]].add(d)
assert xdMap == gdMap
finally:
pt = db_schema.ProcessorsTable(logger)
pt.drop(cursor)
pt.create(cursor)
cursor.execute("DELETE from jobs")
me.connection.commit()
def testAddSomeJobs():
global me
cursor = me.connection.cursor()
cursor.execute("SELECT id from processors")
me.connection.commit()
jdMap = {1: 1, 2: 2, 3: 3, 4: 0}
xdMap = {1: set(), 2: set(), 3: set(), 4: set()}
gdMap = {1: set(), 2: set(), 3: set(), 4: set()}
data = _makeJobDetails(jdMap)
for d in data:
xdMap[d[2]].add(d)
try:
dbtu.fillProcessorTable(cursor, 3, logger=logger)
cursor.execute("SELECT id from processors")
me.connection.commit()
addedJobs = dbtu.addSomeJobs(cursor, jdMap)
me.connection.commit()
assert data == addedJobs
cursor.execute(
"SELECT pathname,uuid,owner FROM jobs ORDER BY OWNER ASC")
me.connection.commit()
data2 = cursor.fetchall()
assert len(data) == len(data2)
for d in data2:
gdMap[d[2]].add(d)
assert xdMap == gdMap
finally:
pt = db_schema.ProcessorsTable(logger)
pt.drop(cursor)
pt.create(cursor)
cursor.execute("DELETE from jobs")
me.connection.commit()
def testSetPriority_PriorityJobs():
global me
cursor = me.connection.cursor()
try:
dbtu.fillProcessorTable(cursor,
3,
stamp=dt.datetime(2008,
3,
4,
5,
6,
7,
tzinfo=UTC))
cursor.execute("SELECT id FROM processors")
counts = dict((x[0], x[0]) for x in cursor.fetchall())
dbtu.addSomeJobs(cursor, counts, logger)
cursor.execute("SELECT id,uuid FROM jobs")
me.connection.commit()
data = cursor.fetchall()
jobIds = [x[0] for x in data]
jobUuids = [x[1] for x in data]
half = len(jobIds) / 2
expect1Pri = jobIds[:half]
expect2Pri = jobIds[half:]
expect1Uuid = sorted(jobUuids[:half])
expect2Uuid = sorted(jobUuids[half:])
dbtu.setPriority(cursor, expect1Pri, 'priority_jobs_1')
dbtu.setPriority(cursor, expect2Pri, 'priority_jobs_2')
sql = "SELECT uuid from %s ORDER BY uuid"
cursor.execute(sql % 'priority_jobs_1')
got1Uuid = [x[0] for x in cursor.fetchall()]
cursor.execute(sql % 'priority_jobs_2')
got2Uuid = [x[0] for x in cursor.fetchall()]
me.connection.commit()
assert expect1Uuid == got1Uuid
assert expect2Uuid == got2Uuid
finally:
jt = db_schema.JobsTable(logger)
jt.drop(cursor)
jt.create(cursor)
pt = db_schema.ProcessorsTable(logger)
pt.drop(cursor)
pt.create(cursor)
me.connection.commit()
def testSetPriority_Jobs():
global me
cursor = me.connection.cursor()
try:
dbtu.fillProcessorTable(cursor,
3,
stamp=dt.datetime(2008,
3,
4,
5,
6,
7,
tzinfo=UTC))
cursor.execute("SELECT id FROM processors")
me.connection.commit()
counts = dict((x[0], x[0]) for x in cursor.fetchall())
dbtu.addSomeJobs(cursor, counts, logger)
cursor.execute("SELECT id FROM jobs")
me.connection.commit()
jobIds = [x[0] for x in cursor.fetchall()]
half = len(jobIds) / 2
expectPri = jobIds[:half]
expectNon = jobIds[half:]
dbtu.setPriority(cursor, expectPri)
cursor.execute("SELECT id FROM jobs WHERE priority > 0 ORDER BY id")
gotPri = [x[0] for x in cursor.fetchall()]
cursor.execute("SELECT id FROM jobs WHERE priority = 0 ORDER BY id")
gotNon = [x[0] for x in cursor.fetchall()]
me.connection.commit()
assert expectPri == gotPri
assert expectNon == gotNon
finally:
jt = db_schema.JobsTable(logger)
jt.drop(cursor)
jt.create(cursor)
pt = db_schema.ProcessorsTable(logger)
pt.drop(cursor)
pt.create(cursor)
me.connection.commit()
def testSetPriority_PriorityJobs():
global me
cursor = me.connection.cursor()
try:
dbtu.fillProcessorTable(cursor,3,stamp=dt.datetime(2008,3,4,5,6,7,tzinfo=UTC))
cursor.execute("SELECT id FROM processors")
counts = dict((x[0],x[0]) for x in cursor.fetchall())
dbtu.addSomeJobs(cursor,counts,logger)
cursor.execute("SELECT id,uuid FROM jobs")
me.connection.commit()
data = cursor.fetchall()
jobIds = [x[0] for x in data]
jobUuids = [x[1] for x in data]
half = len(jobIds)/2
expect1Pri = jobIds[:half]
expect2Pri = jobIds[half:]
expect1Uuid = sorted(jobUuids[:half])
expect2Uuid = sorted(jobUuids[half:])
dbtu.setPriority(cursor,expect1Pri,'priority_jobs_1')
dbtu.setPriority(cursor,expect2Pri,'priority_jobs_2')
sql = "SELECT uuid from %s ORDER BY uuid"
cursor.execute(sql%'priority_jobs_1')
got1Uuid = [x[0] for x in cursor.fetchall()]
cursor.execute(sql%'priority_jobs_2')
got2Uuid = [x[0] for x in cursor.fetchall()]
me.connection.commit()
assert expect1Uuid == got1Uuid
assert expect2Uuid == got2Uuid
finally:
jt = db_schema.JobsTable(logger)
jt.drop(cursor)
jt.create(cursor)
pt = db_schema.ProcessorsTable(logger)
pt.drop(cursor)
pt.create(cursor)
me.connection.commit()