def testUnbalancedJobSchedulerIter(self):
   """
   testUnbalancedJobSchedulerIter(self):
   With an unbalanced load on the processors, each processor still gets the same number of hits
   """
   global me
   numProcessors = 5
   loopCount = 3
   dbtestutil.fillProcessorTable(self.connection.cursor(),numProcessors)
   self.connection.commit()
   m = monitor.Monitor(me.config)
   dbCon,dbCur = m.getDatabaseConnectionPair()
   try:
     dbtestutil.addSomeJobs(dbCur,{1:12},logger=me.logger)
     iter = m.unbalancedJobSchedulerIter(dbCur)
     num = 0
     hits = dict(((1+x,0) for x in range (numProcessors)))
     for id in iter:
       num += 1
       hits[int(id)] += 1
       if num >= loopCount*numProcessors: break
     for i in range(numProcessors):
       assert hits[i+1] == loopCount, 'expected %d for processor %d, but got %d'%(loopCount,i+1,hits[i+1])
   finally:
     m.databaseConnectionPool.cleanup()
示例#2
0
def testSetPriority_Jobs():
  global me
  cursor = me.connection.cursor()
  try:
    dbtu.fillProcessorTable(cursor,3,stamp=dt.datetime(2008,3,4,5,6,7,tzinfo=UTC))
    cursor.execute("SELECT id FROM processors")
    me.connection.commit()
    counts = dict((x[0],x[0]) for x in cursor.fetchall())
    dbtu.addSomeJobs(cursor,counts,logger)
    cursor.execute("SELECT id FROM jobs")
    me.connection.commit()
    jobIds = [x[0] for x in cursor.fetchall()]
    half = len(jobIds)/2
    expectPri = jobIds[:half]
    expectNon = jobIds[half:]
    dbtu.setPriority(cursor,expectPri)
    cursor.execute("SELECT id FROM jobs WHERE priority > 0 ORDER BY id")
    gotPri = [x[0] for x in cursor.fetchall()]
    cursor.execute("SELECT id FROM jobs WHERE priority = 0 ORDER BY id")
    gotNon = [x[0] for x in cursor.fetchall()]
    me.connection.commit()
    assert expectPri == gotPri
    assert expectNon == gotNon
  finally:
    jt = db_schema.JobsTable(logger)
    jt.drop(cursor)
    jt.create(cursor)
    pt = db_schema.ProcessorsTable(logger)
    pt.drop(cursor)
    pt.create(cursor)
    me.connection.commit()
 def testJobScheduleIter_StartUnbalanced(self):
   """
   testJobScheduleIter_StartUnbalanced(self):
   Assure that an unbalanced start eventually produces balanced result
   """
   numProcessors = 5
   dbtestutil.fillProcessorTable(self.connection.cursor(),numProcessors)
   self.connection.commit()
   m = monitor.Monitor(me.config)
   dbCon,dbCur = m.getDatabaseConnectionPair()
   try:
     dbtestutil.addSomeJobs(dbCur,dict([(1+x,1+x) for x in range(numProcessors)]),logger=me.logger)
     iter = m.jobSchedulerIter(dbCur)
     num = 0
     hits = dict(((1+x,0) for x in range (numProcessors)))
     for id in iter:
       num += 1
       hits[int(id)] += 1
       me.logger.debug('HIT on %d: %d'%(id,hits[id]))
       if num >= 3*numProcessors: break
     for i in range(numProcessors):
       assert hits[i+1] == 5 - i, 'Expected num hits to be count down sequence from 5 to 1, but at idx %d, got %d'%(i+1,hits[i+1])
       me.logger.debug('ONE: At index %d, got count %d'%(i+1, hits[i+1]))
   finally:
     m.databaseConnectionPool.cleanup()
示例#4
0
def testAddSomeJobs():
  global me
  cursor = me.connection.cursor()
  cursor.execute("SELECT id from processors")
  me.connection.commit()
  jdMap = {1:1,2:2,3:3,4:0}
  xdMap = {1:set(),2:set(),3:set(),4:set()}
  gdMap = {1:set(),2:set(),3:set(),4:set()}
  data = _makeJobDetails(jdMap)
  for d in data:
    xdMap[d[2]].add(d)
  try:
    dbtu.fillProcessorTable(cursor,3,logger=logger)
    cursor.execute("SELECT id from processors")
    me.connection.commit()
    addedJobs = dbtu.addSomeJobs(cursor,jdMap)
    me.connection.commit()
    assert data == addedJobs
    cursor.execute("SELECT pathname,uuid,owner FROM jobs ORDER BY OWNER ASC")
    me.connection.commit()
    data2 = cursor.fetchall()
    assert len(data) == len(data2)
    for d in data2:
      gdMap[d[2]].add(d)
    assert xdMap == gdMap
  finally:
    pt = db_schema.ProcessorsTable(logger)
    pt.drop(cursor)
    pt.create(cursor)
    cursor.execute("DELETE from jobs")
    me.connection.commit()
示例#5
0
def testAddSomeJobs():
    global me
    cursor = me.connection.cursor()
    cursor.execute("SELECT id from processors")
    me.connection.commit()
    jdMap = {1: 1, 2: 2, 3: 3, 4: 0}
    xdMap = {1: set(), 2: set(), 3: set(), 4: set()}
    gdMap = {1: set(), 2: set(), 3: set(), 4: set()}
    data = _makeJobDetails(jdMap)
    for d in data:
        xdMap[d[2]].add(d)
    try:
        dbtu.fillProcessorTable(cursor, 3, logger=logger)
        cursor.execute("SELECT id from processors")
        me.connection.commit()
        addedJobs = dbtu.addSomeJobs(cursor, jdMap)
        me.connection.commit()
        assert data == addedJobs
        cursor.execute(
            "SELECT pathname,uuid,owner FROM jobs ORDER BY OWNER ASC")
        me.connection.commit()
        data2 = cursor.fetchall()
        assert len(data) == len(data2)
        for d in data2:
            gdMap[d[2]].add(d)
        assert xdMap == gdMap
    finally:
        pt = db_schema.ProcessorsTable(logger)
        pt.drop(cursor)
        pt.create(cursor)
        cursor.execute("DELETE from jobs")
        me.connection.commit()
示例#6
0
def testSetPriority_PriorityJobs():
    global me
    cursor = me.connection.cursor()
    try:
        dbtu.fillProcessorTable(cursor,
                                3,
                                stamp=dt.datetime(2008,
                                                  3,
                                                  4,
                                                  5,
                                                  6,
                                                  7,
                                                  tzinfo=UTC))
        cursor.execute("SELECT id FROM processors")
        counts = dict((x[0], x[0]) for x in cursor.fetchall())
        dbtu.addSomeJobs(cursor, counts, logger)
        cursor.execute("SELECT id,uuid FROM jobs")
        me.connection.commit()
        data = cursor.fetchall()
        jobIds = [x[0] for x in data]
        jobUuids = [x[1] for x in data]
        half = len(jobIds) / 2
        expect1Pri = jobIds[:half]
        expect2Pri = jobIds[half:]
        expect1Uuid = sorted(jobUuids[:half])
        expect2Uuid = sorted(jobUuids[half:])
        dbtu.setPriority(cursor, expect1Pri, 'priority_jobs_1')
        dbtu.setPriority(cursor, expect2Pri, 'priority_jobs_2')
        sql = "SELECT uuid from %s ORDER BY uuid"
        cursor.execute(sql % 'priority_jobs_1')
        got1Uuid = [x[0] for x in cursor.fetchall()]
        cursor.execute(sql % 'priority_jobs_2')
        got2Uuid = [x[0] for x in cursor.fetchall()]
        me.connection.commit()
        assert expect1Uuid == got1Uuid
        assert expect2Uuid == got2Uuid
    finally:
        jt = db_schema.JobsTable(logger)
        jt.drop(cursor)
        jt.create(cursor)
        pt = db_schema.ProcessorsTable(logger)
        pt.drop(cursor)
        pt.create(cursor)
        me.connection.commit()
示例#7
0
def testSetPriority_Jobs():
    global me
    cursor = me.connection.cursor()
    try:
        dbtu.fillProcessorTable(cursor,
                                3,
                                stamp=dt.datetime(2008,
                                                  3,
                                                  4,
                                                  5,
                                                  6,
                                                  7,
                                                  tzinfo=UTC))
        cursor.execute("SELECT id FROM processors")
        me.connection.commit()
        counts = dict((x[0], x[0]) for x in cursor.fetchall())
        dbtu.addSomeJobs(cursor, counts, logger)
        cursor.execute("SELECT id FROM jobs")
        me.connection.commit()
        jobIds = [x[0] for x in cursor.fetchall()]
        half = len(jobIds) / 2
        expectPri = jobIds[:half]
        expectNon = jobIds[half:]
        dbtu.setPriority(cursor, expectPri)
        cursor.execute("SELECT id FROM jobs WHERE priority > 0 ORDER BY id")
        gotPri = [x[0] for x in cursor.fetchall()]
        cursor.execute("SELECT id FROM jobs WHERE priority = 0 ORDER BY id")
        gotNon = [x[0] for x in cursor.fetchall()]
        me.connection.commit()
        assert expectPri == gotPri
        assert expectNon == gotNon
    finally:
        jt = db_schema.JobsTable(logger)
        jt.drop(cursor)
        jt.create(cursor)
        pt = db_schema.ProcessorsTable(logger)
        pt.drop(cursor)
        pt.create(cursor)
        me.connection.commit()
示例#8
0
def testSetPriority_PriorityJobs():
  global me
  cursor = me.connection.cursor()
  try:
    dbtu.fillProcessorTable(cursor,3,stamp=dt.datetime(2008,3,4,5,6,7,tzinfo=UTC))
    cursor.execute("SELECT id FROM processors")
    counts = dict((x[0],x[0]) for x in cursor.fetchall())
    dbtu.addSomeJobs(cursor,counts,logger)
    cursor.execute("SELECT id,uuid FROM jobs")
    me.connection.commit()
    data = cursor.fetchall()
    jobIds = [x[0] for x in data]
    jobUuids = [x[1] for x in data]
    half = len(jobIds)/2
    expect1Pri = jobIds[:half]
    expect2Pri = jobIds[half:]
    expect1Uuid = sorted(jobUuids[:half])
    expect2Uuid = sorted(jobUuids[half:])
    dbtu.setPriority(cursor,expect1Pri,'priority_jobs_1')
    dbtu.setPriority(cursor,expect2Pri,'priority_jobs_2')
    sql = "SELECT uuid from %s ORDER BY uuid"
    cursor.execute(sql%'priority_jobs_1')
    got1Uuid = [x[0] for x in cursor.fetchall()]
    cursor.execute(sql%'priority_jobs_2')
    got2Uuid = [x[0] for x in cursor.fetchall()]
    me.connection.commit()
    assert expect1Uuid == got1Uuid
    assert expect2Uuid == got2Uuid
  finally:
    jt = db_schema.JobsTable(logger)
    jt.drop(cursor)
    jt.create(cursor)
    pt = db_schema.ProcessorsTable(logger)
    pt.drop(cursor)
    pt.create(cursor)
    me.connection.commit()