예제 #1
0
 def test_init_configure (self):
     assert os.path.exists(CONFIG_FILE)
     system = Simulator()
     assert not system.partitions
     system = Simulator(config_file=CONFIG_FILE)
     assert system._partitions
     assert len(system._partitions) > 0
예제 #2
0
 def setup(self):
     self.slp = TimingServiceLocator()
     self.system = Simulator(config_file="simulator.xml")
     self.system_thr = ComponentProgressThread(self.system)
     self.system_thr.start()
     self.scriptm = ScriptManager()
     self.scriptm_thr = ComponentProgressThread(self.scriptm)
     self.scriptm_thr.start()
     self.qm = QueueManager()
     self.qm_thr = ComponentProgressThread(self.qm)
     self.qm_thr.start()
예제 #3
0
    def setup (self):
        TestComponent.setup(self)

        assert os.path.exists(CONFIG_FILE)
        self.system = Simulator(config_file=CONFIG_FILE)
        assert self.system._partitions
        assert len(self.system._partitions) > 0

        part_names = self.system._partitions.keys()
        for part_name in part_names:
            partitions = self.system.add_partitions([{'name':part_name}])
            assert len(partitions) == 1
            partitions = self.system.set_partitions([{'tag':"partition", 'name':part_name}], {'functional':True, 'scheduled':True})
            assert len(partitions) == 1
            idle_partitions = self.system.get_partitions([{'state':"idle"}])
            assert part_name in [p.name for p in idle_partitions]

        partition_sizes = [p.size for p in self.system._partitions.itervalues()]
        partition_sizes.sort()
        self.min_size = partition_sizes[0]
        self.median_size = partition_sizes[len(partition_sizes) / 2]
        self.max_size = partition_sizes[-1]
예제 #4
0
 def test_configure (self):
     assert os.path.exists(CONFIG_FILE)
     system = Simulator()
     assert not system._partitions
     system.configure(CONFIG_FILE)
     assert system._partitions
예제 #5
0
class TestSimulator (TestComponent):
    
    def setup (self):
        TestComponent.setup(self)

        assert os.path.exists(CONFIG_FILE)
        self.system = Simulator(config_file=CONFIG_FILE)
        assert self.system._partitions
        assert len(self.system._partitions) > 0

        part_names = self.system._partitions.keys()
        for part_name in part_names:
            partitions = self.system.add_partitions([{'name':part_name}])
            assert len(partitions) == 1
            partitions = self.system.set_partitions([{'tag':"partition", 'name':part_name}], {'functional':True, 'scheduled':True})
            assert len(partitions) == 1
            idle_partitions = self.system.get_partitions([{'state':"idle"}])
            assert part_name in [p.name for p in idle_partitions]

        partition_sizes = [p.size for p in self.system._partitions.itervalues()]
        partition_sizes.sort()
        self.min_size = partition_sizes[0]
        self.median_size = partition_sizes[len(partition_sizes) / 2]
        self.max_size = partition_sizes[-1]
    
    def test_init_configure (self):
        assert os.path.exists(CONFIG_FILE)
        system = Simulator()
        assert not system.partitions
        system = Simulator(config_file=CONFIG_FILE)
        assert system._partitions
        assert len(system._partitions) > 0

    def test_configure (self):
        assert os.path.exists(CONFIG_FILE)
        system = Simulator()
        assert not system._partitions
        system.configure(CONFIG_FILE)
        assert system._partitions
    
    def _find_location(self, jobid, size):
        job_location_args = [{
            'jobid': jobid,
            'nodes': self.median_size,
            'queue': "default",
            'utility_score': 1,
            'threshold': 0,
            'walltime': 1,
            'attrs': {}}]
        locations = self.system.find_job_location(job_location_args, [])
        assert locations.has_key(jobid)
        return locations[jobid]

    def test_reserve_partition (self):
        jobid = 3002
        location = self._find_location(jobid, self.max_size / 2)
        partition = self.system._partitions[location[0]]
        reserved = self.system.reserve_partition(partition.name)
        assert reserved
        assert partition.state == "busy"
        for parent in self.system._partitions.q_get([{'name':parent} for parent in partition.parents]):
            assert parent.state[0:7] == "blocked"
        for child in self.system._partitions.q_get([{'name':child} for child in partition.children]):
            assert child.state[0:7] == "blocked"
    
    def test_release_partition (self):
        jobid = 3003
        idle_partitions_before = self.system.get_partitions([{'state':"idle"}])
        location = self._find_location(jobid, self.median_size)
        partition = self.system._partitions[location[0]]
        reserved = self.system.reserve_partition(partition.name)
        assert reserved
        idle_partitions_after = self.system.get_partitions([{'state':"idle"}])
        assert len(idle_partitions_after) != len(idle_partitions_before)
        self.system.release_partition(partition.name)
        idle_partitions_after = self.system.get_partitions([{'state':"idle"}])
        assert len(idle_partitions_after) == len(idle_partitions_before)

    def _add_pg(self, jobid):
        location = self._find_location(jobid, self.median_size)
        self.system.add_process_groups([dict(
            id = "*",
            jobid = jobid,
            size = self.median_size,
            mode = "vn",
            executable = "/bin/ls",
            location = location,
            cwd = os.getcwd(),
            inputfile = "infile",
            outputfile = "outfile",
            errorfile = "errfile",
            user = os.getlogin(),
            args = [],
        )])

    def test_add_process_groups (self):
        jobid = 3004
        self._add_pg(jobid)

    def test_get_process_groups (self):
        jobid = 3005
        process_groups = self.system.get_process_groups([{'id':"*"}])
        assert not process_groups
        self._add_pg(jobid)
        process_groups = self.system.get_process_groups([{'id':"*"}])
        assert process_groups
        assert process_groups[0].jobid == jobid

    @timeout(Simulator.MAX_RUN_TIME + 30)
    def test_run_process_groups (self):
        jobid = 3006
        self._add_pg(jobid)
        process_groups = self.system.get_process_groups([{'id':"*"}])
        assert process_groups
        assert len(process_groups) == 1
        id = process_groups[0].id
        assert process_groups[0].jobid == jobid
        while True:
            process_groups = self.system.wait_process_groups([{'id': id}])
            if len(process_groups) > 0:
                assert len(process_groups) == 1
                assert process_groups[0].id == id
                assert process_groups[0].jobid == jobid
                break
            time.sleep(1)
    
    def test_signal_process_groups (self):
        jobid = 3007
        self._add_pg(jobid)
        for signal in ["SIGINT", "SIGKILL"]:
            process_groups = self.system.signal_process_groups([{'id':"*"}], signal)
            assert len(process_groups) == 1
            assert process_groups[0].jobid == jobid
            assert process_groups[0].signals[-1] == signal
예제 #6
0
    def __init__(self, *args, **kwargs):

        print "kwargs= ",  kwargs

        #initialize partitions
        Simulator.__init__(self, *args, **kwargs)
        partnames = self._partitions.keys()
        self.init_partition(partnames)
        self.part_size_list = []

        for part in self.partitions.itervalues():
            if int(part.size) not in self.part_size_list:
                self.part_size_list.append(int(part.size))
        self.part_size_list.sort()

        #get command line parameters
        self.FAILURE_FREE = True
        self.FRACTION = kwargs.get("fraction", 1)
        self.workload_file =  kwargs.get("workload")
        self.output_log = kwargs.get("outputlog")
        self.failure_log = kwargs.get('failurelog')

        self.weibull = kwargs.get('weibull')
        if self.weibull:
            self.SCALE = float(kwargs.get('scale'))
            if self.SCALE == 0:
                self.SCALE = default_SCALE
            self.SHAPE = float(kwargs.get('shape'))
            if self.SHAPE == 0:
                self.SHAPE = default_SHAPE

        self.fault_aware = kwargs.get('faultaware')
        self.SENSITIVITY = default_SENSITIVITY
        self.SPECIFICITY = default_SPECIFICITY
        if self.fault_aware:
            self.SENSITIVITY = float(kwargs.get('sensitivity', default_SENSITIVITY))
            self.SPECIFICITY = float(kwargs.get('specificity', defalt_SPECIFICITY))

        if self.failure_log or self.weibull:
            self.FAILURE_FREE = False

        #initialize time stamps and job queues
        #time stamp format: ('EVENT', 'time_stamp_date', time_stamp_second, {'job_id':str(jobid), 'location':[partition1, partition2,...]})
        self.time_stamps = [('I', '0', 0, {})]
        self.cur_time_index = 0
        self.queues = SimQueueDict(policy=kwargs['policy'])
        self.init_queues()
        self.visible_jobs = []

        #initialize failures
        self.failure_dict = {}
        if not self.FAILURE_FREE:
            if self.failure_log:
                #if specified failure log, use log trace failure
                self.inject_failures()
            elif self.weibull:
                #else MAKE failures by Weibull distribution
                self.make_failures()

        #initialize PBS-style logger
        self.pbslog = PBSlogger(self.output_log)

        #initialize debug logger
        self.dbglog = PBSlogger(self.output_log+"-debug")

        #finish tag
        self.finished = False

        #tag for controlling time stamp increment
        self.increment_tag = True

        #register local alias "system" for this component
        local_components["system"] = self
        print "Simulation starts:"
예제 #7
0
 def setup (self):
     self.system = Simulator(config_file="simulator.xml")
     self.system.add_partitions([{'name':"ANL-R00-1024"}])
     self.test_partition = self.system.partitions.values()[0]
예제 #8
0
class TestReservation (object):

    def setup (self):
        self.system = Simulator(config_file="simulator.xml")
        self.system.add_partitions([{'name':"ANL-R00-1024"}])
        self.test_partition = self.system.partitions.values()[0]

    def teardown(self):
        Cobalt.Proxy.local_components.clear()      

    def test_required_name (self):
        spec = {'start':0, 'duration':0}
        try:
            reservation = Reservation(spec)
        except DataCreationError:
            pass
        else:
            assert not "didn't require name"
        spec['name'] = "my_reservation"
        try:
            reservation = Reservation(spec)
        except DataCreationError:
            assert not "failed with name specified"
    
    def test_init (self):
        reservation = Reservation({'name':"mine", 'start':0, 'duration':0})
        assert reservation.tag == "reservation"
        assert reservation.name == "mine"
        assert reservation.start == 0
        assert reservation.duration == 0
        assert reservation.cycle is None
        assert reservation.users == ""
        assert reservation.partitions == ""

    def test_update (self):
        reservation = Reservation({'name':"mine", 'start':100, 'duration':10})
       
        try: 
            reservation.update({'users':"newuser"})                   
        except ComponentLookupError:
            cqm = QueueManager()      
            reservation.update({'users':"newuser"})

        assert reservation.users == "newuser"
        
        reservation = Reservation({'name':"mine", 'start':100, 'duration':10, 'users':"group"})
        reservation.update({'users':"newuser"})
        assert not reservation.users == "group"
        assert reservation.users == "newuser"


    def test_active (self):
        reservation = Reservation({'name':"mine", 'start':100, 'duration':50})
        for current_time in xrange(1, 100):
            assert not reservation.is_active(current_time)
        for current_time in xrange(100, 150):
            assert reservation.is_active(current_time)
        for current_time in xrange(151, 250):
            assert not reservation.is_active(current_time)

    def test_active_cyclic (self):
        reservation = Reservation({'name':"mine", 'start':100, 'duration':10, 'cycle':50})
        assert not reservation.is_active(99)
        assert reservation.is_active(100)
        assert reservation.is_active(109)
        assert not reservation.is_active(111)
        assert not reservation.is_active(149)
        assert reservation.is_active(150)
        assert reservation.is_active(159)
        assert not reservation.is_active(161)
    
    def test_overlaps (self):
        reservation = Reservation({'name':"mine", 'start':100, 'duration':50, 'partitions':"ANL-R00-1024"})
        assert not reservation.overlaps(partition=self.test_partition, start=0, duration=99)
        assert reservation.overlaps(partition=self.test_partition, start=0, duration=100)
        assert reservation.overlaps(partition=self.test_partition, start=0, duration=151)
        assert reservation.overlaps(partition=self.test_partition, start=99, duration=1)   
        assert reservation.overlaps(partition=self.test_partition, start=99, duration=50)
        assert reservation.overlaps(partition=self.test_partition, start=149, duration=1)
        assert not reservation.overlaps(partition=self.test_partition, start=150, duration=100)
 
        #check different partition than constructed
        reservation = Reservation({'name':"mine", 'start':100, 'duration':50, 'partitions':"anotherpartion"})
        assert not reservation.overlaps(partition=self.test_partition, start=0, duration=100)
       
    def test_overlaps_cyclic (self):
        reservation = Reservation({'name':"mine", 'start':100, 'duration':10, 'cycle':50, 'partitions':"ANL-R00-1024"})
        assert not reservation.overlaps(partition=self.test_partition, start=0, duration=99)
        assert reservation.overlaps(partition=self.test_partition, start=0, duration=100)
        assert reservation.overlaps(partition=self.test_partition, start=99, duration=1)
        assert reservation.overlaps(partition=self.test_partition, start=99, duration=10)
        assert reservation.overlaps(partition=self.test_partition, start=101, duration=1)
        assert reservation.overlaps(partition=self.test_partition, start=109, duration=1)
        assert not reservation.overlaps(partition=self.test_partition, start=110, duration=39)
        assert reservation.overlaps(partition=self.test_partition, start=90, duration=100)
        

    def test_job_within_reservation (self):
        # past reservation
        reservation = Reservation({'name':"mine", 'start':100, 'duration':3600, 'partitions':"ANLR00", 'queue':"default"})
        j = Job(5, "default")
        assert not reservation.job_within_reservation(j)
        j = Job(70, "default")
        assert not reservation.job_within_reservation(j)
        
        # current reservation
        reservation = Reservation({'name':"mine", 'start':time.time(), 'duration':3600, 'partitions':"ANLR00", 'queue':"default"})
        j = Job(5, "default")
        assert reservation.job_within_reservation(j)
        j = Job(70, "default")
        assert not reservation.job_within_reservation(j)
        
        # future reservation
        reservation = Reservation({'name':"mine", 'start':time.time() + 3600, 'duration':3600, 'partitions':"ANLR00", 'queue':"default"})
        j = Job(5, "default")
        assert not reservation.job_within_reservation(j)
        j = Job(40, "default")
        assert not reservation.job_within_reservation(j)
        j = Job(70, "default")
        assert not reservation.job_within_reservation(j)

    def test_job_within_reservation_cyclic (self):
        reservation = Reservation({'name':"mine", 'start':time.time()-3000, 'duration':3600, 'cycle':4000, 'partitions':"ANLR00", 'queue':"default"})
        # jobs ends inside the reservation
        j = Job(6, "default")
        assert reservation.job_within_reservation(j)
        # job ends in the "dead zone"
        j = Job(12, "default")
        assert not reservation.job_within_reservation(j)
        # job ends the next time the reservation is active
        j = Job(50, "default")
        assert not reservation.job_within_reservation(j)
        # job lasts longer than the reservation
        j = Job(100, "default")
        assert not reservation.job_within_reservation(j)
        # queue doesn't exist
        j = Job(0,"notaqueue")
        assert not reservation.job_within_reservation(j)

    def test_is_over(self):
        reservation = Reservation({'name':"mine", 'start':100, 'duration':10, 'cycle':50})
        assert not reservation.is_over()
        reservation = Reservation({'name':"mine", 'start':100, 'duration':10})
        assert reservation.is_over()
        reservation = Reservation({'name':"mine", 'start':time.time(), 'duration':1000})
        assert not reservation.is_over()

    def test_q_add (self):     #|finish| assert logger
        reservationdict = ReservationDict()  
        reservationdict['res1'] = Reservation({'name':"mine1", 'start':100, 'duration':10})
        reservationdict['res2'] = Reservation({'name':"mine2", 'start':200, 'duration':10})
        resqueue = {'queue':"newqueue", 'name':"newres", 'start':10, 'duration':20}

        try:
            reslist = reservationdict.q_add([resqueue])
        except ComponentLookupError:
            cqm = QueueManager() 

        #new reservation created
        reslist = reservationdict.q_add([resqueue])        

        assert reservationdict['newres'].queue == "newqueue"
        assert reservationdict['newres'].name == "newres"
        assert len(reslist) == 1
        assert reslist[0].name == "newres"
        assert reslist[0].queue == "newqueue"
        assert reslist[0].createdQueue

        #reservation already exists
        try:
            new = reservationdict.q_add([resqueue])
        except ReservationError:
            pass

        #change queue for existing reservations
        res1 = {'queue':"newqueue1", 'name':"mine1", 'start':10, 'duration':20}
        res2 = {'queue':"newqueue2", 'name':"mine2", 'start':10, 'duration':20}
        reslist = reservationdict.q_add([res1,res2])
        assert reservationdict['mine1'].queue == "newqueue1"
        assert reservationdict['mine2'].queue == "newqueue2"

        #queue already exists
        cqm.add_queues([{'tag':"queue", 'name':"default"}])     
        resqueue = {'queue':"default", 'name':"other", 'start':10, 'duration':20}
        reslist = reservationdict.q_add([resqueue])

    def test_q_del (self): #|alert| there is not ComponenentLookup exception for the queuemanger
        cqm = QueueManager()
        reservationdict = ReservationDict()
        reservationdict['res1'] = Reservation({'queue':"myqueue", 'name':"mine1", 'start':100, 'duration':10})
        reservationdict['res2'] = Reservation({'name':"mine2", 'start':200, 'duration':10})
        resqueue = {'queue':"myqueue", 'name':"mine1", 'start':10, 'duration':20}
        
        reslist = reservationdict.q_del([resqueue])