def test_init_configure (self):
assert os.path.exists(CONFIG_FILE)
system = Simulator()
assert not system.partitions
system = Simulator(config_file=CONFIG_FILE)
assert system._partitions
assert len(system._partitions) > 0
def setup(self):
self.slp = TimingServiceLocator()
self.system = Simulator(config_file="simulator.xml")
self.system_thr = ComponentProgressThread(self.system)
self.system_thr.start()
self.scriptm = ScriptManager()
self.scriptm_thr = ComponentProgressThread(self.scriptm)
self.scriptm_thr.start()
self.qm = QueueManager()
self.qm_thr = ComponentProgressThread(self.qm)
self.qm_thr.start()
def setup (self):
TestComponent.setup(self)
assert os.path.exists(CONFIG_FILE)
self.system = Simulator(config_file=CONFIG_FILE)
assert self.system._partitions
assert len(self.system._partitions) > 0
part_names = self.system._partitions.keys()
for part_name in part_names:
partitions = self.system.add_partitions([{'name':part_name}])
assert len(partitions) == 1
partitions = self.system.set_partitions([{'tag':"partition", 'name':part_name}], {'functional':True, 'scheduled':True})
assert len(partitions) == 1
idle_partitions = self.system.get_partitions([{'state':"idle"}])
assert part_name in [p.name for p in idle_partitions]
partition_sizes = [p.size for p in self.system._partitions.itervalues()]
partition_sizes.sort()
self.min_size = partition_sizes[0]
self.median_size = partition_sizes[len(partition_sizes) / 2]
self.max_size = partition_sizes[-1]
def test_configure (self):
assert os.path.exists(CONFIG_FILE)
system = Simulator()
assert not system._partitions
system.configure(CONFIG_FILE)
assert system._partitions
class TestSimulator (TestComponent):
def setup (self):
TestComponent.setup(self)
assert os.path.exists(CONFIG_FILE)
self.system = Simulator(config_file=CONFIG_FILE)
assert self.system._partitions
assert len(self.system._partitions) > 0
part_names = self.system._partitions.keys()
for part_name in part_names:
partitions = self.system.add_partitions([{'name':part_name}])
assert len(partitions) == 1
partitions = self.system.set_partitions([{'tag':"partition", 'name':part_name}], {'functional':True, 'scheduled':True})
assert len(partitions) == 1
idle_partitions = self.system.get_partitions([{'state':"idle"}])
assert part_name in [p.name for p in idle_partitions]
partition_sizes = [p.size for p in self.system._partitions.itervalues()]
partition_sizes.sort()
self.min_size = partition_sizes[0]
self.median_size = partition_sizes[len(partition_sizes) / 2]
self.max_size = partition_sizes[-1]
def test_init_configure (self):
assert os.path.exists(CONFIG_FILE)
system = Simulator()
assert not system.partitions
system = Simulator(config_file=CONFIG_FILE)
assert system._partitions
assert len(system._partitions) > 0
def test_configure (self):
assert os.path.exists(CONFIG_FILE)
system = Simulator()
assert not system._partitions
system.configure(CONFIG_FILE)
assert system._partitions
def _find_location(self, jobid, size):
job_location_args = [{
'jobid': jobid,
'nodes': self.median_size,
'queue': "default",
'utility_score': 1,
'threshold': 0,
'walltime': 1,
'attrs': {}}]
locations = self.system.find_job_location(job_location_args, [])
assert locations.has_key(jobid)
return locations[jobid]
def test_reserve_partition (self):
jobid = 3002
location = self._find_location(jobid, self.max_size / 2)
partition = self.system._partitions[location[0]]
reserved = self.system.reserve_partition(partition.name)
assert reserved
assert partition.state == "busy"
for parent in self.system._partitions.q_get([{'name':parent} for parent in partition.parents]):
assert parent.state[0:7] == "blocked"
for child in self.system._partitions.q_get([{'name':child} for child in partition.children]):
assert child.state[0:7] == "blocked"
def test_release_partition (self):
jobid = 3003
idle_partitions_before = self.system.get_partitions([{'state':"idle"}])
location = self._find_location(jobid, self.median_size)
partition = self.system._partitions[location[0]]
reserved = self.system.reserve_partition(partition.name)
assert reserved
idle_partitions_after = self.system.get_partitions([{'state':"idle"}])
assert len(idle_partitions_after) != len(idle_partitions_before)
self.system.release_partition(partition.name)
idle_partitions_after = self.system.get_partitions([{'state':"idle"}])
assert len(idle_partitions_after) == len(idle_partitions_before)
def _add_pg(self, jobid):
location = self._find_location(jobid, self.median_size)
self.system.add_process_groups([dict(
id = "*",
jobid = jobid,
size = self.median_size,
mode = "vn",
executable = "/bin/ls",
location = location,
cwd = os.getcwd(),
inputfile = "infile",
outputfile = "outfile",
errorfile = "errfile",
user = os.getlogin(),
args = [],
)])
def test_add_process_groups (self):
jobid = 3004
self._add_pg(jobid)
def test_get_process_groups (self):
jobid = 3005
process_groups = self.system.get_process_groups([{'id':"*"}])
assert not process_groups
self._add_pg(jobid)
process_groups = self.system.get_process_groups([{'id':"*"}])
assert process_groups
assert process_groups[0].jobid == jobid
@timeout(Simulator.MAX_RUN_TIME + 30)
def test_run_process_groups (self):
jobid = 3006
self._add_pg(jobid)
process_groups = self.system.get_process_groups([{'id':"*"}])
assert process_groups
assert len(process_groups) == 1
id = process_groups[0].id
assert process_groups[0].jobid == jobid
while True:
process_groups = self.system.wait_process_groups([{'id': id}])
if len(process_groups) > 0:
assert len(process_groups) == 1
assert process_groups[0].id == id
assert process_groups[0].jobid == jobid
break
time.sleep(1)
def test_signal_process_groups (self):
jobid = 3007
self._add_pg(jobid)
for signal in ["SIGINT", "SIGKILL"]:
process_groups = self.system.signal_process_groups([{'id':"*"}], signal)
assert len(process_groups) == 1
assert process_groups[0].jobid == jobid
assert process_groups[0].signals[-1] == signal
def __init__(self, *args, **kwargs):
print "kwargs= ", kwargs
#initialize partitions
Simulator.__init__(self, *args, **kwargs)
partnames = self._partitions.keys()
self.init_partition(partnames)
self.part_size_list = []
for part in self.partitions.itervalues():
if int(part.size) not in self.part_size_list:
self.part_size_list.append(int(part.size))
self.part_size_list.sort()
#get command line parameters
self.FAILURE_FREE = True
self.FRACTION = kwargs.get("fraction", 1)
self.workload_file = kwargs.get("workload")
self.output_log = kwargs.get("outputlog")
self.failure_log = kwargs.get('failurelog')
self.weibull = kwargs.get('weibull')
if self.weibull:
self.SCALE = float(kwargs.get('scale'))
if self.SCALE == 0:
self.SCALE = default_SCALE
self.SHAPE = float(kwargs.get('shape'))
if self.SHAPE == 0:
self.SHAPE = default_SHAPE
self.fault_aware = kwargs.get('faultaware')
self.SENSITIVITY = default_SENSITIVITY
self.SPECIFICITY = default_SPECIFICITY
if self.fault_aware:
self.SENSITIVITY = float(kwargs.get('sensitivity', default_SENSITIVITY))
self.SPECIFICITY = float(kwargs.get('specificity', defalt_SPECIFICITY))
if self.failure_log or self.weibull:
self.FAILURE_FREE = False
#initialize time stamps and job queues
#time stamp format: ('EVENT', 'time_stamp_date', time_stamp_second, {'job_id':str(jobid), 'location':[partition1, partition2,...]})
self.time_stamps = [('I', '0', 0, {})]
self.cur_time_index = 0
self.queues = SimQueueDict(policy=kwargs['policy'])
self.init_queues()
self.visible_jobs = []
#initialize failures
self.failure_dict = {}
if not self.FAILURE_FREE:
if self.failure_log:
#if specified failure log, use log trace failure
self.inject_failures()
elif self.weibull:
#else MAKE failures by Weibull distribution
self.make_failures()
#initialize PBS-style logger
self.pbslog = PBSlogger(self.output_log)
#initialize debug logger
self.dbglog = PBSlogger(self.output_log+"-debug")
#finish tag
self.finished = False
#tag for controlling time stamp increment
self.increment_tag = True
#register local alias "system" for this component
local_components["system"] = self
print "Simulation starts:"
def setup (self):
self.system = Simulator(config_file="simulator.xml")
self.system.add_partitions([{'name':"ANL-R00-1024"}])
self.test_partition = self.system.partitions.values()[0]
class TestReservation (object):
def setup (self):
self.system = Simulator(config_file="simulator.xml")
self.system.add_partitions([{'name':"ANL-R00-1024"}])
self.test_partition = self.system.partitions.values()[0]
def teardown(self):
Cobalt.Proxy.local_components.clear()
def test_required_name (self):
spec = {'start':0, 'duration':0}
try:
reservation = Reservation(spec)
except DataCreationError:
pass
else:
assert not "didn't require name"
spec['name'] = "my_reservation"
try:
reservation = Reservation(spec)
except DataCreationError:
assert not "failed with name specified"
def test_init (self):
reservation = Reservation({'name':"mine", 'start':0, 'duration':0})
assert reservation.tag == "reservation"
assert reservation.name == "mine"
assert reservation.start == 0
assert reservation.duration == 0
assert reservation.cycle is None
assert reservation.users == ""
assert reservation.partitions == ""
def test_update (self):
reservation = Reservation({'name':"mine", 'start':100, 'duration':10})
try:
reservation.update({'users':"newuser"})
except ComponentLookupError:
cqm = QueueManager()
reservation.update({'users':"newuser"})
assert reservation.users == "newuser"
reservation = Reservation({'name':"mine", 'start':100, 'duration':10, 'users':"group"})
reservation.update({'users':"newuser"})
assert not reservation.users == "group"
assert reservation.users == "newuser"
def test_active (self):
reservation = Reservation({'name':"mine", 'start':100, 'duration':50})
for current_time in xrange(1, 100):
assert not reservation.is_active(current_time)
for current_time in xrange(100, 150):
assert reservation.is_active(current_time)
for current_time in xrange(151, 250):
assert not reservation.is_active(current_time)
def test_active_cyclic (self):
reservation = Reservation({'name':"mine", 'start':100, 'duration':10, 'cycle':50})
assert not reservation.is_active(99)
assert reservation.is_active(100)
assert reservation.is_active(109)
assert not reservation.is_active(111)
assert not reservation.is_active(149)
assert reservation.is_active(150)
assert reservation.is_active(159)
assert not reservation.is_active(161)
def test_overlaps (self):
reservation = Reservation({'name':"mine", 'start':100, 'duration':50, 'partitions':"ANL-R00-1024"})
assert not reservation.overlaps(partition=self.test_partition, start=0, duration=99)
assert reservation.overlaps(partition=self.test_partition, start=0, duration=100)
assert reservation.overlaps(partition=self.test_partition, start=0, duration=151)
assert reservation.overlaps(partition=self.test_partition, start=99, duration=1)
assert reservation.overlaps(partition=self.test_partition, start=99, duration=50)
assert reservation.overlaps(partition=self.test_partition, start=149, duration=1)
assert not reservation.overlaps(partition=self.test_partition, start=150, duration=100)
#check different partition than constructed
reservation = Reservation({'name':"mine", 'start':100, 'duration':50, 'partitions':"anotherpartion"})
assert not reservation.overlaps(partition=self.test_partition, start=0, duration=100)
def test_overlaps_cyclic (self):
reservation = Reservation({'name':"mine", 'start':100, 'duration':10, 'cycle':50, 'partitions':"ANL-R00-1024"})
assert not reservation.overlaps(partition=self.test_partition, start=0, duration=99)
assert reservation.overlaps(partition=self.test_partition, start=0, duration=100)
assert reservation.overlaps(partition=self.test_partition, start=99, duration=1)
assert reservation.overlaps(partition=self.test_partition, start=99, duration=10)
assert reservation.overlaps(partition=self.test_partition, start=101, duration=1)
assert reservation.overlaps(partition=self.test_partition, start=109, duration=1)
assert not reservation.overlaps(partition=self.test_partition, start=110, duration=39)
assert reservation.overlaps(partition=self.test_partition, start=90, duration=100)
def test_job_within_reservation (self):
# past reservation
reservation = Reservation({'name':"mine", 'start':100, 'duration':3600, 'partitions':"ANLR00", 'queue':"default"})
j = Job(5, "default")
assert not reservation.job_within_reservation(j)
j = Job(70, "default")
assert not reservation.job_within_reservation(j)
# current reservation
reservation = Reservation({'name':"mine", 'start':time.time(), 'duration':3600, 'partitions':"ANLR00", 'queue':"default"})
j = Job(5, "default")
assert reservation.job_within_reservation(j)
j = Job(70, "default")
assert not reservation.job_within_reservation(j)
# future reservation
reservation = Reservation({'name':"mine", 'start':time.time() + 3600, 'duration':3600, 'partitions':"ANLR00", 'queue':"default"})
j = Job(5, "default")
assert not reservation.job_within_reservation(j)
j = Job(40, "default")
assert not reservation.job_within_reservation(j)
j = Job(70, "default")
assert not reservation.job_within_reservation(j)
def test_job_within_reservation_cyclic (self):
reservation = Reservation({'name':"mine", 'start':time.time()-3000, 'duration':3600, 'cycle':4000, 'partitions':"ANLR00", 'queue':"default"})
# jobs ends inside the reservation
j = Job(6, "default")
assert reservation.job_within_reservation(j)
# job ends in the "dead zone"
j = Job(12, "default")
assert not reservation.job_within_reservation(j)
# job ends the next time the reservation is active
j = Job(50, "default")
assert not reservation.job_within_reservation(j)
# job lasts longer than the reservation
j = Job(100, "default")
assert not reservation.job_within_reservation(j)
# queue doesn't exist
j = Job(0,"notaqueue")
assert not reservation.job_within_reservation(j)
def test_is_over(self):
reservation = Reservation({'name':"mine", 'start':100, 'duration':10, 'cycle':50})
assert not reservation.is_over()
reservation = Reservation({'name':"mine", 'start':100, 'duration':10})
assert reservation.is_over()
reservation = Reservation({'name':"mine", 'start':time.time(), 'duration':1000})
assert not reservation.is_over()
def test_q_add (self): #|finish| assert logger
reservationdict = ReservationDict()
reservationdict['res1'] = Reservation({'name':"mine1", 'start':100, 'duration':10})
reservationdict['res2'] = Reservation({'name':"mine2", 'start':200, 'duration':10})
resqueue = {'queue':"newqueue", 'name':"newres", 'start':10, 'duration':20}
try:
reslist = reservationdict.q_add([resqueue])
except ComponentLookupError:
cqm = QueueManager()
#new reservation created
reslist = reservationdict.q_add([resqueue])
assert reservationdict['newres'].queue == "newqueue"
assert reservationdict['newres'].name == "newres"
assert len(reslist) == 1
assert reslist[0].name == "newres"
assert reslist[0].queue == "newqueue"
assert reslist[0].createdQueue
#reservation already exists
try:
new = reservationdict.q_add([resqueue])
except ReservationError:
pass
#change queue for existing reservations
res1 = {'queue':"newqueue1", 'name':"mine1", 'start':10, 'duration':20}
res2 = {'queue':"newqueue2", 'name':"mine2", 'start':10, 'duration':20}
reslist = reservationdict.q_add([res1,res2])
assert reservationdict['mine1'].queue == "newqueue1"
assert reservationdict['mine2'].queue == "newqueue2"
#queue already exists
cqm.add_queues([{'tag':"queue", 'name':"default"}])
resqueue = {'queue':"default", 'name':"other", 'start':10, 'duration':20}
reslist = reservationdict.q_add([resqueue])
def test_q_del (self): #|alert| there is not ComponenentLookup exception for the queuemanger
cqm = QueueManager()
reservationdict = ReservationDict()
reservationdict['res1'] = Reservation({'queue':"myqueue", 'name':"mine1", 'start':100, 'duration':10})
reservationdict['res2'] = Reservation({'name':"mine2", 'start':200, 'duration':10})
resqueue = {'queue':"myqueue", 'name':"mine1", 'start':10, 'duration':20}
reslist = reservationdict.q_del([resqueue])