def setup(self):
pybgsched.Block('TB-1', 512)
pybgsched.Block('TB-2', 512)
pybgsched.Block('TB-3', 512)
self.test_blocks = {}
self.test_blocks['TB-1'] = test_BGQBooter.TestBlock('TB-1')
self.test_blocks['TB-2'] = test_BGQBooter.TestBlock('TB-2')
self.test_blocks['TB-3'] = test_BGQBooter.TestBlock('TB-3')
self.block_lock = threading.Lock()
pybgsched.IOBlock('IO-1', 8)
self.test_io_blocks = {}
self.test_io_blocks['IO-1'] = test_BGQBooter.TestIOBlock('IO-1')
self.booter = BGQBooter(self.test_blocks, self.block_lock)
self.booter.restore_boot_id(1)
def setup(self):
pybgsched.Block('TB-1', 512)
pybgsched.Block('TB-2', 512)
pybgsched.Block('TB-3', 512)
self.test_blocks = {}
self.test_blocks['TB-1'] = test_BGQBooter.TestBlock('TB-1')
self.test_blocks['TB-2'] = test_BGQBooter.TestBlock('TB-2')
self.test_blocks['TB-3'] = test_BGQBooter.TestBlock('TB-3')
self.block_lock = threading.Lock()
pybgsched.IOBlock('IO-1', 8)
self.test_io_blocks = {}
self.test_io_blocks['IO-1'] = test_BGQBooter.TestIOBlock('IO-1')
self.booter = BGQBooter(self.test_blocks, self.block_lock)
self.booter.restore_boot_id(1)
class test_BGQBooter(object):
class TestBlock(object):
def __init__(self, name, reserved=True, block_type='normal', subblock_parent=None):
self.name = name
self.reserved = reserved
self.block_type = block_type
self.current_kernel_options = " "
self.current_kernel = "default"
self.subblock_parent = self.name
if block_type == 'pseudoblock':
self.subblock_parent = subblock_parent
def under_resource_reservation(self, job_id):
return self.reserved
class TestIOBlock(object):
def __init__(self, name):
self.name = name
self.current_kernel_options=" "
self.current_kernel = "default"
def setup(self):
pybgsched.Block('TB-1', 512)
pybgsched.Block('TB-2', 512)
pybgsched.Block('TB-3', 512)
self.test_blocks = {}
self.test_blocks['TB-1'] = test_BGQBooter.TestBlock('TB-1')
self.test_blocks['TB-2'] = test_BGQBooter.TestBlock('TB-2')
self.test_blocks['TB-3'] = test_BGQBooter.TestBlock('TB-3')
self.block_lock = threading.Lock()
pybgsched.IOBlock('IO-1', 8)
self.test_io_blocks = {}
self.test_io_blocks['IO-1'] = test_BGQBooter.TestIOBlock('IO-1')
self.booter = BGQBooter(self.test_blocks, self.block_lock)
self.booter.restore_boot_id(1)
def teardown(self):
#if self.booter.is_alive():
# self.booter.close()
pybgsched.block_dict = {}
@timeout(3)
def test_message_creation(self):
self.booter.start()
self.booter.initiate_boot('TB-1', 1, 'testuser')
assert self.booter.all_blocks['TB-1'].under_resource_reservation(1)
time.sleep(2)
assert self.booter.pending_boots, 'boot not added: %s' % self.booter.pending_boots
@timeout(3)
def test_boot_list(self):
correct_boot_list_str = "{'block_name': 'TB-1', 'user': '******', 'state': 'pending', 'job_id': 1, 'boot_id': 1} "\
"{'block_name': 'TB-2', 'user': '******', 'state': 'pending', 'job_id': 2, 'boot_id': 2} "\
"{'block_name': 'TB-3', 'user': '******', 'state': 'pending', 'job_id': 3, 'boot_id': 3}"
boot_list = []
self.booter.start()
assert self.booter.is_alive(), "Error starting up booter"
self.booter.suspend_booting()
assert self.booter.booting_suspended, "Booting not suspended"
self.booter.initiate_boot('TB-1', 1, 'testuser')
self.booter.initiate_boot('TB-2', 2, 'testuser')
self.booter.initiate_boot('TB-3', 3, 'testuser')
time.sleep(2)
boot_list = self.booter.stat()
bl_str = " ".join([str(boot) for boot in boot_list])
assert boot_list != [] and bl_str == correct_boot_list_str , "Boot list failed: got %s" % bl_str
@timeout(9)
def test_boot_sequence(self):
#set up pybgsched block transitions, treat these as events
#It's LIFO, so revere everything, and make sure to reset the final state
pybgsched.block_dict['TB-1'].set_status(pybgsched.Block.Initialized)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Booting)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Booting)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Allocated)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Free)
self.booter.start()
self.booter.initiate_boot('TB-1',1,'testuser')
while True:
if self.booter.pending_boots != set():
break
pybgsched.block_dict['TB-1'].set_action(pybgsched.Action._None)
while (True):
time.sleep(1)
if pybgsched.block_dict['TB-1'].statuses == [pybgsched.Block.Initialized]:
time.sleep(3)
break
#assert str(self.booter.stat('TB-1')[0].state) != 'failed'
boot_state = str(self.booter.stat('TB-1')[0].state)
assert boot_state == 'complete', 'Boot state not complete, is %s instead' % boot_state
@timeout(14)
def test_reboot_sequence(self):
pybgsched.block_dict['TB-1'].set_status(pybgsched.Block.Initialized)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Booting)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Booting)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Allocated)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Free)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Terminating)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Booting)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Allocated)
self.booter.start()
self.booter.initiate_boot('TB-1',1,'testuser')
while True:
if self.booter.pending_boots != set():
break
pybgsched.block_dict['TB-1'].set_action(pybgsched.Action._None)
while (True):
time.sleep(1)
if pybgsched.block_dict['TB-1'].statuses == [pybgsched.Block.Initialized]:
time.sleep(3)
break
boot_state = str(self.booter.stat('TB-1')[0].state)
assert boot_state == 'complete', 'Boot state not complete, is %s instead' % boot_state
reboot_count = self.booter.stat('TB-1')[0].context.reboot_attempts
assert reboot_count == 1, "Failure: reboot attempts were %s" % reboot_count
@timeout(20)
def test_excessive_reboots(self):
#ensure that we honor the max_reboots variable
pybgsched.block_dict['TB-1'].set_status(pybgsched.Block.Free)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Terminating)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Free)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Terminating)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Free)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Terminating)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Free)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Terminating)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Booting)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Allocated)
self.booter.start()
self.booter.initiate_boot('TB-1',1,'testuser')
while True:
if self.booter.pending_boots != set():
break
pybgsched.block_dict['TB-1'].set_action(pybgsched.Action._None)
for boot in self.booter.pending_boots:
boot.context.max_reboot_attempts = 3
while (True):
time.sleep(1)
if len(pybgsched.block_dict['TB-1'].statuses)== 1:
time.sleep(1)
break
boot_state = str(self.booter.stat('TB-1')[0].state)
assert boot_state == 'failed', 'Boot state not failed, is %s instead' % boot_state
reboot_count = self.booter.stat('TB-1')[0].context.reboot_attempts
assert reboot_count == 3, "Failure: reboot attempts were %s" % reboot_count
def test_queued_message_fetch(self):
#this came out of a bug where the deepcopy blew up on an early
#iteration of the Reap Boot message. Make sure all messages
#can be returned by the fetch function of the underlying queue --PMR
#Run without the queue being started, so we can ensure messages stay put
self.booter.initiate_boot('TB-1', 1, 'testuser')
boot = BGQBoot(pybgsched.block_dict['TB-1'], 2, 'testuser', self.block_lock)
self.booter.send(ReapBootMsg(2))
pending_messages = ",".join([str(msg) for msg in self.booter.fetch_queued_messages()])
correct_messages = "<InitiateBootMsg: msg_type=initiate_boot, block_id=TB-1, job_id=1, user=testuser, subblock_parent=None, tag=None, timeout=None>,<ReapBootMsg: boot_id=2>"
assert pending_messages == correct_messages, "Pending messages failed: expected %s, got %s" % (correct_messages, pending_messages)
@timeout(10)
def test_io_boot_normal_sequence(self):
pybgsched.io_block_dict['IO-1'].set_status(pybgsched.IOBlock.Initialized)
pybgsched.io_block_dict['IO-1'].add_status(pybgsched.IOBlock.Booting)
pybgsched.io_block_dict['IO-1'].add_status(pybgsched.IOBlock.Booting)
pybgsched.io_block_dict['IO-1'].add_status(pybgsched.IOBlock.Allocated)
pybgsched.io_block_dict['IO-1'].add_status(pybgsched.IOBlock.Free)
self.booter.start()
self.booter.initiate_io_boot('IO-1', 'testuser')
while True:
if self.booter.pending_boots != set([]):
break
pybgsched.io_block_dict['IO-1'].set_action(pybgsched.Action._None)
while (True):
time.sleep(1)
if pybgsched.io_block_dict['IO-1'].statuses == [pybgsched.IOBlock.Initialized]:
time.sleep(3)
break
boot_state = str(self.booter.stat('IO-1')[0].state)
assert boot_state == 'complete', 'Boot state not complete, is %s instead' % boot_state
@timeout(10)
def test_io_compute_simultaneous_boot(self):
#make sure an IO boot doesn't interfere with a compute boot
pybgsched.io_block_dict['IO-1'].set_status(pybgsched.IOBlock.Initialized)
pybgsched.io_block_dict['IO-1'].add_status(pybgsched.IOBlock.Booting)
pybgsched.io_block_dict['IO-1'].add_status(pybgsched.IOBlock.Booting)
pybgsched.io_block_dict['IO-1'].add_status(pybgsched.IOBlock.Allocated)
pybgsched.io_block_dict['IO-1'].add_status(pybgsched.IOBlock.Free)
pybgsched.block_dict['TB-1'].set_status(pybgsched.Block.Initialized)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Booting)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Booting)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Allocated)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Free)
self.booter.start()
self.booter.initiate_io_boot('IO-1', 'testuser')
self.booter.initiate_boot('TB-1',1,'testuser')
while True:
if self.booter.pending_boots != set([]):
break
pybgsched.io_block_dict['IO-1'].set_action(pybgsched.Action._None)
pybgsched.block_dict['TB-1'].set_action(pybgsched.Action._None)
while (True):
time.sleep(1)
if (pybgsched.io_block_dict['IO-1'].statuses == [pybgsched.IOBlock.Initialized] and
pybgsched.block_dict['TB-1'].statuses == [pybgsched.Block.Initialized]):
time.sleep(3)
break
io_boot_state = str(self.booter.stat('IO-1')[0].state)
boot_state = str(self.booter.stat('TB-1')[0].state)
assert boot_state == 'complete', 'Boot state not complete, is %s instead' % boot_state
assert io_boot_state == 'complete', 'Boot state not complete, is %s instead' % io_boot_state
def test_io_boot_already_initialized(self):
pybgsched.io_block_dict['IO-1'].set_status(pybgsched.IOBlock.Initialized)
self.booter.start()
self.booter.initiate_io_boot('IO-1', 'testuser')
time.sleep(3)
boot_state = str(self.booter.stat('IO-1')[0].state)
assert boot_state == 'failed', 'Boot state not complete, is %s instead' % boot_state
def test_io_boot_already_allocated(self):
pybgsched.io_block_dict['IO-1'].set_status(pybgsched.IOBlock.Allocated)
self.booter.start()
self.booter.initiate_io_boot('IO-1', 'testuser')
time.sleep(3)
boot_state = str(self.booter.stat('IO-1')[0].state)
assert boot_state == 'failed', 'Boot state not complete, is %s instead' % boot_state
def test_io_boot_already_booting(self):
pybgsched.io_block_dict['IO-1'].set_status(pybgsched.IOBlock.Booting)
self.booter.start()
self.booter.initiate_io_boot('IO-1', 'testuser')
time.sleep(3)
boot_state = str(self.booter.stat('IO-1')[0].state)
assert boot_state == 'failed', 'Boot state not complete, is %s instead' % boot_state
def test_io_boot_already_terminating(self):
pybgsched.io_block_dict['IO-1'].set_status(pybgsched.IOBlock.Terminating)
self.booter.start()
self.booter.initiate_io_boot('IO-1', 'testuser')
time.sleep(3)
boot_state = str(self.booter.stat('IO-1')[0].state)
assert boot_state == 'failed', 'Boot state not complete, is %s instead' % boot_state
def test_io_boot_already_free_boot_action_set(self):
pybgsched.io_block_dict['IO-1'].set_action(pybgsched.Action.Boot)
self.booter.start()
self.booter.initiate_io_boot('IO-1', 'testuser')
time.sleep(3)
boot_state = str(self.booter.stat('IO-1')[0].state)
assert boot_state == 'failed', 'Boot state not complete, is %s instead' % boot_state
@timeout(25)
def test_boot_timeout_completed(self):
#walk the boot through its transitions, make sure the boot object goes away without external reaping.
pybgsched.block_dict['TB-1'].set_status(pybgsched.Block.Initialized)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Booting)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Booting)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Allocated)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Free)
self.booter.start()
self.booter.initiate_boot('TB-1', 1, 'testuser', timeout=10)
while True:
if self.booter.pending_boots != set():
break
pybgsched.block_dict['TB-1'].set_action(pybgsched.Action._None)
while (True):
time.sleep(1)
if pybgsched.block_dict['TB-1'].statuses == [pybgsched.Block.Initialized]:
time.sleep(3)
break
boot_state = str(self.booter.stat('TB-1')[0].state)
assert boot_state == 'complete', 'Boot state not complete, is %s instead' % boot_state
while(True):
time.sleep(1)
boot = self.booter.stat('TB-1')[0]
if boot.context.force_clean == True:
self.booter.reap(boot.context.block_id)
break
while(True): #Make sure the boot really died
msg_count = len(self.booter.fetch_queued_messages())
if msg_count == 0:
break
assert len(self.booter.stat('TB-1')) == 0, "Boot should be deleted, but reference still in boot list."
@timeout(18)
def test_boot_timeout_failed(self):
#take a boot into a failed state and make sure that the object is cleaned up within the expected timeout
pybgsched.block_dict['TB-1'].set_status(pybgsched.Block.Terminating)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Allocated)
self.booter.start()
self.booter.initiate_boot('TB-1', 1, 'testuser', timeout=10)
while True:
if self.booter.pending_boots != set():
break
for boot in self.booter.pending_boots:
boot.context.max_reboot_attempts = 0
pybgsched.block_dict['TB-1'].set_action(pybgsched.Action._None)
while (True):
time.sleep(1)
current_status = pybgsched.block_dict['TB-1'].statuses
if current_status == [pybgsched.Block.Terminating]:
time.sleep(3)
break
boot_state = str(self.booter.stat('TB-1')[0].state)
assert boot_state == 'failed', 'Boot state not failed, is %s instead' % boot_state
while(True):
time.sleep(1)
boot = self.booter.stat('TB-1')[0]
if boot.context.force_clean == True:
self.booter.reap(boot.context.block_id)
break
while(True): #Make sure the boot really died
msg_count = len(self.booter.fetch_queued_messages())
if msg_count == 0:
break
assert len(self.booter.stat('TB-1')) == 0, "Boot should be deleted, but reference still in boot list."
@timeout(10)
def test_subblock_boot(self):
#Test to make sure that the right block gets booted if we pass in a block with a different subblock parent.
self.test_blocks['TB-P'] = test_BGQBooter.TestBlock('TB-P', block_type='pseudoblock', subblock_parent='TB-1')
pybgsched.block_dict['TB-1'].set_status(pybgsched.Block.Initialized)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Booting)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Booting)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Allocated)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Free)
self.booter.start()
self.booter.initiate_boot('TB-P', 1, 'testuser')
while True:
if self.booter.pending_boots != set():
break
pybgsched.block_dict['TB-1'].set_action(pybgsched.Action._None)
while (True):
time.sleep(1)
if pybgsched.block_dict['TB-1'].statuses == [pybgsched.Block.Initialized]:
time.sleep(3)
break
#assert str(self.booter.stat('TB-1')[0].state) != 'failed'
boot_state = str(self.booter.stat('TB-P')[0].state)
assert boot_state == 'complete', 'Boot state not complete, is %s instead' % boot_state
class test_BGQBooter(object):
class TestBlock(object):
def __init__(self, name, reserved=True, block_type='normal'):
self.name = name
self.reserved = reserved
self.block_type = 'normal'
self.current_kernel_options=" "
self.current_kernel = "default"
def under_resource_reservation(self, job_id):
return self.reserved
class TestIOBlock(object):
def __init__(self, name):
self.name = name
self.current_kernel_options=" "
self.current_kernel = "default"
def setup(self):
pybgsched.Block('TB-1', 512)
pybgsched.Block('TB-2', 512)
pybgsched.Block('TB-3', 512)
self.test_blocks = {}
self.test_blocks['TB-1'] = test_BGQBooter.TestBlock('TB-1')
self.test_blocks['TB-2'] = test_BGQBooter.TestBlock('TB-2')
self.test_blocks['TB-3'] = test_BGQBooter.TestBlock('TB-3')
self.block_lock = threading.Lock()
pybgsched.IOBlock('IO-1', 8)
self.test_io_blocks = {}
self.test_io_blocks['IO-1'] = test_BGQBooter.TestIOBlock('IO-1')
self.booter = BGQBooter(self.test_blocks, self.block_lock)
self.booter.restore_boot_id(1)
def teardown(self):
#if self.booter.is_alive():
# self.booter.close()
pybgsched.block_dict = {}
@timeout(3)
def test_message_creation(self):
self.booter.start()
self.booter.initiate_boot('TB-1', 1, 'testuser')
assert self.booter.all_blocks['TB-1'].under_resource_reservation(1)
time.sleep(2)
assert self.booter.pending_boots, 'boot not added: %s' % self.booter.pending_boots
@timeout(3)
def test_boot_list(self):
correct_boot_list_str = "{'block_name': 'TB-1', 'user': '******', 'state': 'pending', 'job_id': 1, 'boot_id': 1} "\
"{'block_name': 'TB-2', 'user': '******', 'state': 'pending', 'job_id': 2, 'boot_id': 2} "\
"{'block_name': 'TB-3', 'user': '******', 'state': 'pending', 'job_id': 3, 'boot_id': 3}"
boot_list = []
self.booter.start()
assert self.booter.is_alive(), "Error starting up booter"
self.booter.suspend_booting()
assert self.booter.booting_suspended, "Booting not suspended"
self.booter.initiate_boot('TB-1', 1, 'testuser')
self.booter.initiate_boot('TB-2', 2, 'testuser')
self.booter.initiate_boot('TB-3', 3, 'testuser')
time.sleep(2)
boot_list = self.booter.stat()
bl_str = " ".join([str(boot) for boot in boot_list])
assert boot_list != [] and bl_str == correct_boot_list_str , "Boot list failed: got %s" % bl_str
@timeout(9)
def test_boot_sequence(self):
#set up pybgsched block transitions, treat these as events
#It's LIFO, so revere everything, and make sure to reset the final state
pybgsched.block_dict['TB-1'].set_status(pybgsched.Block.Initialized)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Booting)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Booting)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Allocated)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Free)
self.booter.start()
self.booter.initiate_boot('TB-1',1,'testuser')
while True:
if self.booter.pending_boots != set():
break
pybgsched.block_dict['TB-1'].set_action(pybgsched.Action._None)
while (True):
time.sleep(1)
if pybgsched.block_dict['TB-1'].statuses == [pybgsched.Block.Initialized]:
time.sleep(3)
break
#assert str(self.booter.stat('TB-1')[0].state) != 'failed'
boot_state = str(self.booter.stat('TB-1')[0].state)
assert boot_state == 'complete', 'Boot state not complete, is %s instead' % boot_state
@timeout(14)
def test_reboot_sequence(self):
pybgsched.block_dict['TB-1'].set_status(pybgsched.Block.Initialized)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Booting)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Booting)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Allocated)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Free)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Terminating)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Booting)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Allocated)
self.booter.start()
self.booter.initiate_boot('TB-1',1,'testuser')
while True:
if self.booter.pending_boots != set():
break
pybgsched.block_dict['TB-1'].set_action(pybgsched.Action._None)
while (True):
time.sleep(1)
if pybgsched.block_dict['TB-1'].statuses == [pybgsched.Block.Initialized]:
time.sleep(3)
break
boot_state = str(self.booter.stat('TB-1')[0].state)
assert boot_state == 'complete', 'Boot state not complete, is %s instead' % boot_state
reboot_count = self.booter.stat('TB-1')[0].context.reboot_attempts
assert reboot_count == 1, "Failure: reboot attempts were %s" % reboot_count
@timeout(20)
def test_excessive_reboots(self):
#ensure that we honor the max_reboots variable
pybgsched.block_dict['TB-1'].set_status(pybgsched.Block.Free)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Terminating)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Free)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Terminating)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Free)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Terminating)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Free)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Terminating)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Booting)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Allocated)
self.booter.start()
self.booter.initiate_boot('TB-1',1,'testuser')
while True:
if self.booter.pending_boots != set():
break
pybgsched.block_dict['TB-1'].set_action(pybgsched.Action._None)
for boot in self.booter.pending_boots:
boot.context.max_reboot_attempts = 3
while (True):
time.sleep(1)
if len(pybgsched.block_dict['TB-1'].statuses)== 1:
time.sleep(1)
break
boot_state = str(self.booter.stat('TB-1')[0].state)
assert boot_state == 'failed', 'Boot state not failed, is %s instead' % boot_state
reboot_count = self.booter.stat('TB-1')[0].context.reboot_attempts
assert reboot_count == 3, "Failure: reboot attempts were %s" % reboot_count
def test_queued_message_fetch(self):
#this came out of a bug where the deepcopy blew up on an early
#iteration of the Reap Boot message. Make sure all messages
#can be returned by the fetch function of the underlying queue --PMR
#Run without the queue being started, so we can ensure messages stay put
self.booter.initiate_boot('TB-1', 1, 'testuser')
boot = BGQBoot(pybgsched.block_dict['TB-1'], 2, 'testuser', self.block_lock)
self.booter.send(ReapBootMsg(2))
pending_messages = ",".join([str(msg) for msg in self.booter.fetch_queued_messages()])
correct_messages = "<InitiateBootMsg: msg_type=initiate_boot, block_id=TB-1, job_id=1, user=testuser, subblock_parent=None, tag=None, timeout=None>,<ReapBootMsg: boot_id=2>"
assert pending_messages == correct_messages, "Pending messages failed: expected %s, got %s" % (correct_messages, pending_messages)
@timeout(10)
def test_io_boot_normal_sequence(self):
pybgsched.io_block_dict['IO-1'].set_status(pybgsched.IOBlock.Initialized)
pybgsched.io_block_dict['IO-1'].add_status(pybgsched.IOBlock.Booting)
pybgsched.io_block_dict['IO-1'].add_status(pybgsched.IOBlock.Booting)
pybgsched.io_block_dict['IO-1'].add_status(pybgsched.IOBlock.Allocated)
pybgsched.io_block_dict['IO-1'].add_status(pybgsched.IOBlock.Free)
self.booter.start()
self.booter.initiate_io_boot('IO-1', 'testuser')
while True:
if self.booter.pending_boots != set([]):
break
pybgsched.io_block_dict['IO-1'].set_action(pybgsched.Action._None)
while (True):
time.sleep(1)
if pybgsched.io_block_dict['IO-1'].statuses == [pybgsched.IOBlock.Initialized]:
time.sleep(3)
break
boot_state = str(self.booter.stat('IO-1')[0].state)
assert boot_state == 'complete', 'Boot state not complete, is %s instead' % boot_state
@timeout(10)
def test_io_compute_simultaneous_boot(self):
#make sure an IO boot doesn't interfere with a compute boot
pybgsched.io_block_dict['IO-1'].set_status(pybgsched.IOBlock.Initialized)
pybgsched.io_block_dict['IO-1'].add_status(pybgsched.IOBlock.Booting)
pybgsched.io_block_dict['IO-1'].add_status(pybgsched.IOBlock.Booting)
pybgsched.io_block_dict['IO-1'].add_status(pybgsched.IOBlock.Allocated)
pybgsched.io_block_dict['IO-1'].add_status(pybgsched.IOBlock.Free)
pybgsched.block_dict['TB-1'].set_status(pybgsched.Block.Initialized)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Booting)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Booting)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Allocated)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Free)
self.booter.start()
self.booter.initiate_io_boot('IO-1', 'testuser')
self.booter.initiate_boot('TB-1',1,'testuser')
while True:
if self.booter.pending_boots != set([]):
break
pybgsched.io_block_dict['IO-1'].set_action(pybgsched.Action._None)
pybgsched.block_dict['TB-1'].set_action(pybgsched.Action._None)
while (True):
time.sleep(1)
if (pybgsched.io_block_dict['IO-1'].statuses == [pybgsched.IOBlock.Initialized] and
pybgsched.block_dict['TB-1'].statuses == [pybgsched.Block.Initialized]):
time.sleep(3)
break
io_boot_state = str(self.booter.stat('IO-1')[0].state)
boot_state = str(self.booter.stat('TB-1')[0].state)
assert boot_state == 'complete', 'Boot state not complete, is %s instead' % boot_state
assert io_boot_state == 'complete', 'Boot state not complete, is %s instead' % io_boot_state
def test_io_boot_already_initialized(self):
pybgsched.io_block_dict['IO-1'].set_status(pybgsched.IOBlock.Initialized)
self.booter.start()
self.booter.initiate_io_boot('IO-1', 'testuser')
time.sleep(3)
boot_state = str(self.booter.stat('IO-1')[0].state)
assert boot_state == 'failed', 'Boot state not complete, is %s instead' % boot_state
def test_io_boot_already_allocated(self):
pybgsched.io_block_dict['IO-1'].set_status(pybgsched.IOBlock.Allocated)
self.booter.start()
self.booter.initiate_io_boot('IO-1', 'testuser')
time.sleep(3)
boot_state = str(self.booter.stat('IO-1')[0].state)
assert boot_state == 'failed', 'Boot state not complete, is %s instead' % boot_state
def test_io_boot_already_booting(self):
pybgsched.io_block_dict['IO-1'].set_status(pybgsched.IOBlock.Booting)
self.booter.start()
self.booter.initiate_io_boot('IO-1', 'testuser')
time.sleep(3)
boot_state = str(self.booter.stat('IO-1')[0].state)
assert boot_state == 'failed', 'Boot state not complete, is %s instead' % boot_state
def test_io_boot_already_terminating(self):
pybgsched.io_block_dict['IO-1'].set_status(pybgsched.IOBlock.Terminating)
self.booter.start()
self.booter.initiate_io_boot('IO-1', 'testuser')
time.sleep(3)
boot_state = str(self.booter.stat('IO-1')[0].state)
assert boot_state == 'failed', 'Boot state not complete, is %s instead' % boot_state
def test_io_boot_already_free_boot_action_set(self):
pybgsched.io_block_dict['IO-1'].set_action(pybgsched.Action.Boot)
self.booter.start()
self.booter.initiate_io_boot('IO-1', 'testuser')
time.sleep(3)
boot_state = str(self.booter.stat('IO-1')[0].state)
assert boot_state == 'failed', 'Boot state not complete, is %s instead' % boot_state
@timeout(25)
def test_boot_timeout_completed(self):
#walk the boot through its transitions, make sure the boot object goes away without external reaping.
pybgsched.block_dict['TB-1'].set_status(pybgsched.Block.Initialized)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Booting)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Booting)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Allocated)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Free)
self.booter.start()
self.booter.initiate_boot('TB-1', 1, 'testuser', timeout=10)
while True:
if self.booter.pending_boots != set():
break
pybgsched.block_dict['TB-1'].set_action(pybgsched.Action._None)
while (True):
time.sleep(1)
if pybgsched.block_dict['TB-1'].statuses == [pybgsched.Block.Initialized]:
time.sleep(3)
break
boot_state = str(self.booter.stat('TB-1')[0].state)
assert boot_state == 'complete', 'Boot state not complete, is %s instead' % boot_state
while(True):
time.sleep(1)
boot = self.booter.stat('TB-1')[0]
if boot.context.force_clean == True:
self.booter.reap(boot.context.block_id)
break
while(True): #Make sure the boot really died
msg_count = len(self.booter.fetch_queued_messages())
if msg_count == 0:
break
assert len(self.booter.stat('TB-1')) == 0, "Boot should be deleted, but reference still in boot list."
@timeout(18)
def test_boot_timeout_failed(self):
#take a boot into a failed state and make sure that the object is cleaned up within the expected timeout
pybgsched.block_dict['TB-1'].set_status(pybgsched.Block.Terminating)
pybgsched.block_dict['TB-1'].add_status(pybgsched.Block.Allocated)
self.booter.start()
self.booter.initiate_boot('TB-1', 1, 'testuser', timeout=10)
while True:
if self.booter.pending_boots != set():
break
for boot in self.booter.pending_boots:
boot.context.max_reboot_attempts = 0
pybgsched.block_dict['TB-1'].set_action(pybgsched.Action._None)
while (True):
time.sleep(1)
current_status = pybgsched.block_dict['TB-1'].statuses
if current_status == [pybgsched.Block.Terminating]:
time.sleep(3)
break
boot_state = str(self.booter.stat('TB-1')[0].state)
assert boot_state == 'failed', 'Boot state not failed, is %s instead' % boot_state
while(True):
time.sleep(1)
boot = self.booter.stat('TB-1')[0]
if boot.context.force_clean == True:
self.booter.reap(boot.context.block_id)
break
while(True): #Make sure the boot really died
msg_count = len(self.booter.fetch_queued_messages())
if msg_count == 0:
break
assert len(self.booter.stat('TB-1')) == 0, "Boot should be deleted, but reference still in boot list."
