def test_01_invalid_system_distro_combo(self):
beakerd.process_new_recipes()
beakerd.update_dirty_jobs()
with session.begin():
self.assertEqual(
Job.by_id(self.job1.id).status, TaskStatus.aborted)
self.assertEqual(
Job.by_id(self.job2.id).status, TaskStatus.processed)
def test_02_abort_dead_recipes(self):
beakerd.process_new_recipes()
beakerd.update_dirty_jobs()
beakerd.queue_processed_recipesets()
beakerd.update_dirty_jobs()
with session.begin():
self.assertEqual(Job.by_id(self.job2.id).status, TaskStatus.queued)
# Remove distro_tree2 from lab1, should cause remaining recipe to abort.
for lca in self.distro_tree2.lab_controller_assocs[:]:
session.delete(lca)
beakerd.abort_dead_recipes()
beakerd.update_dirty_jobs()
with session.begin():
self.assertEqual(Job.by_id(self.job2.id).status, TaskStatus.aborted)
def test_broken_power_aborts_recipe(self):
# Start a recipe, let it be provisioned, mark the power command as failed,
# and the recipe should be aborted.
with session.begin():
system = data_setup.create_system(
fqdn=u"broken.dreams.example.org",
lab_controller=self.lab_controller,
status=SystemStatus.automated,
shared=True,
)
distro_tree = data_setup.create_distro_tree(osmajor=u"Fedora20")
job = data_setup.create_job(distro_tree=distro_tree)
job.recipesets[0].recipes[0]._host_requires = (
u"""
<hostRequires>
<hostname op="=" value="%s" />
</hostRequires>
"""
% system.fqdn
)
beakerd.process_new_recipes()
beakerd.update_dirty_jobs()
beakerd.queue_processed_recipesets()
beakerd.update_dirty_jobs()
beakerd.schedule_queued_recipes()
beakerd.update_dirty_jobs()
beakerd.provision_scheduled_recipesets()
beakerd.update_dirty_jobs()
with session.begin():
job = Job.query.get(job.id)
self.assertEqual(job.status, TaskStatus.waiting)
system = System.query.get(system.id)
command = system.command_queue[0]
self.assertEquals(command.action, "on")
self.server.labcontrollers.mark_command_running(command.id)
self.server.labcontrollers.mark_command_failed(command.id, u"needs moar powa")
with session.begin():
job = Job.query.get(job.id)
job.update_status()
self.assertEqual(job.recipesets[0].recipes[0].status, TaskStatus.aborted)
def test_broken_power_aborts_recipe(self):
# Start a recipe, let it be provisioned, mark the power command as failed,
# and the recipe should be aborted.
with session.begin():
system = data_setup.create_system(
fqdn=u'broken.dreams.example.org',
lab_controller=self.lab_controller,
status=SystemStatus.automated,
shared=True)
distro_tree = data_setup.create_distro_tree(osmajor=u'Fedora20')
job = data_setup.create_job(distro_tree=distro_tree)
job.recipesets[0].recipes[0]._host_requires = (u"""
<hostRequires>
<hostname op="=" value="%s" />
</hostRequires>
""" % system.fqdn)
beakerd.process_new_recipes()
beakerd.update_dirty_jobs()
beakerd.queue_processed_recipesets()
beakerd.update_dirty_jobs()
beakerd.schedule_queued_recipes()
beakerd.update_dirty_jobs()
beakerd.provision_scheduled_recipesets()
beakerd.update_dirty_jobs()
with session.begin():
job = Job.query.get(job.id)
self.assertEqual(job.status, TaskStatus.waiting)
system = System.query.get(system.id)
command = system.command_queue[0]
self.assertEquals(command.action, 'on')
self.server.labcontrollers.mark_command_running(command.id)
self.server.labcontrollers.mark_command_failed(command.id,
u'needs moar powa')
with session.begin():
job = Job.query.get(job.id)
job.update_status()
self.assertEqual(job.recipesets[0].recipes[0].status,
TaskStatus.aborted)
def test_02_abort_dead_recipes(self):
beakerd.process_new_recipes()
beakerd.update_dirty_jobs()
beakerd.queue_processed_recipesets()
beakerd.update_dirty_jobs()
with session.begin():
job = Job.by_id(self.job2.id)
self.assertEqual(job.status, TaskStatus.queued)
# check if rows in system_recipe_map
self.assertNotEqual(len(job.recipesets[0].recipes[0].systems), 0)
# Remove distro_tree2 from lab1, should cause remaining recipe to abort.
for lca in self.distro_tree2.lab_controller_assocs[:]:
session.delete(lca)
beakerd.abort_dead_recipes()
beakerd.update_dirty_jobs()
with session.begin():
job = Job.by_id(self.job2.id)
self.assertEqual(job.status, TaskStatus.aborted)
# https://bugzilla.redhat.com/show_bug.cgi?id=1173376
# check if no rows system_recipe_map
self.assertEqual(len(job.recipesets[0].recipes[0].systems), 0)
def test_02_abort_dead_recipes(self):
beakerd.process_new_recipes()
beakerd.update_dirty_jobs()
with session.begin():
job = Job.by_id(self.job2.id)
self.assertEqual(job.status, TaskStatus.processed)
# check if rows in system_recipe_map
self.assertNotEqual(len(job.recipesets[0].recipes[0].systems), 0)
# Remove distro_tree2 from lab1, should cause remaining recipe to abort.
for lca in self.distro_tree2.lab_controller_assocs[:]:
session.delete(lca)
beakerd.queue_processed_recipesets()
beakerd.update_dirty_jobs()
beakerd.abort_dead_recipes()
beakerd.update_dirty_jobs()
with session.begin():
job = Job.by_id(self.job2.id)
self.assertEqual(job.status, TaskStatus.aborted)
# https://bugzilla.redhat.com/show_bug.cgi?id=1173376
# check if no rows system_recipe_map
self.assertEqual(len(job.recipesets[0].recipes[0].systems), 0)
def test_failure_in_configure_netboot_aborts_recipe(self):
with session.begin():
system = data_setup.create_system(
lab_controller=self.lab_controller,
status=SystemStatus.automated,
shared=True)
distro_tree = data_setup.create_distro_tree(osmajor=u'Fedora20')
job = data_setup.create_job(distro_tree=distro_tree)
job.recipesets[0].recipes[0]._host_requires = (u"""
<hostRequires>
<hostname op="=" value="%s" />
</hostRequires>
""" % system.fqdn)
beakerd.process_new_recipes()
beakerd.update_dirty_jobs()
beakerd.queue_processed_recipesets()
beakerd.update_dirty_jobs()
beakerd.schedule_queued_recipes()
beakerd.update_dirty_jobs()
beakerd.provision_scheduled_recipesets()
beakerd.update_dirty_jobs()
with session.begin():
job = Job.query.get(job.id)
self.assertEqual(job.status, TaskStatus.waiting)
system = System.query.get(system.id)
command = system.command_queue[2]
self.assertEquals(command.action, 'configure_netboot')
self.server.labcontrollers.mark_command_running(command.id)
self.server.labcontrollers.mark_command_failed(command.id,
u'oops it borked')
with session.begin():
job = Job.query.get(job.id)
job.update_status()
self.assertEqual(job.recipesets[0].recipes[0].status,
TaskStatus.aborted)
def test_failure_in_configure_netboot_aborts_recipe(self):
with session.begin():
system = data_setup.create_system(
lab_controller=self.lab_controller, status=SystemStatus.automated, shared=True
)
distro_tree = data_setup.create_distro_tree(osmajor=u"Fedora20")
job = data_setup.create_job(distro_tree=distro_tree)
job.recipesets[0].recipes[0]._host_requires = (
u"""
<hostRequires>
<hostname op="=" value="%s" />
</hostRequires>
"""
% system.fqdn
)
beakerd.process_new_recipes()
beakerd.update_dirty_jobs()
beakerd.queue_processed_recipesets()
beakerd.update_dirty_jobs()
beakerd.schedule_queued_recipes()
beakerd.update_dirty_jobs()
beakerd.provision_scheduled_recipesets()
beakerd.update_dirty_jobs()
with session.begin():
job = Job.query.get(job.id)
self.assertEqual(job.status, TaskStatus.waiting)
system = System.query.get(system.id)
command = system.command_queue[2]
self.assertEquals(command.action, "configure_netboot")
self.server.labcontrollers.mark_command_running(command.id)
self.server.labcontrollers.mark_command_failed(command.id, u"oops it borked")
with session.begin():
job = Job.query.get(job.id)
job.update_status()
self.assertEqual(job.recipesets[0].recipes[0].status, TaskStatus.aborted)
def test_cancel_while_scheduling(self):
# This test simulates a user cancelling their job at the same time as
# beakerd is scheduling it. beakerd assigns a system and creates
# a watchdog and sets the recipe status to Waiting, then it's
# overwritten by another transaction setting the status to Cancelled.
with session.begin():
lab_controller = data_setup.create_labcontroller()
system = data_setup.create_system(shared=True,
lab_controller=lab_controller)
distro_tree = data_setup.create_distro_tree(osmajor=u'Fedora20',
lab_controllers=[lab_controller])
job = data_setup.create_job(distro_tree=distro_tree)
job.recipesets[0].recipes[0]._host_requires = (u"""
<hostRequires>
<hostname op="=" value="%s" />
</hostRequires>
""" % system.fqdn)
beakerd.process_new_recipes()
beakerd.update_dirty_jobs()
beakerd.queue_processed_recipesets()
beakerd.update_dirty_jobs()
with session.begin():
job = Job.by_id(job.id)
system = System.query.get(system.id)
self.assertEquals(job.status, TaskStatus.queued)
self.assertEquals(job.recipesets[0].recipes[0].systems, [system])
# Two "concurrent" transactions, in the first one beakerd has
# scheduled the recipe and is about to commit...
class ScheduleThread(Thread):
def __init__(self, **kwargs):
super(ScheduleThread, self).__init__(**kwargs)
self.ready_evt = Event()
self.continue_evt = Event()
def run(self):
session.begin()
recipe = Job.by_id(job.id).recipesets[0].recipes[0]
assert recipe.status == TaskStatus.queued
self.ready_evt.set()
self.continue_evt.wait()
try:
beakerd.schedule_queued_recipe(recipe.id)
assert False, 'should raise'
except StaleTaskStatusException:
pass # expected
session.rollback()
# ... and in the second transaction the user is cancelling the recipe.
class CancelThread(Thread):
def __init__(self, **kwargs):
super(CancelThread, self).__init__(**kwargs)
self.ready_evt = Event()
self.continue_evt = Event()
def run(self):
session.begin()
recipe = Job.by_id(job.id).recipesets[0].recipes[0]
assert not recipe.watchdog
assert not recipe.resource
recipe.recipeset.cancel()
self.ready_evt.set()
self.continue_evt.wait()
session.commit()
sched_thread = ScheduleThread()
cancel_thread = CancelThread()
sched_thread.start()
cancel_thread.start()
sched_thread.ready_evt.wait()
cancel_thread.ready_evt.wait()
sched_thread.continue_evt.set()
cancel_thread.continue_evt.set()
sched_thread.join()
cancel_thread.join()
with session.begin():
session.expire_all()
job.update_status()
self.assertEquals(job.status, TaskStatus.cancelled)
self.assertEquals(job.recipesets[0].recipes[0].watchdog, None)
self.assertEquals(system.open_reservation, None)
def test_cancel_while_scheduling(self):
# This test simulates a user cancelling their job at the same time as
# beakerd is scheduling it. beakerd assigns a system and creates
# a watchdog and sets the recipe status to Waiting, then it's
# overwritten by another transaction setting the status to Cancelled.
with session.begin():
lab_controller = data_setup.create_labcontroller()
system = data_setup.create_system(shared=True,
lab_controller=lab_controller)
distro_tree = data_setup.create_distro_tree(
osmajor=u'Fedora20', lab_controllers=[lab_controller])
job = data_setup.create_job(distro_tree=distro_tree)
job.recipesets[0].recipes[0]._host_requires = (u"""
<hostRequires>
<hostname op="=" value="%s" />
</hostRequires>
""" % system.fqdn)
beakerd.process_new_recipes()
beakerd.update_dirty_jobs()
with session.begin():
job = Job.by_id(job.id)
system = System.query.get(system.id)
self.assertEquals(job.status, TaskStatus.processed)
self.assertEquals(job.recipesets[0].recipes[0].systems, [system])
# Two "concurrent" transactions, in the first one beakerd has
# scheduled the recipe and is about to commit...
class ScheduleThread(Thread):
def __init__(self, **kwargs):
super(ScheduleThread, self).__init__(**kwargs)
self.ready_evt = Event()
self.continue_evt = Event()
def run(self):
session.begin()
recipeset = Job.by_id(job.id).recipesets[0]
assert recipeset.status == TaskStatus.processed
self.ready_evt.set()
self.continue_evt.wait()
try:
beakerd.queue_processed_recipeset(recipeset.id)
assert False, 'should raise'
except StaleTaskStatusException:
pass # expected
session.rollback()
# ... and in the second transaction the user is cancelling the recipe.
class CancelThread(Thread):
def __init__(self, **kwargs):
super(CancelThread, self).__init__(**kwargs)
self.ready_evt = Event()
self.continue_evt = Event()
def run(self):
session.begin()
recipe = Job.by_id(job.id).recipesets[0].recipes[0]
assert not recipe.watchdog
assert not recipe.resource
recipe.recipeset.cancel()
self.ready_evt.set()
self.continue_evt.wait()
session.commit()
sched_thread = ScheduleThread()
cancel_thread = CancelThread()
sched_thread.start()
cancel_thread.start()
sched_thread.ready_evt.wait()
cancel_thread.ready_evt.wait()
sched_thread.continue_evt.set()
cancel_thread.continue_evt.set()
sched_thread.join()
cancel_thread.join()
with session.begin():
session.expire_all()
job.update_status()
self.assertEquals(job.status, TaskStatus.cancelled)
self.assertEquals(job.recipesets[0].recipes[0].watchdog, None)
self.assertEquals(system.open_reservation, None)
def test_01_invalid_system_distro_combo(self):
beakerd.process_new_recipes()
beakerd.update_dirty_jobs()
with session.begin():
self.assertEqual(Job.by_id(self.job1.id).status, TaskStatus.aborted)
self.assertEqual(Job.by_id(self.job2.id).status, TaskStatus.processed)