def save(self, task_rpm, *args, **kw):
"""
TurboGears method to upload task rpm package
"""
rpm_path = Task.get_rpm_path(task_rpm.filename)
if not task_rpm.filename:
flash(_(u'No task RPM specified'))
redirect(url("./new"))
# we do it here, since we do not want to proceed
# any further
if len(task_rpm.filename) > 255:
flash(_(u"Task RPM name should be <= 255 characters"))
redirect(url("./new"))
if os.path.exists("%s" % rpm_path):
flash(_(u'Failed to import because we already have %s' %
task_rpm.filename ))
redirect(url("./new"))
try:
def write_data(f):
siphon(task_rpm.file, f)
task = Task.update_task(task_rpm.filename, write_data)
except Exception, err:
session.rollback()
log.exception('Failed to import %s', task_rpm.filename)
flash(_(u'Failed to import task: %s' % err))
redirect(url("./new"))
def default(self, *args, **kw):
# to handle the case one of the flask methods
# have raised a 404 but the intention isn't to redirect
# back to cherrypy, but legitimately 404
if cherrypy.request.method != 'GET':
raise cherrypy.HTTPError(404)
try:
using_task_id = False
if len(args) == 1:
try:
task_id = int(args[0])
using_task_id = True
except ValueError:
pass
if using_task_id:
task = Task.by_id(task_id)
else:
task = Task.by_name("/%s" % "/".join(args))
#Would rather not redirect but do_search expects task_id in URL
#This is the simplest way of dealing with it
redirect("/tasks/%s" % task.id)
except DatabaseLookupError as e:
flash(unicode(e))
redirect("/tasks")
attributes = task.to_dict()
attributes['can_disable'] = bool(identity.current.user
and identity.current.user.is_admin())
return dict(attributes=attributes,
url="/tasks/%s" % task.id,
form=self.task_form,
value=dict(task_id=task.id),
options=dict(hidden=dict(task=1)),
action='./do_search')
def default(self, *args, **kw):
try:
using_task_id = False
if len(args) == 1:
try:
task_id = int(args[0])
using_task_id = True
except ValueError:
pass
if using_task_id:
task = Task.by_id(task_id)
else:
task = Task.by_name("/%s" % "/".join(args))
#Would rather not redirect but do_search expects task_id in URL
#This is the simplest way of dealing with it
redirect("/tasks/%s" % task.id)
except DatabaseLookupError as e:
flash(unicode(e))
redirect("/tasks")
return dict(task=task,
form=self.task_form,
value=dict(task_id=task.id),
options=dict(hidden=dict(task=1)),
action='./do_search')
def default(self, *args, **kw):
try:
using_task_id = False
if len(args) == 1:
try:
task_id = int(args[0])
using_task_id = True
except ValueError:
pass
if using_task_id:
task = Task.by_id(task_id)
else:
task = Task.by_name("/%s" % "/".join(args))
#Would rather not redirect but do_search expects task_id in URL
#This is the simplest way of dealing with it
redirect("/tasks/%s" % task.id)
except DatabaseLookupError as e:
flash(unicode(e))
redirect("/tasks")
return dict(task=task,
form = self.task_form,
value = dict(task_id = task.id),
options = dict(hidden=dict(task = 1)),
action = './do_search')
def update_db(self):
self.logger.info('Updating local Beaker database..')
for task_rpm in self.tasks_added:
self.logger.debug('Adding %s'% task_rpm)
with open(os.path.join(self.task_dir,task_rpm)) as f:
try:
session.begin()
task = self.tasks.process_taskinfo(self.tasks.read_taskinfo(f))
old_rpm = task.rpm
task.rpm = task_rpm
session.commit()
except Exception:
session.rollback()
session.close()
self.logger.critical('Error adding task %s' % task_rpm)
unlink_ignore(task_rpm)
else:
session.close()
self.logger.debug('Successfully added %s' % task.rpm)
if old_rpm:
unlink_ignore(os.path.join(self.task_dir, old_rpm))
# Update task repo
self.logger.info('Creating repodata..')
Task.update_repo()
return
def default(self, *args, **kw):
try:
using_task_id = False
if len(args) == 1:
try:
task_id = int(args[0])
using_task_id = True
except ValueError:
pass
if using_task_id:
task = Task.by_id(task_id)
else:
task = Task.by_name("/%s" % "/".join(args))
#Would rather not redirect but do_search expects task_id in URL
#This is the simplest way of dealing with it
redirect("/tasks/%s" % task.id)
except InvalidRequestError:
if using_task_id:
err_msg = u'Invalid task_id %s' % args[0]
else:
err_msg = u'Invalid task /%s' % '/'.join(args)
flash(_(err_msg))
redirect("/tasks")
return dict(task=task,
form = self.task_form,
value = dict(task_id = task.id),
options = dict(hidden=dict(task = 1)),
action = './do_search')
def save(self, task_rpm, *args, **kw):
"""
TurboGears method to upload task rpm package
"""
rpm_path = Task.get_rpm_path(task_rpm.filename)
if not task_rpm.filename:
flash(_(u'No task RPM specified'))
redirect(url("./new"))
# we do it here, since we do not want to proceed
# any further
if len(task_rpm.filename) > 255:
flash(_(u"Task RPM name should be <= 255 characters"))
redirect(url("./new"))
if os.path.exists("%s" % rpm_path):
flash(
_(u'Failed to import because we already have %s' %
task_rpm.filename))
redirect(url("./new"))
try:
def write_data(f):
siphon(task_rpm.file, f)
task = Task.update_task(task_rpm.filename, write_data)
except Exception, err:
session.rollback()
log.exception('Failed to import %s', task_rpm.filename)
flash(_(u'Failed to import task: %s' % err))
redirect(url("./new"))
def _create_recipe(self, system=None):
with session.begin():
install_task = Task.by_name(u'/distribution/install')
reserve_task = Task.by_name(u'/distribution/reservesys')
lc = create_lab_controller()
rhel62_server_x86_64 = create_rhel62_server_x86_64(lab_controller=lc)
if not system:
system = create_x86_64_automated(lc)
recipe = data_setup.create_recipe(distro_tree=rhel62_server_x86_64, task_list=[install_task, reserve_task])
data_setup.create_job_for_recipes([recipe], owner=create_user(), whiteboard=u'')
data_setup.mark_recipe_complete(recipe, system=system)
self.recipe_id = recipe.id
return recipe
def _create_recipe_with_user_defined_distro(self, **kwargs):
with session.begin():
install_task = Task.by_name(u'/distribution/check-install')
reserve_task = Task.by_name(u'/distribution/reservesys')
lc = create_lab_controller()
system = create_x86_64_automated(lc)
recipe = data_setup.create_recipe(custom_distro=True, osmajor=kwargs['osmajor'],
task_list=[install_task, reserve_task]) if \
'osmajor' in kwargs else data_setup.create_recipe(custom_distro=True, task_list=[install_task, reserve_task])
data_setup.create_job_for_recipes([recipe], owner=create_user(), whiteboard=u'')
data_setup.mark_recipe_complete(recipe, system=system)
self.recipe_id = recipe.id
return recipe
def _create_recipe(self, system=None):
with session.begin():
install_task = Task.by_name(u'/distribution/check-install')
reserve_task = Task.by_name(u'/distribution/reservesys')
lc = create_lab_controller()
rhel62_server_x86_64 = create_rhel62_server_x86_64(lab_controller=lc)
if not system:
system = create_x86_64_automated(lc)
recipe = data_setup.create_recipe(distro_tree=rhel62_server_x86_64, task_list=[install_task, reserve_task])
data_setup.create_job_for_recipes([recipe], owner=create_user(), whiteboard=u'')
data_setup.mark_recipe_complete(recipe, system=system)
self.recipe_id = recipe.id
return recipe
def _create_recipe_with_user_defined_distro(self, **kwargs):
with session.begin():
install_task = Task.by_name(u'/distribution/check-install')
reserve_task = Task.by_name(u'/distribution/reservesys')
lc = create_lab_controller()
system = create_x86_64_automated(lc)
recipe = data_setup.create_recipe(custom_distro=True, osmajor=kwargs['osmajor'],
task_list=[install_task, reserve_task]) if \
'osmajor' in kwargs else data_setup.create_recipe(custom_distro=True, task_list=[install_task, reserve_task])
data_setup.create_job_for_recipes([recipe], owner=create_user(), whiteboard=u'')
data_setup.mark_recipe_complete(recipe, system=system)
self.recipe_id = recipe.id
return recipe
def test_task_aborted_return_reservation(self):
""" This tests the case where the task was aborted, then
the recipe goes to Reserved state and then finally the reservation
is returned
"""
with session.begin():
recipe = data_setup.create_recipe(
task_list=[Task.by_name(u'/distribution/install')],
reservesys=True)
job = data_setup.create_job_for_recipes([recipe])
job_id = job.id
data_setup.mark_recipe_tasks_finished(recipe, result=TaskResult.warn,
task_status=TaskStatus.aborted)
job._mark_dirty()
beakerd.update_dirty_jobs()
with session.begin():
job = Job.by_id(job_id)
self.assertEqual(job.recipesets[0].recipes[0].status,
TaskStatus.reserved)
job.recipesets[0].recipes[0].return_reservation()
beakerd.update_dirty_jobs()
with session.begin():
job = Job.by_id(job_id)
self.assertEqual(job.recipesets[0].recipes[0].status,
TaskStatus.aborted)
def test_recipe_running_then_watchdog_expired(self):
""" This tests the case where the recipe is running, has a valid
reservation request, but the watchdog expires before it's
completed.
"""
with session.begin():
recipe = data_setup.create_recipe(
task_list=[Task.by_name(u'/distribution/install')],
reservesys=True)
job = data_setup.create_job_for_recipes([recipe])
job_id = job.id
data_setup.mark_recipe_tasks_finished(recipe,
task_status=TaskStatus.aborted)
job.recipesets[0].recipes[0].abort()
beakerd.update_dirty_jobs()
with session.begin():
job = Job.by_id(job_id)
self.assertEqual(job.recipesets[0].recipes[0].status,
TaskStatus.reserved)
job.recipesets[0].recipes[0].return_reservation()
beakerd.update_dirty_jobs()
with session.begin():
job = Job.by_id(job_id)
self.assertEqual(job.recipesets[0].recipes[0].status,
TaskStatus.aborted)
def test_task_without_failed_results_should_not_expand(self):
with session.begin():
recipe = data_setup.create_recipe(task_list=[
Task.by_name(u'/distribution/install'),
Task.by_name(u'/distribution/reservesys')
])
job = data_setup.create_job_for_recipes([recipe])
data_setup.mark_recipe_tasks_finished(recipe, result=TaskResult.pass_)
job.update_status()
b = self.browser
go_to_recipe_view(b, recipe)
for task in recipe.tasks:
b.find_element_by_xpath('//div[@id="recipe-task-details-%s" and '
'not(contains(@class, "in"))]' % task.id)
_, fragment = urlparse.urldefrag(b.current_url)
self.assertEquals(fragment, 'tasks')
def test_recipe_running_then_watchdog_expired(self):
""" This tests the case where the recipe is running, has a valid
reservation request, but the watchdog expires before it's
completed.
"""
with session.begin():
recipe = data_setup.create_recipe(
task_list=[Task.by_name(u'/distribution/install')],
reservesys=True)
job = data_setup.create_job_for_recipes([recipe])
job_id = job.id
data_setup.mark_recipe_tasks_finished(
recipe, task_status=TaskStatus.aborted)
job.recipesets[0].recipes[0].abort()
beakerd.update_dirty_jobs()
with session.begin():
job = Job.by_id(job_id)
self.assertEqual(job.recipesets[0].recipes[0].status,
TaskStatus.reserved)
job.recipesets[0].recipes[0].return_reservation()
beakerd.update_dirty_jobs()
with session.begin():
job = Job.by_id(job_id)
self.assertEqual(job.recipesets[0].recipes[0].status,
TaskStatus.aborted)
def test_task_aborted_return_reservation(self):
""" This tests the case where the task was aborted, then
the recipe goes to Reserved state and then finally the reservation
is returned
"""
with session.begin():
recipe = data_setup.create_recipe(
task_list=[Task.by_name(u'/distribution/install')],
reservesys=True)
job = data_setup.create_job_for_recipes([recipe])
job_id = job.id
data_setup.mark_recipe_tasks_finished(
recipe, result=TaskResult.warn, task_status=TaskStatus.aborted)
job._mark_dirty()
beakerd.update_dirty_jobs()
with session.begin():
job = Job.by_id(job_id)
self.assertEqual(job.recipesets[0].recipes[0].status,
TaskStatus.reserved)
job.recipesets[0].recipes[0].return_reservation()
beakerd.update_dirty_jobs()
with session.begin():
job = Job.by_id(job_id)
self.assertEqual(job.recipesets[0].recipes[0].status,
TaskStatus.aborted)
def _get_task_xml(self, server, task):
# if local, directly read the database
if server == 'local':
try:
self.logger.debug(
'Getting task XML for %s from local database' % task)
return Task.by_name(task, True).to_xml(False)
except Exception:
self.logger.error(
'Could not get task XML for %s from local Beaker DB. Continuing.'
% task)
return None
try:
self.logger.debug('Getting task XML for %s from %s' %
(task, getattr(self, server)))
return self.proxy.tasks.to_xml(task, False)
except (xmlrpclib.Fault, xmlrpclib.ProtocolError) as e:
# If something goes wrong with this task, for example:
# https://bugzilla.redhat.com/show_bug.cgi?id=915549
# we do our best to continue anyway...
self.logger.error(
'Could not get task XML for %s from %s. Continuing.' %
(task, server))
self.logger.error('Error message: %s' % e)
return None
def test_task_without_failed_results_should_not_expand(self):
with session.begin():
recipe = data_setup.create_recipe(task_list=[
Task.by_name(u'/distribution/install'),
Task.by_name(u'/distribution/reservesys')
])
job = data_setup.create_job_for_recipes([recipe])
data_setup.mark_recipe_tasks_finished(recipe, result=TaskResult.pass_)
job.update_status()
b = self.browser
go_to_recipe_view(b, recipe)
for task in recipe.tasks:
b.find_element_by_xpath('//div[@id="recipe-task-details-%s" and '
'not(contains(@class, "in"))]' % task.id)
_, fragment = urlparse.urldefrag(b.current_url)
self.assertEquals(fragment, 'tasks')
def test_recipe_running_then_cancelled(self):
""" This tests the case where the recipe is running, has a valid
reservation request, but is cancelled before it's completed.
"""
with session.begin():
recipe = data_setup.create_recipe(
task_list=[Task.by_name(u'/distribution/install')] * 2,
reservesys=True)
job = data_setup.create_job_for_recipes([recipe])
job_id = job.id
data_setup.mark_recipe_running(recipe)
data_setup.mark_recipe_installation_finished(recipe)
# we want at least one task to be Completed here
# https://bugzilla.redhat.com/show_bug.cgi?id=1195558
job.recipesets[0].recipes[0].tasks[0].stop()
job.recipesets[0].recipes[0].tasks[1].start()
beakerd.update_dirty_jobs()
with session.begin():
job = Job.by_id(job_id)
self.assertEqual(job.recipesets[0].recipes[0].status,
TaskStatus.running)
job.recipesets[0].cancel()
beakerd.update_dirty_jobs()
with session.begin():
job = Job.by_id(job_id)
self.assertEqual(job.recipesets[0].recipes[0].status,
TaskStatus.cancelled)
def test_first_failed_task_should_expand_when_first_loading(self):
with session.begin():
recipe = data_setup.create_recipe(task_list=[
Task.by_name(u'/distribution/install'),
Task.by_name(u'/distribution/reservesys')
])
job = data_setup.create_job_for_recipes([recipe])
data_setup.mark_recipe_tasks_finished(recipe, result=TaskResult.fail)
job.update_status()
b = self.browser
go_to_recipe_view(b, recipe)
# The in class is an indication that a task is expanded.
b.find_element_by_css_selector('#task%s .recipe-task-details.collapse.in'
% recipe.tasks[0].id)
_, fragment = urlparse.urldefrag(b.current_url)
self.assertEquals(fragment, 'task%s' % recipe.tasks[0].id)
def test_recipe_running_then_cancelled(self):
""" This tests the case where the recipe is running, has a valid
reservation request, but is cancelled before it's completed.
"""
with session.begin():
recipe = data_setup.create_recipe(
task_list=[Task.by_name(u'/distribution/install')] * 2,
reservesys=True)
job = data_setup.create_job_for_recipes([recipe])
job_id = job.id
data_setup.mark_recipe_running(recipe)
data_setup.mark_recipe_installation_finished(recipe)
# we want at least one task to be Completed here
# https://bugzilla.redhat.com/show_bug.cgi?id=1195558
job.recipesets[0].recipes[0].tasks[0].stop()
job.recipesets[0].recipes[0].tasks[1].start()
beakerd.update_dirty_jobs()
with session.begin():
job = Job.by_id(job_id)
self.assertEqual(job.recipesets[0].recipes[0].status,
TaskStatus.running)
job.recipesets[0].cancel()
beakerd.update_dirty_jobs()
with session.begin():
job = Job.by_id(job_id)
self.assertEqual(job.recipesets[0].recipes[0].status,
TaskStatus.cancelled)
def update_task(task_id):
"""
Updates a task - only handles disabling at this time.
:param task_id: The task id to update/disable
:jsonparam bool disabled: Whether the task should be disabled.
:status 200: Task was successfully updated/disabled
:status 404: Task was not found (to be disabled)
"""
try:
task = Task.by_id(task_id)
except DatabaseLookupError as e:
# This should be NotFound404 but due to still using cherrypy
# 404's are handled there which then will then do a GET /tasks/id
# which will resolve correctly, which isn't desired
raise NotFound404('Task %s does not exist' % task_id)
data = read_json_request(request)
if data:
with convert_internal_errors():
if data.get('disabled', False) and task.valid:
task.disable()
response = jsonify(task.to_dict())
return response
def test_first_failed_task_should_expand_when_first_loading(self):
with session.begin():
recipe = data_setup.create_recipe(task_list=[
Task.by_name(u'/distribution/install'),
Task.by_name(u'/distribution/reservesys')
])
job = data_setup.create_job_for_recipes([recipe])
data_setup.mark_recipe_tasks_finished(recipe, result=TaskResult.fail)
job.update_status()
b = self.browser
go_to_recipe_view(b, recipe)
# The in class is an indication that a task is expanded.
b.find_element_by_css_selector('#task%s .recipe-task-details.collapse.in'
% recipe.tasks[0].id)
_, fragment = urlparse.urldefrag(b.current_url)
self.assertEquals(fragment, 'task%s' % recipe.tasks[0].id)
def _disable(self, t_id, *args, **kw):
"""
disable task
task.valid=False
remove task rpms from filesystem
"""
task = Task.by_id(t_id)
return task.disable()
def _disable(self, t_id, *args, **kw):
"""
disable task
task.valid=False
remove task rpms from filesystem
"""
task = Task.by_id(t_id)
return task.disable()
def create_task(name=None,
exclude_arch=None,
exclude_osmajor=None,
version=u'1.0-1',
uploader=None,
owner=None,
priority=u'Manual',
valid=None,
path=None,
description=None,
requires=None,
runfor=None,
type=None):
if name is None:
name = unique_name(u'/distribution/test_task_%s')
if path is None:
path = u'/mnt/tests/%s' % name
if description is None:
description = unique_name(u'description%s')
if uploader is None:
uploader = create_user(user_name=u'task-uploader%s' %
name.replace('/', '-'))
if owner is None:
owner = u'*****@*****.**' % name.replace('/', '-')
if valid is None:
valid = True
rpm = u'example%s-%s.noarch.rpm' % (name.replace('/', '-'), version)
task = Task.lazy_create(name=name)
task.rpm = rpm
task.version = version
task.uploader = uploader
task.owner = owner
task.priority = priority
task.valid = valid
task.path = path
task.description = description
task.avg_time = 1200
if type:
for t in type:
task.types.append(TaskType.lazy_create(type=t))
if exclude_arch:
for arch in exclude_arch:
task.excluded_arch.append(
TaskExcludeArch(arch_id=Arch.by_name(arch).id))
if exclude_osmajor:
for osmajor in exclude_osmajor:
task.excluded_osmajor.append(
TaskExcludeOSMajor(osmajor=OSMajor.lazy_create(
osmajor=osmajor)))
if requires:
for require in requires:
tp = TaskPackage.lazy_create(package=require)
task.required.append(tp)
if runfor:
for run in runfor:
task.runfor.append(TaskPackage.lazy_create(package=run))
return task
def test_adding_task_with_releases_list(self):
with session.begin():
OSMajor.lazy_create(osmajor=u'RedHatEnterpriseLinux5')
OSMajor.lazy_create(osmajor=u'RedHatEnterpriseLinux6')
rpm_path = pkg_resources.resource_filename('bkr.inttest.server',
'task-rpms/tmp-distribution-beaker-dummy_for_bz1422410-1.0-1.noarch.rpm')
out = run_client(['bkr', 'task-add', rpm_path])
self.assertIn(u'Success', out)
with session.begin():
task = Task.by_name(u'/distribution/beaker/dummy_for_bz1422410')
self.assertItemsEqual([OSMajor.by_name(u'RedHatEnterpriseLinux5')],
task.exclusive_osmajors)
def test_reserved_then_job_cancelled(self):
""" This tests the case where the recipe is Reserved
but the job is cancelled
"""
recipe = data_setup.create_recipe(task_list=[Task.by_name(u"/distribution/install")], reservesys=True)
job = data_setup.create_job_for_recipes([recipe])
data_setup.mark_recipe_tasks_finished(recipe)
job.update_status()
self.assertEqual(job.recipesets[0].recipes[0].status, TaskStatus.reserved)
job.cancel()
job.update_status()
self.assertEqual(job.recipesets[0].recipes[0].status, TaskStatus.completed)
def test_adding_task_with_releases_list(self):
with session.begin():
OSMajor.lazy_create(osmajor=u'RedHatEnterpriseLinux5')
OSMajor.lazy_create(osmajor=u'RedHatEnterpriseLinux6')
rpm_path = pkg_resources.resource_filename('bkr.inttest.server',
'task-rpms/tmp-distribution-beaker-dummy_for_bz1422410-1.0-1.noarch.rpm')
out = run_client(['bkr', 'task-add', rpm_path])
self.assertIn(u'Success', out)
with session.begin():
task = Task.by_name(u'/distribution/beaker/dummy_for_bz1422410')
self.assertItemsEqual([OSMajor.by_name(u'RedHatEnterpriseLinux5')],
task.exclusive_osmajors)
def test_task_aborted_return_reservation(self):
""" This tests the case where the task was aborted, then
the recipe goes to Reserved state and then finally the reservation
is returned
"""
recipe = data_setup.create_recipe(task_list=[Task.by_name(u"/distribution/install")], reservesys=True)
job = data_setup.create_job_for_recipes([recipe])
data_setup.mark_recipe_tasks_finished(recipe, result=TaskResult.warn, task_status=TaskStatus.aborted)
job.update_status()
self.assertEqual(job.recipesets[0].recipes[0].status, TaskStatus.reserved)
job.recipesets[0].recipes[0].return_reservation()
job.update_status()
self.assertEqual(job.recipesets[0].recipes[0].status, TaskStatus.aborted)
def expiring_reservations(self):
"""
Get expiring reservations
"""
tasks = Task.by_name(u'/distribution/reservesys')
query = Recipe.query\
.join(Recipe.recipeset).join(RecipeSet.job).filter(Job.owner == self.user)\
.join(Recipe.watchdog).join(Watchdog.recipetask)\
.join(Recipe.resource)\
.filter(or_(RecipeTask.task == tasks, Recipe.status == TaskStatus.reserved))\
.filter(Watchdog.kill_time <= (datetime.utcnow() + timedelta(hours=self.reservation_expiry)))\
.values(Watchdog.kill_time, RecipeResource.fqdn)
return list(query)
def upload(self, task_rpm_name, task_rpm_data):
"""
Uploads a new task RPM.
:param task_rpm_name: filename of the task RPM, for example
``'beaker-distribution-install-1.10-11.noarch.rpm'``
:type task_rpm_name: string
:param task_rpm_data: contents of the task RPM
:type task_rpm_data: XML-RPC binary
"""
rpm_path = Task.get_rpm_path(task_rpm_name)
# we do it here, since we do not want to proceed
# any further
if len(task_rpm_name) > 255:
raise BX(_("Task RPM name should be <= 255 characters"))
if os.path.exists("%s" % rpm_path):
raise BX(_(u'Cannot import duplicate task %s') % task_rpm_name)
def write_data(f):
f.write(task_rpm_data.data)
Task.update_task(task_rpm_name, write_data)
return "Success"
def test_reserved_then_watchdog_expired(self):
""" This tests the case where the external
watchdog expires when the recipe is in Reserved state
"""
recipe = data_setup.create_recipe(task_list=[Task.by_name(u"/distribution/install")], reservesys=True)
job = data_setup.create_job_for_recipes([recipe])
data_setup.mark_recipe_tasks_finished(recipe)
job._mark_dirty()
job.update_status()
self.assertEqual(job.recipesets[0].recipes[0].status, TaskStatus.reserved)
job.recipesets[0].recipes[0].abort()
job.update_status()
self.assertEqual(job.recipesets[0].recipes[0].status, TaskStatus.completed)
def expiring_reservations(self):
"""
Get expiring reservations
"""
tasks = Task.by_name(u'/distribution/reservesys')
query = Recipe.query\
.join(Recipe.recipeset).join(RecipeSet.job).filter(Job.owner == self.user)\
.join(Recipe.watchdog).join(Watchdog.recipetask)\
.join(Recipe.resource)\
.filter(or_(RecipeTask.task == tasks, Recipe.status == TaskStatus.reserved))\
.filter(Watchdog.kill_time <= (datetime.utcnow() + timedelta(hours=self.reservation_expiry)))\
.values(Watchdog.kill_time, RecipeResource.fqdn)
return list(query)
def test_recipe_running_then_watchdog_expired(self):
""" This tests the case where the recipe is running, has a valid
reservation request, but the watchdog expires before it's
completed.
"""
recipe = data_setup.create_recipe(task_list=[Task.by_name(u"/distribution/install")], reservesys=True)
job = data_setup.create_job_for_recipes([recipe])
data_setup.mark_recipe_tasks_finished(recipe, task_status=TaskStatus.aborted)
job.recipesets[0].recipes[0].abort()
job.update_status()
self.assertEqual(job.recipesets[0].recipes[0].status, TaskStatus.reserved)
job.recipesets[0].recipes[0].return_reservation()
job.update_status()
self.assertEqual(job.recipesets[0].recipes[0].status, TaskStatus.aborted)
def upload(self, task_rpm_name, task_rpm_data):
"""
Uploads a new task RPM.
:param task_rpm_name: filename of the task RPM, for example
``'beaker-distribution-install-1.10-11.noarch.rpm'``
:type task_rpm_name: string
:param task_rpm_data: contents of the task RPM
:type task_rpm_data: XML-RPC binary
"""
rpm_path = Task.get_rpm_path(task_rpm_name)
# we do it here, since we do not want to proceed
# any further
if len(task_rpm_name) > 255:
raise BX(_("Task RPM name should be <= 255 characters"))
if os.path.exists("%s" % rpm_path):
raise BX(_(u'Cannot import duplicate task %s') % task_rpm_name)
def write_data(f):
f.write(task_rpm_data.data)
Task.update_task(task_rpm_name, write_data)
return "Success"
def test_reserves_system_when_recipe_waiting(self):
recipe = data_setup.create_recipe(task_list=[Task.by_name(u"/distribution/utils/dummy")], reservesys=True)
job = data_setup.create_job_for_recipes([recipe])
# Anaconda installs the OS (Status: Installing) and reboots
# beakerd comes along and calls update_dirty_jobs which sets the recipe to: Waiting
data_setup.mark_recipe_waiting(recipe)
# In the meantime however our task has finished really quickly, which
# means the min_status is TaskStatus.completed and therefore finished
data_setup.mark_recipe_tasks_finished(recipe, only=True)
# beakerd hasn't come along and updated our recipe yet, so it's still
# in waiting
self.assertEqual(job.recipesets[0].recipes[0].status, TaskStatus.waiting)
# Now beakerd updates it and should reserve our system
job.update_status()
self.assertEqual(job.recipesets[0].recipes[0].status, TaskStatus.reserved)
def create_recipe(distro_tree=None, task_list=None,
task_name=u'/distribution/reservesys', num_tasks=None, whiteboard=None,
role=None, ks_meta=None, cls=MachineRecipe, **kwargs):
recipe = cls(ttasks=1)
recipe.ks_meta = ks_meta
recipe.whiteboard = whiteboard
recipe.distro_tree = distro_tree
recipe.role = role or u'STANDALONE'
custom_distro = kwargs.get('custom_distro', False)
if not custom_distro:
if not distro_tree:
distro_tree = create_distro_tree(**kwargs)
recipe.distro_tree = distro_tree
recipe.installation = recipe.distro_tree.create_installation_from_tree()
recipe.distro_requires = lxml.etree.tostring(recipe.distro_tree.to_xml(), encoding=unicode)
else:
name = kwargs.get('distro_name', u'MyAwesomeLinux1.0')
tree_url = kwargs.get('tree_url', u'ftp://dummylab.example.com/distros/MyAwesomeLinux1/')
initrd_path = kwargs.get('initrd_path', u'pxeboot/initrd')
kernel_path = kwargs.get('kernel_path', u'pxeboot/vmlinuz')
arch = kwargs.get('arch', u'i386')
variant = kwargs.get('variant', u'Server')
osmajor = kwargs.get('osmajor', u'DansAwesomeLinux6')
osminor = kwargs.get('osminor', u'0')
arch = Arch.by_name(arch)
recipe.installation = Installation(tree_url=tree_url, initrd_path=initrd_path, kernel_path=kernel_path, arch=arch,
distro_name=name, osmajor=osmajor, osminor=osminor, variant=variant)
if kwargs.get('reservesys', False):
recipe.reservation_request = RecipeReservationRequest()
if kwargs.get('reservesys_duration'):
recipe.reservation_request.duration = kwargs['reservesys_duration']
if num_tasks:
task_list = [create_task() for i in range(0, num_tasks)]
if not task_list: #don't specify a task_list and a task_name...
try:
task = Task.by_name(task_name)
except LookupError:
task = create_task(name=task_name)
task_list = [task]
for t in task_list:
rt = RecipeTask.from_task(t)
rt.role = u'STANDALONE'
recipe.tasks.append(rt)
recipe.ttasks = len(task_list)
return recipe
def create_recipe(distro_tree=None, task_list=None,
task_name=u'/distribution/reservesys', num_tasks=None, whiteboard=None,
role=None, ks_meta=None, cls=MachineRecipe, **kwargs):
recipe = cls(ttasks=1)
recipe.ks_meta = ks_meta
recipe.whiteboard = whiteboard
recipe.distro_tree = distro_tree
recipe.role = role or u'STANDALONE'
custom_distro = kwargs.get('custom_distro', False)
if not custom_distro:
if not distro_tree:
distro_tree = create_distro_tree(**kwargs)
recipe.distro_tree = distro_tree
recipe.installation = recipe.distro_tree.create_installation_from_tree()
recipe.distro_requires = lxml.etree.tostring(recipe.distro_tree.to_xml(), encoding=unicode)
else:
name = kwargs.get('distro_name', u'MyAwesomeLinux1.0')
tree_url = kwargs.get('tree_url', u'ftp://dummylab.example.com/distros/MyAwesomeLinux1/')
initrd_path = kwargs.get('initrd_path', u'pxeboot/initrd')
kernel_path = kwargs.get('kernel_path', u'pxeboot/vmlinuz')
arch = kwargs.get('arch', u'i386')
variant = kwargs.get('variant', u'Server')
osmajor = kwargs.get('osmajor', u'DansAwesomeLinux6')
osminor = kwargs.get('osminor', u'0')
arch = Arch.by_name(arch)
recipe.installation = Installation(tree_url=tree_url, initrd_path=initrd_path, kernel_path=kernel_path, arch=arch,
distro_name=name, osmajor=osmajor, osminor=osminor, variant=variant)
if kwargs.get('reservesys', False):
recipe.reservation_request = RecipeReservationRequest()
if kwargs.get('reservesys_duration'):
recipe.reservation_request.duration = kwargs['reservesys_duration']
if num_tasks:
task_list = [create_task() for i in range(0, num_tasks)]
if not task_list: #don't specify a task_list and a task_name...
try:
task = Task.by_name(task_name)
except LookupError:
task = create_task(name=task_name)
task_list = [task]
for t in task_list:
rt = RecipeTask.from_task(t)
rt.role = u'STANDALONE'
recipe.tasks.append(rt)
recipe.ttasks = len(task_list)
return recipe
def test_reserved_then_job_cancelled(self):
""" This tests the case where the recipe is Reserved
but the job is cancelled
"""
recipe = data_setup.create_recipe(
task_list=[Task.by_name(u'/distribution/install')],
reservesys=True)
job = data_setup.create_job_for_recipes([recipe])
data_setup.mark_recipe_tasks_finished(recipe)
job.update_status()
self.assertEqual(job.recipesets[0].recipes[0].status,
TaskStatus.reserved)
job.cancel()
job.update_status()
self.assertEqual(job.recipesets[0].recipes[0].status,
TaskStatus.completed)
def test_recipe_running_then_cancelled(self):
""" This tests the case where the recipe is running, has a valid
reservation request, but is cancelled before it's completed.
"""
recipe = data_setup.create_recipe(task_list=[Task.by_name(u"/distribution/install")] * 2, reservesys=True)
job = data_setup.create_job_for_recipes([recipe])
data_setup.mark_recipe_running(recipe)
# we want at least one task to be Completed here
# https://bugzilla.redhat.com/show_bug.cgi?id=1195558
job.recipesets[0].recipes[0].tasks[0].stop()
job.recipesets[0].recipes[0].tasks[1].start()
job.update_status()
self.assertEqual(job.recipesets[0].recipes[0].status, TaskStatus.running)
job.recipesets[0].cancel()
job.update_status()
self.assertEqual(job.recipesets[0].recipes[0].status, TaskStatus.cancelled)
def test_reserved_then_watchdog_expired(self):
""" This tests the case where the external
watchdog expires when the recipe is in Reserved state
"""
recipe = data_setup.create_recipe(
task_list=[Task.by_name(u'/distribution/install')],
reservesys=True)
job = data_setup.create_job_for_recipes([recipe])
data_setup.mark_recipe_tasks_finished(recipe)
job._mark_dirty()
job.update_status()
self.assertEqual(job.recipesets[0].recipes[0].status,
TaskStatus.reserved)
job.recipesets[0].recipes[0].abort()
job.update_status()
self.assertEqual(job.recipesets[0].recipes[0].status,
TaskStatus.completed)
def create_task(name=None, exclude_arch=None, exclude_osmajor=None, version=u'1.0-1',
uploader=None, owner=None, priority=u'Manual', valid=None, path=None,
description=None, requires=None, runfor=None, type=None):
if name is None:
name = unique_name(u'/distribution/test_task_%s')
if path is None:
path = u'/mnt/tests/%s' % name
if description is None:
description = unique_name(u'description%s')
if uploader is None:
uploader = create_user(user_name=u'task-uploader%s' % name.replace('/', '-'))
if owner is None:
owner = u'*****@*****.**' % name.replace('/', '-')
if valid is None:
valid = True
rpm = u'example%s-%s.noarch.rpm' % (name.replace('/', '-'), version)
task = Task.lazy_create(name=name)
task.rpm = rpm
task.version = version
task.uploader = uploader
task.owner = owner
task.priority = priority
task.valid = valid
task.path = path
task.description = description
task.avg_time = 1200
if type:
for t in type:
task.types.append(TaskType.lazy_create(type=t))
if exclude_arch:
for arch in exclude_arch:
task.excluded_arch.append(TaskExcludeArch(arch_id=Arch.by_name(arch).id))
if exclude_osmajor:
for osmajor in exclude_osmajor:
task.excluded_osmajor.append(TaskExcludeOSMajor(osmajor=OSMajor.lazy_create(osmajor=osmajor)))
if requires:
for require in requires:
tp = TaskPackage.lazy_create(package=require)
task.required.append(tp)
if runfor:
for run in runfor:
task.runfor.append(TaskPackage.lazy_create(package=run))
return task
def test_task_aborted_return_reservation(self):
""" This tests the case where the task was aborted, then
the recipe goes to Reserved state and then finally the reservation
is returned
"""
recipe = data_setup.create_recipe(
task_list=[Task.by_name(u'/distribution/install')],
reservesys=True)
job = data_setup.create_job_for_recipes([recipe])
data_setup.mark_recipe_tasks_finished(recipe,
result=TaskResult.warn,
task_status=TaskStatus.aborted)
job.update_status()
self.assertEqual(job.recipesets[0].recipes[0].status,
TaskStatus.reserved)
job.recipesets[0].recipes[0].return_reservation()
job.update_status()
self.assertEqual(job.recipesets[0].recipes[0].status,
TaskStatus.aborted)
def _get_task_xml(self, server, task):
# if local, directly read the database
if server == 'local':
try:
self.logger.debug('Getting task XML for %s from local database' % task)
return Task.by_name(task, True).to_xml(False)
except Exception:
self.logger.error('Could not get task XML for %s from local Beaker DB. Continuing.' % task)
return None
try:
self.logger.debug('Getting task XML for %s from %s' % (task, getattr(self, server)))
return self.proxy.tasks.to_xml(task, False)
except (xmlrpclib.Fault, xmlrpclib.ProtocolError) as e:
# If something goes wrong with this task, for example:
# https://bugzilla.redhat.com/show_bug.cgi?id=915549
# we do our best to continue anyway...
self.logger.error('Could not get task XML for %s from %s. Continuing.' % (task, server))
self.logger.error('Error message: %s' % e)
return None
def test_reserved_then_job_cancelled(self):
""" This tests the case where the recipe is Reserved
but the job is cancelled
"""
with session.begin():
recipe = data_setup.create_recipe(
task_list=[Task.by_name(u'/distribution/install')],
reservesys=True)
job = data_setup.create_job_for_recipes([recipe])
job_id = job.id
data_setup.mark_recipe_tasks_finished(recipe)
job._mark_dirty()
beakerd.update_dirty_jobs()
with session.begin():
job = Job.by_id(job_id)
self.assertEqual(job.recipesets[0].recipes[0].status,
TaskStatus.reserved)
job.cancel()
beakerd.update_dirty_jobs()
with session.begin():
job = Job.by_id(job_id)
self.assertEqual(job.recipesets[0].recipes[0].status,
TaskStatus.completed)
def test_reserved_then_job_cancelled(self):
""" This tests the case where the recipe is Reserved
but the job is cancelled
"""
with session.begin():
recipe = data_setup.create_recipe(
task_list=[Task.by_name(u'/distribution/install')],
reservesys=True)
job = data_setup.create_job_for_recipes([recipe])
job_id = job.id
data_setup.mark_recipe_tasks_finished(recipe)
job._mark_dirty()
beakerd.update_dirty_jobs()
with session.begin():
job = Job.by_id(job_id)
self.assertEqual(job.recipesets[0].recipes[0].status,
TaskStatus.reserved)
job.cancel()
beakerd.update_dirty_jobs()
with session.begin():
job = Job.by_id(job_id)
self.assertEqual(job.recipesets[0].recipes[0].status,
TaskStatus.completed)
def test_recipe_running_then_cancelled(self):
""" This tests the case where the recipe is running, has a valid
reservation request, but is cancelled before it's completed.
"""
with session.begin():
recipe = data_setup.create_recipe(
task_list=[Task.by_name(u'/distribution/install')],
reservesys=True)
job = data_setup.create_job_for_recipes([recipe])
job_id = job.id
data_setup.mark_recipe_running(recipe)
job._mark_dirty()
beakerd.update_dirty_jobs()
with session.begin():
job = Job.by_id(job_id)
self.assertEqual(job.recipesets[0].recipes[0].status,
TaskStatus.running)
job.recipesets[0].cancel()
beakerd.update_dirty_jobs()
with session.begin():
job = Job.by_id(job_id)
self.assertEqual(job.recipesets[0].recipes[0].status,
TaskStatus.cancelled)
def test_reserved_then_watchdog_expired(self):
""" This tests the case where the external
watchdog expires when the recipe is in Reserved state
"""
with session.begin():
recipe = data_setup.create_recipe(
task_list=[Task.by_name(u'/distribution/install')],
reservesys=True)
job = data_setup.create_job_for_recipes([recipe])
job_id = job.id
data_setup.mark_recipe_tasks_finished(recipe)
job._mark_dirty()
beakerd.update_dirty_jobs()
with session.begin():
job = Job.by_id(job_id)
self.assertEqual(job.recipesets[0].recipes[0].status,
TaskStatus.reserved)
job.recipesets[0].recipes[0].abort()
job._update_status()
with session.begin():
job = Job.by_id(job_id)
self.assertEqual(job.recipesets[0].recipes[0].status,
TaskStatus.completed)
def test_recipe_running_then_cancelled(self):
""" This tests the case where the recipe is running, has a valid
reservation request, but is cancelled before it's completed.
"""
with session.begin():
recipe = data_setup.create_recipe(
task_list=[Task.by_name(u'/distribution/install')],
reservesys=True)
job = data_setup.create_job_for_recipes([recipe])
job_id = job.id
data_setup.mark_recipe_running(recipe)
job._mark_dirty()
beakerd.update_dirty_jobs()
with session.begin():
job = Job.by_id(job_id)
self.assertEqual(job.recipesets[0].recipes[0].status,
TaskStatus.running)
job.recipesets[0].cancel()
beakerd.update_dirty_jobs()
with session.begin():
job = Job.by_id(job_id)
self.assertEqual(job.recipesets[0].recipes[0].status,
TaskStatus.cancelled)
def to_xml(self, name, pretty, valid=True):
"""
Returns task details as xml
"""
return Task.by_name(name, valid).to_xml(pretty)
package = TaskPackage.lazy_create(package='%s' % xmlpackage.name)
recipe.custom_packages.append(package)
for installPackage in xmlrecipe.installPackages():
package = TaskPackage.lazy_create(package='%s' % installPackage)
recipe.custom_packages.append(package)
for xmlrepo in xmlrecipe.iter_repos():
recipe.repos.append(RecipeRepo(name=xmlrepo.name, url=xmlrepo.url))
for xmlksappend in xmlrecipe.iter_ksappends():
recipe.ks_appends.append(RecipeKSAppend(ks_append=xmlksappend))
xmltasks = []
invalid_tasks = []
for xmltask in xmlrecipe.iter_tasks():
if hasattr(xmltask, 'fetch'):
# If fetch URL is given, the task doesn't need to exist.
xmltasks.append(xmltask)
elif Task.exists_by_name(xmltask.name, valid=True):
xmltasks.append(xmltask)
else:
invalid_tasks.append(xmltask.name)
if invalid_tasks and not ignore_missing_tasks:
raise BX(_('Invalid task(s): %s') % ', '.join(invalid_tasks))
for xmltask in xmltasks:
if hasattr(xmltask, 'fetch'):
recipetask = RecipeTask.from_fetch_url(xmltask.fetch.url,
subdir=xmltask.fetch.subdir, name=xmltask.name)
else:
recipetask = RecipeTask.from_task(Task.by_name(xmltask.name))
recipetask.role = xmltask.role
for xmlparam in xmltask.iter_params():
param = RecipeTaskParam( name=xmlparam.name,
value=xmlparam.value)
def filter(self, filter):
"""
Returns a list of tasks filtered by the given criteria.
The *filter* argument must be an XML-RPC structure (dict), with any of the following keys:
'distro_name'
Distro name. Include only tasks which are compatible
with this distro.
'osmajor'
OSVersion OSMajor, like RedHatEnterpriseLinux6. Include only
tasks which are compatible with this OSMajor.
'names'
Task name. Include only tasks that are named. Useful when
combined with 'osmajor' or 'distro_name'.
'packages'
List of package names. Include only tasks which have a Run-For
entry matching any of these packages.
'types'
List of task types. Include only tasks which have one or more
of these types.
'valid'
bool 0 or 1. Include only tasks which are valid or not.
'destructive'
bool 0 or 1. Set to 0 for only non-destructive tasks. Set to
1 for only destructive tasks.
The return value is an array of dicts, which are name and arches.
name is the name of the matching tasks.
arches is an array of arches which this task does not apply for.
Call :meth:`tasks.to_dict` to fetch metadata for a particular task.
.. versionchanged:: 0.9
Changed 'install_name' to 'distro_name' in the *filter* argument.
"""
tasks = Task.query
if filter.get('distro_name'):
distro = Distro.by_name(filter['distro_name'])
tasks = tasks.filter(Task.compatible_with(distro=distro))
elif 'osmajor' in filter and filter['osmajor']:
try:
osmajor = OSMajor.by_name(filter['osmajor'])
except InvalidRequestError:
raise BX(_('Invalid OSMajor: %s' % filter['osmajor']))
tasks = tasks.filter(Task.compatible_with(osmajor=osmajor))
# Filter by valid task if requested
if 'valid' in filter:
tasks = tasks.filter(Task.valid == bool(filter['valid']))
# Filter by destructive if requested
if 'destructive' in filter:
tasks = tasks.filter(
Task.destructive == bool(filter['destructive']))
# Filter by name if specified
# /distribution/install, /distribution/reservesys
if 'names' in filter and filter['names']:
# if not a list, make it into a list.
if isinstance(filter['names'], str):
filter['names'] = [filter['names']]
or_names = []
for tname in filter['names']:
or_names.append(Task.name == tname)
tasks = tasks.filter(or_(*or_names))
# Filter by packages if specified
# apache, kernel, mysql, etc..
if 'packages' in filter and filter['packages']:
# if not a list, make it into a list.
if isinstance(filter['packages'], str):
filter['packages'] = [filter['packages']]
tasks = tasks.filter(
Task.runfor.any(
or_(*[
TaskPackage.package == package
for package in filter['packages']
])))
# Filter by type if specified
# Tier1, Regression, KernelTier1, etc..
if 'types' in filter and filter['types']:
# if not a list, make it into a list.
if isinstance(filter['types'], str):
filter['types'] = [filter['types']]
tasks = tasks.join('types')
or_types = []
for type in filter['types']:
try:
tasktype = TaskType.by_name(type)
except InvalidRequestError, err:
raise BX(_('Invalid Task Type: %s' % type))
or_types.append(TaskType.id == tasktype.id)
tasks = tasks.filter(or_(*or_types))
def to_xml(self, name, pretty, valid=True):
"""
Returns task details as xml
"""
return Task.by_name(name, valid).to_xml(pretty)
def to_dict(self, name, valid=None):
"""
Returns an XML-RPC structure (dict) with details about the given task.
"""
return Task.by_name(name, valid).to_dict()
def to_dict(self, name, valid=None):
"""
Returns an XML-RPC structure (dict) with details about the given task.
"""
return Task.by_name(name, valid).to_dict()
custom_packages.add(package)
for installPackage in xmlrecipe.installPackages():
package = TaskPackage.lazy_create(package='%s' % installPackage)
custom_packages.add(package)
recipe.custom_packages = list(custom_packages)
for xmlrepo in xmlrecipe.iter_repos():
recipe.repos.append(RecipeRepo(name=xmlrepo.name, url=xmlrepo.url))
for xmlksappend in xmlrecipe.iter_ksappends():
recipe.ks_appends.append(RecipeKSAppend(ks_append=xmlksappend))
xmltasks = []
invalid_tasks = []
for xmltask in xmlrecipe.iter_tasks():
if hasattr(xmltask, 'fetch'):
# If fetch URL is given, the task doesn't need to exist.
xmltasks.append(xmltask)
elif Task.exists_by_name(xmltask.name, valid=True):
xmltasks.append(xmltask)
else:
invalid_tasks.append(xmltask.name)
if invalid_tasks and not ignore_missing_tasks:
raise BX(_('Invalid task(s): %s') % ', '.join(invalid_tasks))
for xmltask in xmltasks:
if hasattr(xmltask, 'fetch'):
recipetask = RecipeTask.from_fetch_url(
xmltask.fetch.url,
subdir=xmltask.fetch.subdir,
name=xmltask.name)
else:
recipetask = RecipeTask.from_task(Task.by_name(xmltask.name))
recipetask.role = xmltask.role
for xmlparam in xmltask.iter_params():
