def _handle_recipe_set(self, xmlrecipeSet, user, ignore_missing_tasks=False):
"""
Handles the processing of recipesets into DB entries from their xml
"""
recipeSet = RecipeSet(ttasks=0)
recipeset_priority = xmlrecipeSet.get_xml_attr('priority',unicode,None)
if recipeset_priority is not None:
try:
my_priority = TaskPriority.from_string(recipeset_priority)
except InvalidRequestError:
raise BX(_('You have specified an invalid recipeSet priority:%s' % recipeset_priority))
allowed_priorities = RecipeSet.allowed_priorities_initial(user)
if my_priority in allowed_priorities:
recipeSet.priority = my_priority
else:
recipeSet.priority = TaskPriority.default_priority()
else:
recipeSet.priority = TaskPriority.default_priority()
for xmlrecipe in xmlrecipeSet.iter_recipes():
recipe = self.handleRecipe(xmlrecipe, user,
ignore_missing_tasks=ignore_missing_tasks)
recipe.ttasks = len(recipe.tasks)
recipeSet.ttasks += recipe.ttasks
recipeSet.recipes.append(recipe)
# We want the guests to be part of the same recipeSet
for guest in recipe.guests:
recipeSet.recipes.append(guest)
guest.ttasks = len(guest.tasks)
recipeSet.ttasks += guest.ttasks
if not recipeSet.recipes:
raise BX(_('No Recipes! You can not have a recipeSet with no recipes!'))
return recipeSet
def create_recipeset_for_recipes(recipes, priority=None, queue_time=None, **kwargs):
if priority is None:
priority = TaskPriority.default_priority()
recipe_set = RecipeSet(ttasks=sum(r.ttasks for r in recipes),
priority=priority)
if queue_time is not None:
recipe_set.queue_time = queue_time
recipe_set.recipes.extend(recipes)
return recipe_set
def create_recipeset_for_recipes(recipes, priority=None, queue_time=None, **kwargs):
if priority is None:
priority = TaskPriority.default_priority()
recipe_set = RecipeSet(ttasks=sum(r.ttasks for r in recipes),
priority=priority)
if queue_time is not None:
recipe_set.queue_time = queue_time
recipe_set.recipes.extend(recipes)
return recipe_set
def create_job_for_recipes(recipes, owner=None, whiteboard=None, cc=None,product=None,
retention_tag=None, group=None, submitter=None, priority=None, **kwargs):
if retention_tag is None:
retention_tag = RetentionTag.by_tag(u'scratch') # Don't use default, unpredictable
else:
retention_tag = RetentionTag.by_tag(retention_tag)
if owner is None:
owner = create_user()
if whiteboard is None:
whiteboard = unique_name(u'job %s')
job = Job(whiteboard=whiteboard, ttasks=sum(r.ttasks for r in recipes),
owner=owner, retention_tag=retention_tag, group=group, product=product,
submitter=submitter)
if cc is not None:
job.cc = cc
if priority is None:
priority = TaskPriority.default_priority()
recipe_set = RecipeSet(ttasks=sum(r.ttasks for r in recipes),
priority=priority)
recipe_set.recipes.extend(recipes)
job.recipesets.append(recipe_set)
session.add(job)
session.flush()
log.debug('Created %s', job.t_id)
return job
def clone(self, job_id=None, recipe_id=None, recipeset_id=None,
textxml=None, filexml=None, confirmed=False, **kw):
"""
Review cloned xml before submitting it.
"""
title = 'Clone Job'
if job_id:
# Clone from Job ID
title = 'Clone Job %s' % job_id
try:
job = Job.by_id(job_id)
except InvalidRequestError:
flash(_(u"Invalid job id %s" % job_id))
redirect(".")
textxml = job.to_xml(clone=True).toprettyxml()
elif recipeset_id:
title = 'Clone Recipeset %s' % recipeset_id
try:
recipeset = RecipeSet.by_id(recipeset_id)
except InvalidRequestError:
flash(_(u"Invalid recipeset id %s" % recipeset_id))
redirect(".")
textxml = recipeset.to_xml(clone=True,from_job=False).toprettyxml()
elif isinstance(filexml, cgi.FieldStorage):
# Clone from file
try:
textxml = filexml.value.decode('utf8')
except UnicodeDecodeError, e:
flash(_(u'Invalid job XML: %s') % e)
redirect('.')
def update_recipe_set_response(self,recipe_set_id,response_id):
rs = RecipeSet.by_id(recipe_set_id)
if rs.nacked is None:
rs.nacked = RecipeSetResponse(response_id=response_id)
else:
rs.nacked.response = Response.by_id(response_id)
return {'success' : 1, 'rs_id' : recipe_set_id }
def provision_scheduled_recipeset(recipeset_id):
recipeset = RecipeSet.by_id(recipeset_id)
# Go through each recipe in the recipeSet
for recipe in recipeset.recipes:
try:
recipe.waiting()
recipe.provision()
except Exception, e:
log.exception("Failed to provision recipeid %s", recipe.id)
recipe.recipeset.abort(u"Failed to provision recipeid %s, %s" %
(recipe.id, e))
return
def provision_scheduled_recipeset(recipeset_id):
recipeset = RecipeSet.by_id(recipeset_id)
# Go through each recipe in the recipeSet
for recipe in recipeset.recipes:
try:
recipe.waiting()
recipe.provision()
except Exception, e:
log.exception("Failed to provision recipeid %s", recipe.id)
recipe.recipeset.abort(u"Failed to provision recipeid %s, %s"
% (recipe.id, e))
return
def stop(self, recipeset_id, stop_type, msg=None):
"""
Set recipeset status to Completed
"""
try:
recipeset = RecipeSet.by_id(recipeset_id)
except InvalidRequestError:
raise BX(_('Invalid recipeset ID: %s' % recipeset_id))
if stop_type not in recipeset.stop_types:
raise BX(_('Invalid stop_type: %s, must be one of %s' %
(stop_type, recipeset.stop_types)))
kwargs = dict(msg = msg)
return getattr(recipeset,stop_type)(**kwargs)
def update_recipe_set_response(self,recipe_set_id,response_id):
rs = RecipeSet.by_id(recipe_set_id)
old_response = None
if rs.nacked is None:
rs.nacked = RecipeSetResponse(response_id=response_id)
else:
old_response = rs.nacked.response
rs.nacked.response = Response.by_id(response_id)
rs.record_activity(user=identity.current.user, service=u'WEBUI',
field=u'Ack/Nak', action=u'Changed', old=old_response,
new=rs.nacked.response)
return {'success' : 1, 'rs_id' : recipe_set_id }
def stop(self, recipeset_id, stop_type, msg=None):
"""
Set recipeset status to Completed
"""
try:
recipeset = RecipeSet.by_id(recipeset_id)
except InvalidRequestError:
raise BX(_('Invalid recipeset ID: %s' % recipeset_id))
if stop_type not in recipeset.stop_types:
raise BX(
_('Invalid stop_type: %s, must be one of %s' %
(stop_type, recipeset.stop_types)))
kwargs = dict(msg=msg)
return getattr(recipeset, stop_type)(**kwargs)
def provision_scheduled_recipeset(recipeset_id):
recipeset = RecipeSet.by_id(recipeset_id)
# Go through each recipe in the recipeSet
for recipe in recipeset.recipes:
log.debug('Provisioning recipe %s in RS:%s', recipe.id, recipeset_id)
try:
recipe.waiting()
recipe.provision()
except Exception as e:
log.exception("Failed to provision recipeid %s", recipe.id)
session.rollback()
session.begin()
recipe.recipeset.abort(u"Failed to provision recipeid %s, %s" % (recipe.id, e))
return
def provision_scheduled_recipeset(recipeset_id):
recipeset = RecipeSet.by_id(recipeset_id)
# Go through each recipe in the recipeSet
for recipe in recipeset.recipes:
log.debug('Provisioning recipe %s in RS:%s', recipe.id, recipeset_id)
try:
recipe.waiting()
recipe.provision()
except Exception as e:
log.exception("Failed to provision recipeid %s", recipe.id)
session.rollback()
session.begin()
recipe.recipeset.abort(u"Failed to provision recipeid %s, %s" % (recipe.id, e))
return
def really_cancel(self, id, msg=None):
"""
Confirm cancel recipeset
"""
try:
recipeset = RecipeSet.by_id(id)
except InvalidRequestError:
flash(_(u"Invalid recipeset id %s" % id))
redirect("/jobs/%s" % recipeset.job.id)
if not recipeset.can_cancel(identity.current.user):
flash(_(u"You don't have permission to cancel recipeset id %s" % id))
redirect("/jobs/%s" % recipeset.job.id)
recipeset.cancel(msg)
flash(_(u"Successfully cancelled recipeset %s" % id))
redirect("/jobs/%s" % recipeset.job.id)
def update_recipe_set_response(self, recipe_set_id, response_id):
rs = RecipeSet.by_id(recipe_set_id)
old_response = None
if rs.nacked is None:
rs.nacked = RecipeSetResponse(response_id=response_id)
else:
old_response = rs.nacked.response
rs.nacked.response = Response.by_id(response_id)
rs.record_activity(user=identity.current.user,
service=u'WEBUI',
field=u'Ack/Nak',
action=u'Changed',
old=old_response,
new=rs.nacked.response)
return {'success': 1, 'rs_id': recipe_set_id}
def really_cancel(self, id, msg=None):
"""
Confirm cancel recipeset
"""
try:
recipeset = RecipeSet.by_id(id)
except InvalidRequestError:
flash(_(u"Invalid recipeset id %s" % id))
redirect("/jobs/%s" % recipeset.job.id)
if not recipeset.can_cancel(identity.current.user):
flash(_(u"You don't have permission to cancel recipeset id %s" % id))
redirect("/jobs/%s" % recipeset.job.id)
recipeset.cancel(msg)
recipeset.record_activity(user=identity.current.user, service=u'WEBUI',
field=u'Status', action=u'Cancelled', old='',
new='')
flash(_(u"Successfully cancelled recipeset %s" % id))
redirect("/jobs/%s" % recipeset.job.id)
def queue_processed_recipesets(*args):
work_done = False
with session.begin():
recipesets = RecipeSet.by_recipe_status(TaskStatus.processed)\
.order_by(RecipeSet.priority.desc())\
.order_by(RecipeSet.id)
recipeset_ids = [rs_id for rs_id, in recipesets.values(RecipeSet.id)]
if recipeset_ids:
log.debug('Queuing processed recipe sets [%s ... %s] (%d total)',
recipeset_ids[0], recipeset_ids[-1], len(recipeset_ids))
for rs_id in recipeset_ids:
session.begin()
try:
queue_processed_recipeset(rs_id)
session.commit()
except Exception, e:
log.exception('Error in queue_processed_recipeset(%s)', rs_id)
session.rollback()
finally:
def queue_processed_recipesets(*args):
work_done = False
with session.begin():
recipesets = RecipeSet.by_recipe_status(TaskStatus.processed)\
.order_by(RecipeSet.priority.desc())\
.order_by(RecipeSet.id)
recipeset_ids = [rs_id for rs_id, in recipesets.values(RecipeSet.id)]
if recipeset_ids:
log.debug('Queuing processed recipe sets [%s ... %s] (%d total)',
recipeset_ids[0], recipeset_ids[-1], len(recipeset_ids))
for rs_id in recipeset_ids:
session.begin()
try:
queue_processed_recipeset(rs_id)
session.commit()
except Exception, e:
log.exception('Error in queue_processed_recipeset(%s)', rs_id)
session.rollback()
finally:
def cancel(self, id):
"""
Confirm cancel recipeset
"""
try:
recipeset = RecipeSet.by_id(id)
except InvalidRequestError:
flash(_(u"Invalid recipeset id %s" % id))
redirect("/jobs/%s" % recipeset.job.id)
if not recipeset.can_cancel(identity.current.user):
flash(_(u"You don't have permission to cancel recipeset id %s" % id))
redirect("/jobs/%s" % recipeset.job.id)
return dict(
title = 'Cancel RecipeSet %s' % id,
form = self.cancel_form,
action = './really_cancel',
options = {},
value = dict(id = recipeset.id,
confirm = 'really cancel recipeset %s?' % id),
)
def cancel(self, id):
"""
Confirm cancel recipeset
"""
try:
recipeset = RecipeSet.by_id(id)
except InvalidRequestError:
flash(_(u"Invalid recipeset id %s" % id))
redirect("/jobs/%s" % recipeset.job.id)
if not recipeset.can_cancel(identity.current.user):
flash(
_(u"You don't have permission to cancel recipeset id %s" % id))
redirect("/jobs/%s" % recipeset.job.id)
return dict(
title='Cancel RecipeSet %s' % id,
form=self.cancel_form,
action='./really_cancel',
options={},
value=dict(id=recipeset.id,
confirm='really cancel recipeset %s?' % id),
)
def really_cancel(self, id, msg=None):
"""
Confirm cancel recipeset
"""
try:
recipeset = RecipeSet.by_id(id)
except InvalidRequestError:
flash(_(u"Invalid recipeset id %s" % id))
redirect("/jobs/%s" % recipeset.job.id)
if not recipeset.can_cancel(identity.current.user):
flash(
_(u"You don't have permission to cancel recipeset id %s" % id))
redirect("/jobs/%s" % recipeset.job.id)
recipeset.cancel(msg)
recipeset.record_activity(user=identity.current.user,
service=u'WEBUI',
field=u'Status',
action=u'Cancelled',
old='',
new='')
flash(_(u"Successfully cancelled recipeset %s" % id))
redirect("/jobs/%s" % recipeset.job.id)
def clone(self,
job_id=None,
recipe_id=None,
recipeset_id=None,
textxml=None,
filexml=None,
confirmed=False,
**kw):
"""
Review cloned xml before submitting it.
"""
title = 'Clone Job'
if job_id:
# Clone from Job ID
title = 'Clone Job %s' % job_id
try:
job = Job.by_id(job_id)
except InvalidRequestError:
flash(_(u"Invalid job id %s" % job_id))
redirect(".")
textxml = job.to_xml(clone=True).toprettyxml()
elif recipeset_id:
title = 'Clone Recipeset %s' % recipeset_id
try:
recipeset = RecipeSet.by_id(recipeset_id)
except InvalidRequestError:
flash(_(u"Invalid recipeset id %s" % recipeset_id))
redirect(".")
textxml = recipeset.to_xml(clone=True,
from_job=False).toprettyxml()
elif isinstance(filexml, cgi.FieldStorage):
# Clone from file
try:
textxml = filexml.value.decode('utf8')
except UnicodeDecodeError, e:
flash(_(u'Invalid job XML: %s') % e)
redirect('.')
def provision_scheduled_recipesets(*args):
"""
if All recipes in a recipeSet are in Scheduled state then move them to
Running.
"""
work_done = False
with session.begin():
recipesets = RecipeSet.by_recipe_status(TaskStatus.scheduled)
recipeset_ids = [rs_id for rs_id, in recipesets.values(RecipeSet.id)]
if recipeset_ids:
log.debug('Provisioning scheduled recipe sets [%s ... %s] (%d total)',
recipeset_ids[0], recipeset_ids[-1], len(recipeset_ids))
for rs_id in recipeset_ids:
session.begin()
try:
provision_scheduled_recipeset(rs_id)
session.commit()
except exceptions.Exception:
log.exception('Error in provision_scheduled_recipeset(%s)', rs_id)
session.rollback()
finally:
session.close()
work_done = True
return work_done
def provision_scheduled_recipesets(*args):
"""
if All recipes in a recipeSet are in Scheduled state then move them to
Running.
"""
work_done = False
with session.begin():
recipesets = RecipeSet.by_recipe_status(TaskStatus.scheduled)
recipeset_ids = [rs_id for rs_id, in recipesets.values(RecipeSet.id)]
if recipeset_ids:
log.debug('Provisioning scheduled recipe sets [%s ... %s] (%d total)',
recipeset_ids[0], recipeset_ids[-1], len(recipeset_ids))
for rs_id in recipeset_ids:
session.begin()
try:
provision_scheduled_recipeset(rs_id)
session.commit()
except exceptions.Exception:
log.exception('Error in provision_scheduled_recipeset(%s)', rs_id)
session.rollback()
finally:
session.close()
work_done = True
return work_done
def _handle_recipe_set(self,
xmlrecipeSet,
user,
ignore_missing_tasks=False):
"""
Handles the processing of recipesets into DB entries from their xml
"""
recipeSet = RecipeSet(ttasks=0)
recipeset_priority = xmlrecipeSet.get_xml_attr('priority', unicode,
None)
if recipeset_priority is not None:
try:
my_priority = TaskPriority.from_string(recipeset_priority)
except InvalidRequestError:
raise BX(
_('You have specified an invalid recipeSet priority:%s' %
recipeset_priority))
allowed_priorities = RecipeSet.allowed_priorities_initial(user)
if my_priority in allowed_priorities:
recipeSet.priority = my_priority
else:
recipeSet.priority = TaskPriority.default_priority()
else:
recipeSet.priority = TaskPriority.default_priority()
for xmlrecipe in xmlrecipeSet.iter_recipes():
recipe = self.handleRecipe(
xmlrecipe, user, ignore_missing_tasks=ignore_missing_tasks)
recipe.ttasks = len(recipe.tasks)
recipeSet.ttasks += recipe.ttasks
recipeSet.recipes.append(recipe)
# We want the guests to be part of the same recipeSet
for guest in recipe.guests:
recipeSet.recipes.append(guest)
guest.ttasks = len(guest.tasks)
recipeSet.ttasks += guest.ttasks
if not recipeSet.recipes:
raise BX(
_('No Recipes! You can not have a recipeSet with no recipes!'))
return recipeSet
def queue_processed_recipeset(recipeset_id):
recipeset = RecipeSet.by_id(recipeset_id)
bad_l_controllers = set()
# We only need to do this processing on multi-host recipes
if len(list(recipeset.machine_recipes)) == 1:
recipe = recipeset.machine_recipes.next()
recipe.queue()
log.info("recipe ID %s moved from Processed to Queued", recipe.id)
for guestrecipe in recipe.guests:
guestrecipe.queue()
return
# Find all the lab controllers that this recipeset may run.
rsl_controllers = set(LabController.query\
.join('systems',
'queued_recipes',
'recipeset')\
.filter(RecipeSet.id==recipeset.id).all())
# Any lab controllers that are not associated to all recipes in the
# recipe set must have those systems on that lab controller removed
# from any recipes. For multi-host all recipes must be schedulable
# on one lab controller
for recipe in recipeset.machine_recipes:
rl_controllers = set(LabController.query\
.join('systems',
'queued_recipes')\
.filter(Recipe.id==recipe.id).all())
bad_l_controllers = bad_l_controllers.union(
rl_controllers.difference(rsl_controllers))
for l_controller in rsl_controllers:
enough_systems = False
for recipe in recipeset.machine_recipes:
systems = recipe.dyn_systems.filter(
System.lab_controller == l_controller).all()
if len(systems) < len(recipeset.recipes):
break
else:
# There are enough choices We don't need to worry about dead
# locks
enough_systems = True
if not enough_systems:
log.debug(
"recipe: %s labController:%s entering not enough systems logic"
% (recipe.id, l_controller))
# Eliminate bad choices.
for recipe in recipeset.machine_recipes_orderby(l_controller)[:]:
for tmprecipe in recipeset.machine_recipes:
systemsa = set(
recipe.dyn_systems.filter(
System.lab_controller == l_controller).all())
systemsb = set(
tmprecipe.dyn_systems.filter(
System.lab_controller == l_controller).all())
if systemsa.difference(systemsb):
for rem_system in systemsa.intersection(systemsb):
if rem_system in recipe.systems:
log.debug(
"recipe: %s labController:%s Removing system %s"
% (recipe.id, l_controller, rem_system))
recipe.systems.remove(rem_system)
for recipe in recipeset.machine_recipes:
count = 0
systems = recipe.dyn_systems.filter(
System.lab_controller == l_controller).all()
for tmprecipe in recipeset.machine_recipes:
tmpsystems = tmprecipe.dyn_systems.filter(
System.lab_controller == l_controller).all()
if recipe != tmprecipe and \
systems == tmpsystems:
count += 1
if len(systems) <= count:
# Remove all systems from this lc on this rs.
log.debug(
"recipe: %s labController:%s %s <= %s Removing lab" %
(recipe.id, l_controller, len(systems), count))
bad_l_controllers = bad_l_controllers.union([l_controller])
# Remove systems that are on bad lab controllers
# This means one of the recipes can be fullfilled on a lab controller
# but not the rest of the recipes in the recipeSet.
# This could very well remove ALL systems from all recipes in this
# recipeSet. If that happens then the recipeSet cannot be scheduled
# and will be aborted by the abort process.
for recipe in recipeset.machine_recipes:
for l_controller in bad_l_controllers:
systems = (recipe.dyn_systems.filter(
System.lab_controller == l_controller).all())
log.debug("recipe: %s labController: %s Removing lab" %
(recipe.id, l_controller))
for system in systems:
if system in recipe.systems:
log.debug(
"recipe: %s labController: %s Removing system %s" %
(recipe.id, l_controller, system))
recipe.systems.remove(system)
# Are we left with any recipes having no candidate systems?
dead_recipes = [
recipe for recipe in recipeset.machine_recipes if not recipe.systems
]
if dead_recipes:
# Set status to Aborted
log.debug(
'Not enough systems logic for %s left %s with no candidate systems',
recipeset.t_id, ', '.join(recipe.t_id for recipe in dead_recipes))
log.info('%s moved from Processed to Aborted' % recipeset.t_id)
recipeset.abort(u'Recipe ID %s does not match any systems' %
', '.join(str(recipe.id) for recipe in dead_recipes))
else:
for recipe in recipeset.machine_recipes:
# Set status to Queued
log.info("recipe: %s moved from Processed to Queued" % recipe.id)
recipe.queue()
for guestrecipe in recipe.guests:
guestrecipe.queue()
def queue_processed_recipeset(recipeset_id):
recipeset = RecipeSet.by_id(recipeset_id)
bad_l_controllers = set()
# We only need to do this processing on multi-host recipes
if len(list(recipeset.machine_recipes)) == 1:
recipe = recipeset.machine_recipes.next()
recipe.queue()
log.info("recipe ID %s moved from Processed to Queued", recipe.id)
for guestrecipe in recipe.guests:
guestrecipe.queue()
return
# Find all the lab controllers that this recipeset may run.
rsl_controllers = set(LabController.query\
.join('systems',
'queued_recipes',
'recipeset')\
.filter(RecipeSet.id==recipeset.id).all())
# Any lab controllers that are not associated to all recipes in the
# recipe set must have those systems on that lab controller removed
# from any recipes. For multi-host all recipes must be schedulable
# on one lab controller
for recipe in recipeset.machine_recipes:
rl_controllers = set(LabController.query\
.join('systems',
'queued_recipes')\
.filter(Recipe.id==recipe.id).all())
bad_l_controllers = bad_l_controllers.union(rl_controllers.difference(rsl_controllers))
for l_controller in rsl_controllers:
enough_systems = False
for recipe in recipeset.machine_recipes:
systems = recipe.dyn_systems.filter(
System.lab_controller==l_controller
).all()
if len(systems) < len(recipeset.recipes):
break
else:
# There are enough choices We don't need to worry about dead
# locks
enough_systems = True
if not enough_systems:
log.debug("recipe: %s labController:%s entering not enough systems logic" %
(recipe.id, l_controller))
# Eliminate bad choices.
for recipe in recipeset.machine_recipes_orderby(l_controller)[:]:
for tmprecipe in recipeset.machine_recipes:
systemsa = set(recipe.dyn_systems.filter(
System.lab_controller==l_controller
).all())
systemsb = set(tmprecipe.dyn_systems.filter(
System.lab_controller==l_controller
).all())
if systemsa.difference(systemsb):
for rem_system in systemsa.intersection(systemsb):
if rem_system in recipe.systems:
log.debug("recipe: %s labController:%s Removing system %s" % (recipe.id, l_controller, rem_system))
recipe.systems.remove(rem_system)
for recipe in recipeset.machine_recipes:
count = 0
systems = recipe.dyn_systems.filter(
System.lab_controller==l_controller
).all()
for tmprecipe in recipeset.machine_recipes:
tmpsystems = tmprecipe.dyn_systems.filter(
System.lab_controller==l_controller
).all()
if recipe != tmprecipe and \
systems == tmpsystems:
count += 1
if len(systems) <= count:
# Remove all systems from this lc on this rs.
log.debug("recipe: %s labController:%s %s <= %s Removing lab" % (recipe.id, l_controller, len(systems), count))
bad_l_controllers = bad_l_controllers.union([l_controller])
# Remove systems that are on bad lab controllers
# This means one of the recipes can be fullfilled on a lab controller
# but not the rest of the recipes in the recipeSet.
# This could very well remove ALL systems from all recipes in this
# recipeSet. If that happens then the recipeSet cannot be scheduled
# and will be aborted by the abort process.
for recipe in recipeset.machine_recipes:
for l_controller in bad_l_controllers:
systems = (recipe.dyn_systems.filter(
System.lab_controller==l_controller
).all()
)
log.debug("recipe: %s labController: %s Removing lab" % (recipe.id, l_controller))
for system in systems:
if system in recipe.systems:
log.debug("recipe: %s labController: %s Removing system %s" % (recipe.id, l_controller, system))
recipe.systems.remove(system)
# Are we left with any recipes having no candidate systems?
dead_recipes = [recipe for recipe in recipeset.machine_recipes if not recipe.systems]
if dead_recipes:
# Set status to Aborted
log.debug('Not enough systems logic for %s left %s with no candidate systems',
recipeset.t_id, ', '.join(recipe.t_id for recipe in dead_recipes))
log.info('%s moved from Processed to Aborted' % recipeset.t_id)
recipeset.abort(u'Recipe ID %s does not match any systems'
% ', '.join(str(recipe.id) for recipe in dead_recipes))
else:
for recipe in recipeset.machine_recipes:
# Set status to Queued
log.info("recipe: %s moved from Processed to Queued" % recipe.id)
recipe.queue()
for guestrecipe in recipe.guests:
guestrecipe.queue()
def queue_processed_recipeset(recipeset_id):
recipeset = RecipeSet.by_id(recipeset_id)
# We only need to check "not enough systems" logic for multi-host recipe sets
if len(list(recipeset.machine_recipes)) > 1:
bad_l_controllers = set()
# Find all the lab controllers that this recipeset may run.
rsl_controllers = set(LabController.query\
.join('systems',
'queued_recipes',
'recipeset')\
.filter(RecipeSet.id==recipeset.id).all())
# Any lab controllers that are not associated to all recipes in the
# recipe set must have those systems on that lab controller removed
# from any recipes. For multi-host all recipes must be schedulable
# on one lab controller
for recipe in recipeset.machine_recipes:
rl_controllers = set(LabController.query\
.join('systems',
'queued_recipes')\
.filter(Recipe.id==recipe.id).all())
bad_l_controllers = bad_l_controllers.union(rl_controllers.difference(rsl_controllers))
for l_controller in rsl_controllers:
enough_systems = False
for recipe in recipeset.machine_recipes:
systems = recipe.dyn_systems.filter(
System.lab_controller==l_controller
).all()
if len(systems) < len(recipeset.recipes):
break
else:
# There are enough choices We don't need to worry about dead
# locks
enough_systems = True
if not enough_systems:
log.debug("recipe: %s labController:%s entering not enough systems logic" %
(recipe.id, l_controller))
# Eliminate bad choices.
for recipe in recipeset.machine_recipes_orderby(l_controller)[:]:
for tmprecipe in recipeset.machine_recipes:
systemsa = set(recipe.dyn_systems.filter(
System.lab_controller==l_controller
).all())
systemsb = set(tmprecipe.dyn_systems.filter(
System.lab_controller==l_controller
).all())
if systemsa.difference(systemsb):
for rem_system in systemsa.intersection(systemsb):
if rem_system in recipe.systems:
log.debug("recipe: %s labController:%s Removing system %s" % (recipe.id, l_controller, rem_system))
recipe.systems.remove(rem_system)
for recipe in recipeset.machine_recipes:
count = 0
systems = recipe.dyn_systems.filter(
System.lab_controller==l_controller
).all()
for tmprecipe in recipeset.machine_recipes:
tmpsystems = tmprecipe.dyn_systems.filter(
System.lab_controller==l_controller
).all()
if recipe != tmprecipe and \
systems == tmpsystems:
count += 1
if len(systems) <= count:
# Remove all systems from this lc on this rs.
log.debug("recipe: %s labController:%s %s <= %s Removing lab" % (recipe.id, l_controller, len(systems), count))
bad_l_controllers = bad_l_controllers.union([l_controller])
# Remove systems that are on bad lab controllers
# This means one of the recipes can be fullfilled on a lab controller
# but not the rest of the recipes in the recipeSet.
# This could very well remove ALL systems from all recipes in this
# recipeSet. If that happens then the recipeSet cannot be scheduled
# and will be aborted by the abort process.
for recipe in recipeset.machine_recipes:
for l_controller in bad_l_controllers:
systems = (recipe.dyn_systems.filter(
System.lab_controller==l_controller
).all()
)
log.debug("recipe: %s labController: %s Removing lab" % (recipe.id, l_controller))
for system in systems:
if system in recipe.systems:
log.debug("recipe: %s labController: %s Removing system %s" % (recipe.id, l_controller, system))
recipe.systems.remove(system)
# Make the removal of systems visible to subsequent queries which use .dyn_systems
session.flush()
# Are we left with any recipes having no candidate systems?
dead_recipes = [recipe for recipe in recipeset.machine_recipes if not recipe.systems]
if dead_recipes:
# Set status to Aborted
log.debug('Not enough systems logic for %s left %s with no candidate systems',
recipeset.t_id, ', '.join(recipe.t_id for recipe in dead_recipes))
log.info('%s moved from Processed to Aborted' % recipeset.t_id)
recipeset.abort(u'Recipe ID %s does not match any systems'
% ', '.join(str(recipe.id) for recipe in dead_recipes))
return
# Can we schedule any recipes immediately?
for recipe in recipeset.machine_recipes:
systems = recipe.matching_systems()
if not systems: # may be None if we know there are no possible LCs
continue
system_count = systems.count()
if not system_count:
continue
# Order systems by owner, then Group, finally shared for everyone.
# FIXME Make this configurable, so that a user can specify their scheduling
# Implemented order, still need to do pool
# preference from the job.
# <recipe>
# <autopick order='sequence|random'>
# <pool>owner</pool>
# <pool>groups</pool>
# <pool>public</pool>
# </autopick>
# </recipe>
if True: #FIXME if pools are defined add them here in the order requested.
systems = System.scheduler_ordering(recipeset.job.owner, query=systems)
if recipe.autopick_random:
system = systems[random.randrange(0, system_count)]
else:
system = systems.first()
schedule_recipe_on_system(recipe, system)
for recipe in recipeset.machine_recipes:
if recipe.status == TaskStatus.processed:
# Leave it Queued until a system becomes free
log.info("recipe: %s moved from Processed to Queued" % recipe.id)
recipe.queue()
for guestrecipe in recipe.guests:
guestrecipe.queue()
def queue_processed_recipeset(recipeset_id):
recipeset = RecipeSet.by_id(recipeset_id)
# We only need to check "not enough systems" logic for multi-host recipe sets
if len(list(recipeset.machine_recipes)) > 1:
bad_l_controllers = set()
# Find all the lab controllers that this recipeset may run.
rsl_controllers = set(LabController.query\
.join('systems',
'queued_recipes',
'recipeset')\
.filter(RecipeSet.id==recipeset.id).all())
# Any lab controllers that are not associated to all recipes in the
# recipe set must have those systems on that lab controller removed
# from any recipes. For multi-host all recipes must be schedulable
# on one lab controller
for recipe in recipeset.machine_recipes:
rl_controllers = set(LabController.query\
.join('systems',
'queued_recipes')\
.filter(Recipe.id==recipe.id).all())
bad_l_controllers = bad_l_controllers.union(rl_controllers.difference(rsl_controllers))
for l_controller in rsl_controllers:
enough_systems = False
for recipe in recipeset.machine_recipes:
systems = recipe.dyn_systems.filter(
System.lab_controller==l_controller
).all()
if len(systems) < len(recipeset.recipes):
break
else:
# There are enough choices We don't need to worry about dead
# locks
enough_systems = True
if not enough_systems:
log.debug("recipe: %s labController:%s entering not enough systems logic" %
(recipe.id, l_controller))
# Eliminate bad choices.
for recipe in recipeset.machine_recipes_orderby(l_controller)[:]:
for tmprecipe in recipeset.machine_recipes:
systemsa = set(recipe.dyn_systems.filter(
System.lab_controller==l_controller
).all())
systemsb = set(tmprecipe.dyn_systems.filter(
System.lab_controller==l_controller
).all())
if systemsa.difference(systemsb):
for rem_system in systemsa.intersection(systemsb):
if rem_system in recipe.systems:
log.debug("recipe: %s labController:%s Removing system %s" % (recipe.id, l_controller, rem_system))
recipe.systems.remove(rem_system)
for recipe in recipeset.machine_recipes:
count = 0
systems = recipe.dyn_systems.filter(
System.lab_controller==l_controller
).all()
for tmprecipe in recipeset.machine_recipes:
tmpsystems = tmprecipe.dyn_systems.filter(
System.lab_controller==l_controller
).all()
if recipe != tmprecipe and \
systems == tmpsystems:
count += 1
if len(systems) <= count:
# Remove all systems from this lc on this rs.
log.debug("recipe: %s labController:%s %s <= %s Removing lab" % (recipe.id, l_controller, len(systems), count))
bad_l_controllers = bad_l_controllers.union([l_controller])
# Remove systems that are on bad lab controllers
# This means one of the recipes can be fullfilled on a lab controller
# but not the rest of the recipes in the recipeSet.
# This could very well remove ALL systems from all recipes in this
# recipeSet. If that happens then the recipeSet cannot be scheduled
# and will be aborted by the abort process.
for recipe in recipeset.machine_recipes:
for l_controller in bad_l_controllers:
systems = (recipe.dyn_systems.filter(
System.lab_controller==l_controller
).all()
)
log.debug("recipe: %s labController: %s Removing lab" % (recipe.id, l_controller))
for system in systems:
if system in recipe.systems:
log.debug("recipe: %s labController: %s Removing system %s" % (recipe.id, l_controller, system))
recipe.systems.remove(system)
# Make the removal of systems visible to subsequent queries which use .dyn_systems
session.flush()
# Are we left with any recipes having no candidate systems?
dead_recipes = [recipe for recipe in recipeset.machine_recipes if not recipe.systems]
if dead_recipes:
# Set status to Aborted
log.debug('Not enough systems logic for %s left %s with no candidate systems',
recipeset.t_id, ', '.join(recipe.t_id for recipe in dead_recipes))
log.info('%s moved from Processed to Aborted' % recipeset.t_id)
recipeset.abort(u'Recipe ID %s does not match any systems'
% ', '.join(str(recipe.id) for recipe in dead_recipes))
return
# Can we schedule any recipes immediately?
for recipe in recipeset.machine_recipes:
systems = recipe.matching_systems()
if not systems: # may be None if we know there are no possible LCs
continue
system_count = systems.count()
if not system_count:
continue
# Order systems by owner, then Group, finally shared for everyone.
# FIXME Make this configurable, so that a user can specify their scheduling
# Implemented order, still need to do pool
# preference from the job.
# <recipe>
# <autopick order='sequence|random'>
# <pool>owner</pool>
# <pool>groups</pool>
# <pool>public</pool>
# </autopick>
# </recipe>
if True: #FIXME if pools are defined add them here in the order requested.
systems = System.scheduler_ordering(recipeset.job.owner, query=systems)
if recipe.autopick_random:
system = systems[random.randrange(0, system_count)]
else:
system = systems.first()
schedule_recipe_on_system(recipe, system)
for recipe in recipeset.machine_recipes:
if recipe.status == TaskStatus.processed:
# Leave it Queued until a system becomes free
log.info("recipe: %s moved from Processed to Queued" % recipe.id)
recipe.queue()
for guestrecipe in recipe.guests:
guestrecipe.queue()
def _get_rs_by_id(id):
try:
return RecipeSet.by_id(id)
except NoResultFound:
raise NotFound404('Recipe set %s not found' % id)
