def schedule_queued_recipe(recipe_id, guest_recipe_id=None):
guest_recipe = aliased(Recipe)
guest_distros_map = aliased(LabControllerDistroTree)
guest_labcontroller = aliased(LabController)
# This query will return all the systems that a recipe is
# able to run on. A system is deemed eligible if:
# * If the recipe's distro tree is available to the system's lab controller
# * The system is available (see the filter criteria).
# * If it's a host recipe, then the system needs to be on a lab controller
# that can access the distro tree of both the host recipe,
# and the guest recipe.
systems = System.query.join(System.queued_recipes) \
.outerjoin(System.cpu) \
.join(Recipe.recipeset, RecipeSet.job) \
.join(System.lab_controller, LabController._distro_trees)\
.join((DistroTree,
and_(LabControllerDistroTree.distro_tree_id ==
DistroTree.id, Recipe.distro_tree_id == DistroTree.id)))\
.outerjoin((machine_guest_map,
Recipe.id == machine_guest_map.c.machine_recipe_id))\
.outerjoin((guest_recipe,
machine_guest_map.c.guest_recipe_id == guest_recipe.id ))\
.outerjoin((guest_distros_map,
guest_recipe.distro_tree_id == guest_distros_map.distro_tree_id))\
.outerjoin((guest_labcontroller,
guest_distros_map.lab_controller_id == guest_labcontroller.id))\
.filter(Recipe.id == recipe_id) \
.filter(or_(guest_recipe.id == guest_recipe_id,
guest_recipe.id == None))\
.filter(and_(System.user == None,
or_(guest_distros_map.id == None,
and_(guest_distros_map.id != None,
guest_labcontroller.id == LabController.id,
),
),
LabController.disabled == False,
or_(System.loan_id == None,
System.loan_id == Job.owner_id,
),
), # and
)
# We reapply this filter here in case a peer recipe has locked the recipe
# set in to a particular lab controller earlier in this scheduling pass
recipe = MachineRecipe.by_id(recipe_id)
if recipe.recipeset.lab_controller:
systems = systems.filter(
System.lab_controller==recipe.recipeset.lab_controller)
# Something earlier in this pass meant we can't schedule this recipe
# right now after all. We'll try again next pass.
if not systems.count():
return
# Order systems by owner, then Group, finally shared for everyone.
# FIXME Make this configurable, so that a user can specify their scheduling
# Implemented order, still need to do pool
# preference from the job.
# <recipe>
# <autopick order='sequence|random'>
# <pool>owner</pool>
# <pool>groups</pool>
# <pool>public</pool>
# </autopick>
# </recipe>
user = recipe.recipeset.job.owner
if True: #FIXME if pools are defined add them here in the order requested.
systems = System.scheduler_ordering(user, query=systems)
if recipe.autopick_random:
system = systems[random.randrange(0,systems.count())]
else:
system = systems.first()
log.debug("System : %s is available for Recipe %s" % (system, recipe.id))
# Check to see if user still has proper permissions to use the system.
# Remember the mapping of available systems could have happend hours or even
# days ago and groups or loans could have been put in place since.
if not recipe.candidate_systems().filter(System.id == system.id).first():
log.debug("System : %s recipe: %s no longer has access. removing" % (system,
recipe.id))
recipe.systems.remove(system)
return
recipe.resource = SystemResource(system=system)
# Reserving the system may fail here if someone stole it out from
# underneath us, but that is fine...
recipe.resource.allocate()
recipe.schedule()
recipe.createRepo()
recipe.recipeset.lab_controller = system.lab_controller
recipe.systems = []
# Create the watchdog without an Expire time.
log.debug("Created watchdog for recipe id: %s and system: %s" % (recipe.id, system))
recipe.watchdog = Watchdog()
log.info("recipe ID %s moved from Queued to Scheduled" % recipe.id)
for guestrecipe in recipe.guests:
guestrecipe.resource = GuestResource()
guestrecipe.resource.allocate()
guestrecipe.schedule()
guestrecipe.createRepo()
guestrecipe.watchdog = Watchdog()
log.info('recipe ID %s guest %s moved from Queued to Scheduled',
recipe.id, guestrecipe.id)
def schedule_queued_recipe(recipe_id, guest_recipe_id=None):
log.debug('Selecting a system for recipe %s', recipe_id)
guest_recipe = aliased(Recipe)
guest_distros_map = aliased(LabControllerDistroTree)
guest_labcontroller = aliased(LabController)
# This query will return all the systems that a recipe is
# able to run on. A system is deemed eligible if:
# * If the recipe's distro tree is available to the system's lab controller
# * The system is available (see the filter criteria).
# * If it's a host recipe, then the system needs to be on a lab controller
# that can access the distro tree of both the host recipe,
# and the guest recipe.
systems = System.query.join(System.queued_recipes) \
.outerjoin(System.cpu) \
.join(Recipe.recipeset, RecipeSet.job) \
.join(System.lab_controller, LabController._distro_trees)\
.join((DistroTree,
and_(LabControllerDistroTree.distro_tree_id ==
DistroTree.id, Recipe.distro_tree_id == DistroTree.id)))\
.outerjoin((machine_guest_map,
Recipe.id == machine_guest_map.c.machine_recipe_id))\
.outerjoin((guest_recipe,
machine_guest_map.c.guest_recipe_id == guest_recipe.id ))\
.outerjoin((guest_distros_map,
guest_recipe.distro_tree_id == guest_distros_map.distro_tree_id))\
.outerjoin((guest_labcontroller,
guest_distros_map.lab_controller_id == guest_labcontroller.id))\
.filter(Recipe.id == recipe_id) \
.filter(or_(guest_recipe.id == guest_recipe_id,
guest_recipe.id == None))\
.filter(and_(System.user == None,
or_(guest_distros_map.id == None,
and_(guest_distros_map.id != None,
guest_labcontroller.id == LabController.id,
),
),
LabController.disabled == False,
or_(System.loan_id == None,
System.loan_id == Job.owner_id,
),
), # and
)
# We reapply this filter here in case a peer recipe has locked the recipe
# set in to a particular lab controller earlier in this scheduling pass
recipe = MachineRecipe.by_id(recipe_id)
if recipe.recipeset.lab_controller:
systems = systems.filter(
System.lab_controller == recipe.recipeset.lab_controller)
# Something earlier in this pass meant we can't schedule this recipe
# right now after all. We'll try again next pass.
if not systems.count():
return
# Order systems by owner, then Group, finally shared for everyone.
# FIXME Make this configurable, so that a user can specify their scheduling
# Implemented order, still need to do pool
# preference from the job.
# <recipe>
# <autopick order='sequence|random'>
# <pool>owner</pool>
# <pool>groups</pool>
# <pool>public</pool>
# </autopick>
# </recipe>
user = recipe.recipeset.job.owner
if True: #FIXME if pools are defined add them here in the order requested.
systems = System.scheduler_ordering(user, query=systems)
if recipe.autopick_random:
system = systems[random.randrange(0, systems.count())]
else:
system = systems.first()
log.debug("System : %s is available for Recipe %s" % (system, recipe.id))
# Check to see if user still has proper permissions to use the system.
# Remember the mapping of available systems could have happend hours or even
# days ago and groups or loans could have been put in place since.
if not recipe.candidate_systems().filter(System.id == system.id).first():
log.debug("System : %s recipe: %s no longer has access. removing" %
(system, recipe.id))
recipe.systems.remove(system)
return
recipe.resource = SystemResource(system=system)
# Reserving the system may fail here if someone stole it out from
# underneath us, but that is fine...
recipe.resource.allocate()
recipe.schedule()
recipe.createRepo()
recipe.recipeset.lab_controller = system.lab_controller
recipe.systems = []
# Create the watchdog without an Expire time.
log.debug("Created watchdog for recipe id: %s and system: %s" %
(recipe.id, system))
recipe.watchdog = Watchdog()
log.info("recipe ID %s moved from Queued to Scheduled" % recipe.id)
for guestrecipe in recipe.guests:
guestrecipe.resource = GuestResource()
guestrecipe.resource.allocate()
guestrecipe.schedule()
guestrecipe.createRepo()
guestrecipe.watchdog = Watchdog()
log.info('recipe ID %s guest %s moved from Queued to Scheduled',
recipe.id, guestrecipe.id)
def queue_processed_recipeset(recipeset_id):
recipeset = RecipeSet.by_id(recipeset_id)
# We only need to check "not enough systems" logic for multi-host recipe sets
if len(list(recipeset.machine_recipes)) > 1:
bad_l_controllers = set()
# Find all the lab controllers that this recipeset may run.
rsl_controllers = set(LabController.query\
.join('systems',
'queued_recipes',
'recipeset')\
.filter(RecipeSet.id==recipeset.id).all())
# Any lab controllers that are not associated to all recipes in the
# recipe set must have those systems on that lab controller removed
# from any recipes. For multi-host all recipes must be schedulable
# on one lab controller
for recipe in recipeset.machine_recipes:
rl_controllers = set(LabController.query\
.join('systems',
'queued_recipes')\
.filter(Recipe.id==recipe.id).all())
bad_l_controllers = bad_l_controllers.union(rl_controllers.difference(rsl_controllers))
for l_controller in rsl_controllers:
enough_systems = False
for recipe in recipeset.machine_recipes:
systems = recipe.dyn_systems.filter(
System.lab_controller==l_controller
).all()
if len(systems) < len(recipeset.recipes):
break
else:
# There are enough choices We don't need to worry about dead
# locks
enough_systems = True
if not enough_systems:
log.debug("recipe: %s labController:%s entering not enough systems logic" %
(recipe.id, l_controller))
# Eliminate bad choices.
for recipe in recipeset.machine_recipes_orderby(l_controller)[:]:
for tmprecipe in recipeset.machine_recipes:
systemsa = set(recipe.dyn_systems.filter(
System.lab_controller==l_controller
).all())
systemsb = set(tmprecipe.dyn_systems.filter(
System.lab_controller==l_controller
).all())
if systemsa.difference(systemsb):
for rem_system in systemsa.intersection(systemsb):
if rem_system in recipe.systems:
log.debug("recipe: %s labController:%s Removing system %s" % (recipe.id, l_controller, rem_system))
recipe.systems.remove(rem_system)
for recipe in recipeset.machine_recipes:
count = 0
systems = recipe.dyn_systems.filter(
System.lab_controller==l_controller
).all()
for tmprecipe in recipeset.machine_recipes:
tmpsystems = tmprecipe.dyn_systems.filter(
System.lab_controller==l_controller
).all()
if recipe != tmprecipe and \
systems == tmpsystems:
count += 1
if len(systems) <= count:
# Remove all systems from this lc on this rs.
log.debug("recipe: %s labController:%s %s <= %s Removing lab" % (recipe.id, l_controller, len(systems), count))
bad_l_controllers = bad_l_controllers.union([l_controller])
# Remove systems that are on bad lab controllers
# This means one of the recipes can be fullfilled on a lab controller
# but not the rest of the recipes in the recipeSet.
# This could very well remove ALL systems from all recipes in this
# recipeSet. If that happens then the recipeSet cannot be scheduled
# and will be aborted by the abort process.
for recipe in recipeset.machine_recipes:
for l_controller in bad_l_controllers:
systems = (recipe.dyn_systems.filter(
System.lab_controller==l_controller
).all()
)
log.debug("recipe: %s labController: %s Removing lab" % (recipe.id, l_controller))
for system in systems:
if system in recipe.systems:
log.debug("recipe: %s labController: %s Removing system %s" % (recipe.id, l_controller, system))
recipe.systems.remove(system)
# Make the removal of systems visible to subsequent queries which use .dyn_systems
session.flush()
# Are we left with any recipes having no candidate systems?
dead_recipes = [recipe for recipe in recipeset.machine_recipes if not recipe.systems]
if dead_recipes:
# Set status to Aborted
log.debug('Not enough systems logic for %s left %s with no candidate systems',
recipeset.t_id, ', '.join(recipe.t_id for recipe in dead_recipes))
log.info('%s moved from Processed to Aborted' % recipeset.t_id)
recipeset.abort(u'Recipe ID %s does not match any systems'
% ', '.join(str(recipe.id) for recipe in dead_recipes))
return
# Can we schedule any recipes immediately?
for recipe in recipeset.machine_recipes:
systems = recipe.matching_systems()
if not systems: # may be None if we know there are no possible LCs
continue
system_count = systems.count()
if not system_count:
continue
# Order systems by owner, then Group, finally shared for everyone.
# FIXME Make this configurable, so that a user can specify their scheduling
# Implemented order, still need to do pool
# preference from the job.
# <recipe>
# <autopick order='sequence|random'>
# <pool>owner</pool>
# <pool>groups</pool>
# <pool>public</pool>
# </autopick>
# </recipe>
if True: #FIXME if pools are defined add them here in the order requested.
systems = System.scheduler_ordering(recipeset.job.owner, query=systems)
if recipe.autopick_random:
system = systems[random.randrange(0, system_count)]
else:
system = systems.first()
schedule_recipe_on_system(recipe, system)
for recipe in recipeset.machine_recipes:
if recipe.status == TaskStatus.processed:
# Leave it Queued until a system becomes free
log.info("recipe: %s moved from Processed to Queued" % recipe.id)
recipe.queue()
for guestrecipe in recipe.guests:
guestrecipe.queue()
def queue_processed_recipeset(recipeset_id):
recipeset = RecipeSet.by_id(recipeset_id)
# We only need to check "not enough systems" logic for multi-host recipe sets
if len(list(recipeset.machine_recipes)) > 1:
bad_l_controllers = set()
# Find all the lab controllers that this recipeset may run.
rsl_controllers = set(LabController.query\
.join('systems',
'queued_recipes',
'recipeset')\
.filter(RecipeSet.id==recipeset.id).all())
# Any lab controllers that are not associated to all recipes in the
# recipe set must have those systems on that lab controller removed
# from any recipes. For multi-host all recipes must be schedulable
# on one lab controller
for recipe in recipeset.machine_recipes:
rl_controllers = set(LabController.query\
.join('systems',
'queued_recipes')\
.filter(Recipe.id==recipe.id).all())
bad_l_controllers = bad_l_controllers.union(rl_controllers.difference(rsl_controllers))
for l_controller in rsl_controllers:
enough_systems = False
for recipe in recipeset.machine_recipes:
systems = recipe.dyn_systems.filter(
System.lab_controller==l_controller
).all()
if len(systems) < len(recipeset.recipes):
break
else:
# There are enough choices We don't need to worry about dead
# locks
enough_systems = True
if not enough_systems:
log.debug("recipe: %s labController:%s entering not enough systems logic" %
(recipe.id, l_controller))
# Eliminate bad choices.
for recipe in recipeset.machine_recipes_orderby(l_controller)[:]:
for tmprecipe in recipeset.machine_recipes:
systemsa = set(recipe.dyn_systems.filter(
System.lab_controller==l_controller
).all())
systemsb = set(tmprecipe.dyn_systems.filter(
System.lab_controller==l_controller
).all())
if systemsa.difference(systemsb):
for rem_system in systemsa.intersection(systemsb):
if rem_system in recipe.systems:
log.debug("recipe: %s labController:%s Removing system %s" % (recipe.id, l_controller, rem_system))
recipe.systems.remove(rem_system)
for recipe in recipeset.machine_recipes:
count = 0
systems = recipe.dyn_systems.filter(
System.lab_controller==l_controller
).all()
for tmprecipe in recipeset.machine_recipes:
tmpsystems = tmprecipe.dyn_systems.filter(
System.lab_controller==l_controller
).all()
if recipe != tmprecipe and \
systems == tmpsystems:
count += 1
if len(systems) <= count:
# Remove all systems from this lc on this rs.
log.debug("recipe: %s labController:%s %s <= %s Removing lab" % (recipe.id, l_controller, len(systems), count))
bad_l_controllers = bad_l_controllers.union([l_controller])
# Remove systems that are on bad lab controllers
# This means one of the recipes can be fullfilled on a lab controller
# but not the rest of the recipes in the recipeSet.
# This could very well remove ALL systems from all recipes in this
# recipeSet. If that happens then the recipeSet cannot be scheduled
# and will be aborted by the abort process.
for recipe in recipeset.machine_recipes:
for l_controller in bad_l_controllers:
systems = (recipe.dyn_systems.filter(
System.lab_controller==l_controller
).all()
)
log.debug("recipe: %s labController: %s Removing lab" % (recipe.id, l_controller))
for system in systems:
if system in recipe.systems:
log.debug("recipe: %s labController: %s Removing system %s" % (recipe.id, l_controller, system))
recipe.systems.remove(system)
# Make the removal of systems visible to subsequent queries which use .dyn_systems
session.flush()
# Are we left with any recipes having no candidate systems?
dead_recipes = [recipe for recipe in recipeset.machine_recipes if not recipe.systems]
if dead_recipes:
# Set status to Aborted
log.debug('Not enough systems logic for %s left %s with no candidate systems',
recipeset.t_id, ', '.join(recipe.t_id for recipe in dead_recipes))
log.info('%s moved from Processed to Aborted' % recipeset.t_id)
recipeset.abort(u'Recipe ID %s does not match any systems'
% ', '.join(str(recipe.id) for recipe in dead_recipes))
return
# Can we schedule any recipes immediately?
for recipe in recipeset.machine_recipes:
systems = recipe.matching_systems()
if not systems: # may be None if we know there are no possible LCs
continue
system_count = systems.count()
if not system_count:
continue
# Order systems by owner, then Group, finally shared for everyone.
# FIXME Make this configurable, so that a user can specify their scheduling
# Implemented order, still need to do pool
# preference from the job.
# <recipe>
# <autopick order='sequence|random'>
# <pool>owner</pool>
# <pool>groups</pool>
# <pool>public</pool>
# </autopick>
# </recipe>
if True: #FIXME if pools are defined add them here in the order requested.
systems = System.scheduler_ordering(recipeset.job.owner, query=systems)
if recipe.autopick_random:
system = systems[random.randrange(0, system_count)]
else:
system = systems.first()
schedule_recipe_on_system(recipe, system)
for recipe in recipeset.machine_recipes:
if recipe.status == TaskStatus.processed:
# Leave it Queued until a system becomes free
log.info("recipe: %s moved from Processed to Queued" % recipe.id)
recipe.queue()
for guestrecipe in recipe.guests:
guestrecipe.queue()
