def mark_recipe_waiting(recipe, start_time=None, system=None,
lab_controller=None, virt=False, instance_id=None, **kwargs):
if start_time is None:
start_time = datetime.datetime.utcnow()
recipe.process()
recipe.queue()
recipe.schedule()
if not recipe.resource:
if isinstance(recipe, MachineRecipe):
if virt:
if not lab_controller:
lab_controller = create_labcontroller(fqdn=u'dummylab.example.invalid')
if not instance_id:
instance_id = uuid.uuid4()
recipe.resource = VirtResource(instance_id, lab_controller)
recipe.recipeset.lab_controller = lab_controller
else:
if not system:
if not lab_controller:
lab_controller = create_labcontroller(fqdn=u'dummylab.example.invalid')
system = create_system(arch=recipe.arch,
lab_controller=lab_controller)
recipe.resource = SystemResource(system=system)
recipe.resource.allocate()
recipe.resource.reservation.start_time = start_time
recipe.recipeset.lab_controller = system.lab_controller
elif isinstance(recipe, GuestRecipe):
recipe.resource = GuestResource()
recipe.resource.allocate()
recipe.start_time = start_time
recipe.watchdog = Watchdog()
recipe.waiting()
recipe.resource.rebooted = start_time
recipe.recipeset.job.update_status()
log.debug('Marked %s as waiting with system %s', recipe.t_id, recipe.resource.fqdn)
def provision_virt_recipe(recipe_id):
log.debug('Attempting to provision dynamic virt guest for recipe %s', recipe_id)
session.begin()
try:
recipe = Recipe.by_id(recipe_id)
manager = dynamic_virt.VirtManager(recipe.recipeset.job.owner)
available_flavors = manager.available_flavors()
# We want them in order of smallest to largest, so that we can pick the
# smallest flavor that satisfies the recipe's requirements. Sorting by RAM
# is a decent approximation.
possible_flavors = XmlHost.from_string(recipe.host_requires)\
.filter_openstack_flavors(available_flavors, manager.lab_controller)
if not possible_flavors:
log.debug('No OpenStack flavors matched recipe %s, marking precluded',
recipe.id)
recipe.virt_status = RecipeVirtStatus.precluded
return
possible_flavors = sorted(possible_flavors, key=lambda flavor: flavor.ram)
flavor = possible_flavors[0]
vm_name = '%srecipe-%s' % (
ConfigItem.by_name(u'guest_name_prefix').current_value(u'beaker-'),
recipe.id)
# FIXME can we control use of virtio?
#virtio_possible = True
#if self.recipe.distro_tree.distro.osversion.osmajor.osmajor == "RedHatEnterpriseLinux3":
# virtio_possible = False
vm = manager.create_vm(vm_name, flavor)
vm.instance_created = datetime.utcnow()
try:
recipe.createRepo()
recipe.systems = []
recipe.watchdog = Watchdog()
recipe.resource = vm
recipe.recipeset.lab_controller = manager.lab_controller
recipe.virt_status = RecipeVirtStatus.succeeded
recipe.schedule()
log.info("recipe ID %s moved from Queued to Scheduled by provision_virt_recipe" % recipe.id)
recipe.waiting()
recipe.provision()
log.info("recipe ID %s moved from Scheduled to Waiting by provision_virt_recipe" % recipe.id)
except:
exc_type, exc_value, exc_tb = sys.exc_info()
try:
manager.destroy_vm(vm)
except Exception:
log.exception('Failed to clean up vm %s '
'during provision_virt_recipe, leaked!', vm.instance_id)
# suppress this exception so the original one is not masked
raise exc_type, exc_value, exc_tb
session.commit()
except Exception, e:
log.exception('Error in provision_virt_recipe(%s)', recipe_id)
session.rollback()
# As an added precaution, let's try and avoid this recipe in future
with session.begin():
recipe = Recipe.by_id(recipe_id)
recipe.virt_status = RecipeVirtStatus.failed
def schedule_recipe_on_system(recipe, system):
log.debug('Assigning recipe %s to system %s', recipe.id, system.fqdn)
recipe.resource = SystemResource(system=system)
# Reserving the system may fail here if someone stole it out from
# underneath us, but that is fine...
recipe.resource.allocate()
recipe.schedule()
recipe.createRepo()
recipe.recipeset.lab_controller = system.lab_controller
recipe.systems = []
# Create the watchdog without an Expire time.
log.debug("Created watchdog for recipe id: %s and system: %s" % (recipe.id, system))
recipe.watchdog = Watchdog()
log.info("recipe ID %s moved from Queued to Scheduled" % recipe.id)
for guestrecipe in recipe.guests:
guestrecipe.resource = GuestResource()
guestrecipe.resource.allocate()
guestrecipe.schedule()
guestrecipe.createRepo()
guestrecipe.watchdog = Watchdog()
log.info('recipe ID %s guest %s moved from Queued to Scheduled',
recipe.id, guestrecipe.id)
def mark_recipe_scheduled(recipe,
start_time=None,
system=None,
fqdn=None,
mac_address=None,
lab_controller=None,
virt=False,
instance_id=None,
**kwargs):
recipe.process()
recipe.queue()
recipe.schedule()
if not recipe.resource:
if isinstance(recipe, MachineRecipe):
if virt:
if not lab_controller:
lab_controller = create_labcontroller(
fqdn=u'dummylab.example.invalid')
if not instance_id:
instance_id = uuid.uuid4()
recipe.resource = VirtResource(instance_id, lab_controller)
recipe.recipeset.lab_controller = lab_controller
else:
if not system:
if not lab_controller:
lab_controller = create_labcontroller(
fqdn=u'dummylab.example.invalid')
system = create_system(arch=recipe.arch.arch,
fqdn=fqdn,
lab_controller=lab_controller)
recipe.resource = SystemResource(system=system)
recipe.resource.allocate()
recipe.resource.reservation.start_time = start_time or datetime.datetime.utcnow(
)
recipe.recipeset.lab_controller = system.lab_controller
elif isinstance(recipe, GuestRecipe):
recipe.resource = GuestResource()
recipe.resource.allocate()
if mac_address is not None:
recipe.resource.mac_address = netaddr.EUI(
mac_address, dialect=mac_unix_padded_dialect)
if not recipe.distro_tree.url_in_lab(recipe.recipeset.lab_controller):
add_distro_tree_to_lab(recipe.distro_tree,
recipe.recipeset.lab_controller)
recipe.watchdog = Watchdog()
log.debug('Marked %s as scheduled with system %s', recipe.t_id,
recipe.resource.fqdn)
def test_recovers_leaked_system(self):
# recipe is cancelled but system has still been assigned
recipe = data_setup.create_recipe()
data_setup.create_job_for_recipes([recipe])
recipe._abort_cancel(TaskStatus.cancelled)
recipe.recipeset.job.update_status()
system = data_setup.create_system()
recipe.resource = SystemResource(system=system)
recipe.resource.allocate()
recipe.watchdog = Watchdog()
session.flush()
self.assert_(system.open_reservation is not None)
self.assert_(system.open_reservation is recipe.resource.reservation)
self.assert_(system.user is not None)
self.assert_(recipe.watchdog is not None)
recipe.recipeset.job.update_status()
session.flush()
session.expire_all()
self.assertEquals(system.open_reservation, None)
self.assertEquals(system.user, None)
self.assertEquals(recipe.watchdog, None)
def schedule_queued_recipe(recipe_id, guest_recipe_id=None):
log.debug('Selecting a system for recipe %s', recipe_id)
guest_recipe = aliased(Recipe)
guest_distros_map = aliased(LabControllerDistroTree)
guest_labcontroller = aliased(LabController)
# This query will return all the systems that a recipe is
# able to run on. A system is deemed eligible if:
# * If the recipe's distro tree is available to the system's lab controller
# * The system is available (see the filter criteria).
# * If it's a host recipe, then the system needs to be on a lab controller
# that can access the distro tree of both the host recipe,
# and the guest recipe.
systems = System.query.join(System.queued_recipes) \
.outerjoin(System.cpu) \
.join(Recipe.recipeset, RecipeSet.job) \
.join(System.lab_controller, LabController._distro_trees)\
.join((DistroTree,
and_(LabControllerDistroTree.distro_tree_id ==
DistroTree.id, Recipe.distro_tree_id == DistroTree.id)))\
.outerjoin((machine_guest_map,
Recipe.id == machine_guest_map.c.machine_recipe_id))\
.outerjoin((guest_recipe,
machine_guest_map.c.guest_recipe_id == guest_recipe.id ))\
.outerjoin((guest_distros_map,
guest_recipe.distro_tree_id == guest_distros_map.distro_tree_id))\
.outerjoin((guest_labcontroller,
guest_distros_map.lab_controller_id == guest_labcontroller.id))\
.filter(Recipe.id == recipe_id) \
.filter(or_(guest_recipe.id == guest_recipe_id,
guest_recipe.id == None))\
.filter(and_(System.user == None,
or_(guest_distros_map.id == None,
and_(guest_distros_map.id != None,
guest_labcontroller.id == LabController.id,
),
),
LabController.disabled == False,
or_(System.loan_id == None,
System.loan_id == Job.owner_id,
),
), # and
)
# We reapply this filter here in case a peer recipe has locked the recipe
# set in to a particular lab controller earlier in this scheduling pass
recipe = MachineRecipe.by_id(recipe_id)
if recipe.recipeset.lab_controller:
systems = systems.filter(
System.lab_controller == recipe.recipeset.lab_controller)
# Something earlier in this pass meant we can't schedule this recipe
# right now after all. We'll try again next pass.
if not systems.count():
return
# Order systems by owner, then Group, finally shared for everyone.
# FIXME Make this configurable, so that a user can specify their scheduling
# Implemented order, still need to do pool
# preference from the job.
# <recipe>
# <autopick order='sequence|random'>
# <pool>owner</pool>
# <pool>groups</pool>
# <pool>public</pool>
# </autopick>
# </recipe>
user = recipe.recipeset.job.owner
if True: #FIXME if pools are defined add them here in the order requested.
systems = System.scheduler_ordering(user, query=systems)
if recipe.autopick_random:
system = systems[random.randrange(0, systems.count())]
else:
system = systems.first()
log.debug("System : %s is available for Recipe %s" % (system, recipe.id))
# Check to see if user still has proper permissions to use the system.
# Remember the mapping of available systems could have happend hours or even
# days ago and groups or loans could have been put in place since.
if not recipe.candidate_systems().filter(System.id == system.id).first():
log.debug("System : %s recipe: %s no longer has access. removing" %
(system, recipe.id))
recipe.systems.remove(system)
return
recipe.resource = SystemResource(system=system)
# Reserving the system may fail here if someone stole it out from
# underneath us, but that is fine...
recipe.resource.allocate()
recipe.schedule()
recipe.createRepo()
recipe.recipeset.lab_controller = system.lab_controller
recipe.systems = []
# Create the watchdog without an Expire time.
log.debug("Created watchdog for recipe id: %s and system: %s" %
(recipe.id, system))
recipe.watchdog = Watchdog()
log.info("recipe ID %s moved from Queued to Scheduled" % recipe.id)
for guestrecipe in recipe.guests:
guestrecipe.resource = GuestResource()
guestrecipe.resource.allocate()
guestrecipe.schedule()
guestrecipe.createRepo()
guestrecipe.watchdog = Watchdog()
log.info('recipe ID %s guest %s moved from Queued to Scheduled',
recipe.id, guestrecipe.id)
def provision_virt_recipe(recipe_id):
log.debug('Attempting to provision dynamic virt guest for recipe %s',
recipe_id)
session.begin()
try:
recipe = Recipe.by_id(recipe_id)
job_owner = recipe.recipeset.job.owner
manager = dynamic_virt.VirtManager(job_owner)
available_flavors = manager.available_flavors()
# We want them in order of smallest to largest, so that we can pick the
# smallest flavor that satisfies the recipe's requirements. Sorting by RAM
# is a decent approximation.
possible_flavors = XmlHost.from_string(recipe.host_requires)\
.filter_openstack_flavors(available_flavors, manager.lab_controller)
if not possible_flavors:
log.info(
'No OpenStack flavors matched recipe %s, marking precluded',
recipe.id)
recipe.virt_status = RecipeVirtStatus.precluded
return
# cheapest flavor has the smallest disk and ram
# id guarantees consistency of our results
flavor = min(possible_flavors,
key=lambda flavor: (flavor.ram, flavor.disk, flavor.id))
vm_name = '%srecipe-%s' % (ConfigItem.by_name(
u'guest_name_prefix').current_value(u'beaker-'), recipe.id)
log.debug('Creating VM named %s as flavor %s', vm_name, flavor)
vm = manager.create_vm(vm_name, flavor)
vm.instance_created = datetime.utcnow()
try:
recipe.createRepo()
recipe.clear_candidate_systems()
recipe.watchdog = Watchdog()
recipe.resource = vm
recipe.recipeset.lab_controller = manager.lab_controller
recipe.virt_status = RecipeVirtStatus.succeeded
recipe.schedule()
log.info(
"recipe ID %s moved from Queued to Scheduled by provision_virt_recipe",
recipe.id)
recipe.waiting()
recipe.provision()
log.info(
"recipe ID %s moved from Scheduled to Waiting by provision_virt_recipe",
recipe.id)
except:
exc_type, exc_value, exc_tb = sys.exc_info()
try:
manager.destroy_vm(vm)
except Exception:
log.exception(
'Failed to clean up VM %s during provision_virt_recipe, leaked!',
vm.instance_id)
# suppress this exception so the original one is not masked
raise exc_type, exc_value, exc_tb
session.commit()
except Exception as e:
log.exception('Error in provision_virt_recipe(%s)', recipe_id)
session.rollback()
# As an added precaution, let's try and avoid this recipe in future
with session.begin():
recipe = Recipe.by_id(recipe_id)
recipe.virt_status = RecipeVirtStatus.failed
finally:
session.close()
