def has_active_recipes(self, id): labcontroller = LabController.by_id(id) count = Watchdog.by_status(labcontroller=labcontroller, status='active').count() if count: return {'has_active_recipes' : True} else: return {'has_active_recipes' : False}
def remove(self, id, *args, **kw): try: labcontroller = LabController.by_id(id) labcontroller.removed = datetime.utcnow() systems = System.query.filter_by(lab_controller_id=id).values(System.id) for system_id in systems: sys_activity = SystemActivity(identity.current.user, 'WEBUI', \ 'Changed', 'lab_controller', labcontroller.fqdn, None, system_id=system_id[0]) system_table.update().where(system_table.c.lab_controller_id == id).\ values(lab_controller_id=None).execute() watchdogs = Watchdog.by_status(labcontroller=labcontroller, status='active') for w in watchdogs: w.recipe.recipeset.job.cancel(msg='LabController %s has been deleted' % labcontroller.fqdn) for lca in labcontroller._distro_trees: lca.distro_tree.activity.append(DistroTreeActivity( user=identity.current.user, service=u'WEBUI', action=u'Removed', field_name=u'lab_controller_assocs', old_value=u'%s %s' % (lca.lab_controller, lca.url), new_value=None)) session.delete(lca) labcontroller.disabled = True LabControllerActivity(identity.current.user, 'WEBUI', 'Changed', 'Disabled', unicode(False), unicode(True), lab_controller_id=id) LabControllerActivity(identity.current.user, 'WEBUI', 'Changed', 'Removed', unicode(False), unicode(True), lab_controller_id=id) session.commit() finally: session.close() flash( _(u"%s removed") % labcontroller.fqdn ) raise redirect(".")
def remove(self, id, *args, **kw): labcontroller = LabController.by_id(id) labcontroller.removed = datetime.utcnow() # de-associate systems systems = System.query.filter(System.lab_controller == labcontroller) System.record_bulk_activity(systems, user=identity.current.user, service=u'WEBUI', action=u'Changed', field=u'lab_controller', old=labcontroller.fqdn, new=None) systems.update({'lab_controller_id': None}, synchronize_session=False) # cancel running recipes watchdogs = Watchdog.by_status(labcontroller=labcontroller, status='active') for w in watchdogs: w.recipe.recipeset.job.cancel(msg='LabController %s has been deleted' % labcontroller.fqdn) # remove distro trees distro_tree_assocs = LabControllerDistroTree.query\ .filter(LabControllerDistroTree.lab_controller == labcontroller)\ .join(LabControllerDistroTree.distro_tree) DistroTree.record_bulk_activity(distro_tree_assocs, user=identity.current.user, service=u'WEBUI', action=u'Removed', field=u'lab_controller_assocs', old=labcontroller.fqdn, new=None) distro_tree_assocs.delete(synchronize_session=False) labcontroller.disabled = True labcontroller.record_activity(user=identity.current.user, service=u'WEBUI', field=u'Disabled', action=u'Changed', old=unicode(False), new=unicode(True)) labcontroller.record_activity(user=identity.current.user, service=u'WEBUI', field=u'Removed', action=u'Changed', old=unicode(False), new=unicode(True)) flash( _(u"%s removed") % labcontroller.fqdn ) raise redirect(".")
def watchdogs(self, status='active', lc=None): """ Return all active/expired tasks for this lab controller The lab controllers login with host/fqdn """ # TODO work on logic that determines whether or not originator # was qpid or kobo ? if lc is None: try: labcontroller = identity.current.user.lab_controller except AttributeError: raise BX( _('No lab controller passed in and not currently logged in' )) if not labcontroller: raise BX( _(u'Invalid login: %s, must log in as a lab controller' % identity.current.user)) else: try: labcontroller = LabController.by_name(lc) except InvalidRequestError: raise BX(_(u'Invalid lab controller: %s' % lc)) return [ dict(recipe_id=w.recipe.id, system=w.recipe.resource.fqdn, is_virt_recipe=(w.recipe.resource.type == ResourceType.virt)) for w in Watchdog.by_status(labcontroller, status) ]
def index(self, *args, **kw): query = Watchdog.by_status(status=u'active')\ .join(Watchdog.recipe).join(Recipe.recipeset).join(RecipeSet.job)\ .order_by(Job.id)\ .options( joinedload_all(Watchdog.recipe, Recipe.recipeset, RecipeSet.job), joinedload_all(Watchdog.recipe, Recipe.recipeset, RecipeSet.lab_controller), joinedload_all(Watchdog.recipetask, RecipeTask.task)) col = myPaginateDataGrid.Column fields = [ col(name='job_id', getter=lambda x: x.recipe.recipeset.job.link, title="Job ID"), col(name='system_name', getter=lambda x: x.recipe.resource.link, title="System"), col(name='lab_controller', getter=lambda x: x.recipe.recipeset.lab_controller, title="Lab Controller"), col(name='task_name', getter=lambda x: x.recipetask.name_markup if x.recipetask is not None else None, title="Task Name"), col(name='kill_time', getter=lambda x: x.kill_time, title="Kill Time", options=dict(datetime=True)) ] watchdog_grid = myPaginateDataGrid(fields=fields) return dict(title="Watchdogs", grid=watchdog_grid, search_bar=None, list=query)
def mark_recipe_waiting(recipe, start_time=None, system=None, lab_controller=None, virt=False, instance_id=None, **kwargs): if start_time is None: start_time = datetime.datetime.utcnow() recipe.process() recipe.queue() recipe.schedule() if not recipe.resource: if isinstance(recipe, MachineRecipe): if virt: if not lab_controller: lab_controller = create_labcontroller(fqdn=u'dummylab.example.invalid') if not instance_id: instance_id = uuid.uuid4() recipe.resource = VirtResource(instance_id, lab_controller) recipe.recipeset.lab_controller = lab_controller else: if not system: if not lab_controller: lab_controller = create_labcontroller(fqdn=u'dummylab.example.invalid') system = create_system(arch=recipe.arch, lab_controller=lab_controller) recipe.resource = SystemResource(system=system) recipe.resource.allocate() recipe.resource.reservation.start_time = start_time recipe.recipeset.lab_controller = system.lab_controller elif isinstance(recipe, GuestRecipe): recipe.resource = GuestResource() recipe.resource.allocate() recipe.start_time = start_time recipe.watchdog = Watchdog() recipe.waiting() recipe.resource.rebooted = start_time recipe.recipeset.job.update_status() log.debug('Marked %s as waiting with system %s', recipe.t_id, recipe.resource.fqdn)
def has_active_recipes(self, id): labcontroller = LabController.by_id(id) count = Watchdog.by_status(labcontroller=labcontroller, status='active').count() if count: return {'has_active_recipes': True} else: return {'has_active_recipes': False}
def provision_virt_recipe(recipe_id): log.debug('Attempting to provision dynamic virt guest for recipe %s', recipe_id) session.begin() try: recipe = Recipe.by_id(recipe_id) manager = dynamic_virt.VirtManager(recipe.recipeset.job.owner) available_flavors = manager.available_flavors() # We want them in order of smallest to largest, so that we can pick the # smallest flavor that satisfies the recipe's requirements. Sorting by RAM # is a decent approximation. possible_flavors = XmlHost.from_string(recipe.host_requires)\ .filter_openstack_flavors(available_flavors, manager.lab_controller) if not possible_flavors: log.debug('No OpenStack flavors matched recipe %s, marking precluded', recipe.id) recipe.virt_status = RecipeVirtStatus.precluded return possible_flavors = sorted(possible_flavors, key=lambda flavor: flavor.ram) flavor = possible_flavors[0] vm_name = '%srecipe-%s' % ( ConfigItem.by_name(u'guest_name_prefix').current_value(u'beaker-'), recipe.id) # FIXME can we control use of virtio? #virtio_possible = True #if self.recipe.distro_tree.distro.osversion.osmajor.osmajor == "RedHatEnterpriseLinux3": # virtio_possible = False vm = manager.create_vm(vm_name, flavor) vm.instance_created = datetime.utcnow() try: recipe.createRepo() recipe.systems = [] recipe.watchdog = Watchdog() recipe.resource = vm recipe.recipeset.lab_controller = manager.lab_controller recipe.virt_status = RecipeVirtStatus.succeeded recipe.schedule() log.info("recipe ID %s moved from Queued to Scheduled by provision_virt_recipe" % recipe.id) recipe.waiting() recipe.provision() log.info("recipe ID %s moved from Scheduled to Waiting by provision_virt_recipe" % recipe.id) except: exc_type, exc_value, exc_tb = sys.exc_info() try: manager.destroy_vm(vm) except Exception: log.exception('Failed to clean up vm %s ' 'during provision_virt_recipe, leaked!', vm.instance_id) # suppress this exception so the original one is not masked raise exc_type, exc_value, exc_tb session.commit() except Exception, e: log.exception('Error in provision_virt_recipe(%s)', recipe_id) session.rollback() # As an added precaution, let's try and avoid this recipe in future with session.begin(): recipe = Recipe.by_id(recipe_id) recipe.virt_status = RecipeVirtStatus.failed
def remove_labcontroller(labcontroller): """ Disables and marks a lab controller as removed. """ labcontroller.removed = datetime.utcnow() systems = System.query.filter(System.lab_controller == labcontroller) # Record systems set to status=broken. Trigger any event listener listening # for status changes. for sys in systems: sys.mark_broken('Lab controller de-associated') sys.abort_queued_commands("System disassociated from lab controller") # de-associate systems System.record_bulk_activity(systems, user=identity.current.user, service=u'HTTP', action=u'Changed', field=u'Lab Controller', old=labcontroller.fqdn, new=None) systems.update({'lab_controller_id': None}, synchronize_session=False) # cancel running recipes watchdogs = Watchdog.by_status(labcontroller=labcontroller, status='active') for w in watchdogs: w.recipe.recipeset.job.cancel( msg='Lab controller %s has been deleted' % labcontroller.fqdn) # remove distro trees distro_tree_assocs = LabControllerDistroTree.query\ .filter(LabControllerDistroTree.lab_controller == labcontroller) DistroTree.record_bulk_activity(distro_tree_assocs.join( LabControllerDistroTree.distro_tree), user=identity.current.user, service=u'HTTP', action=u'Removed', field=u'lab_controller_assocs', old=labcontroller.fqdn, new=None) distro_tree_assocs.delete(synchronize_session=False) labcontroller.disabled = True labcontroller.record_activity(user=identity.current.user, service=u'HTTP', field=u'Disabled', action=u'Changed', old=unicode(False), new=unicode(True)) labcontroller.record_activity(user=identity.current.user, service=u'HTTP', field=u'Removed', action=u'Changed', old=unicode(False), new=unicode(True))
def extend(self, time): '''Allow admins to push watchdog times out after an outage''' watchdogs = [] for w in Watchdog.by_status(status=u'active'): n_kill_time = w.kill_time + timedelta(seconds=time) watchdogs.append("R:%s watchdog moved from %s to %s" % (w.recipe_id, w.kill_time, n_kill_time)) w.kill_time = n_kill_time if watchdogs: return "\n".join(watchdogs) else: return 'No active watchdogs found'
def extend(self, time): '''Allow admins to push watchdog times out after an outage''' watchdogs = [] for w in Watchdog.by_status(status=u'active'): n_kill_time = w.kill_time + timedelta(seconds=time) watchdogs.append("R:%s watchdog moved from %s to %s" % ( w.recipe_id, w.kill_time, n_kill_time)) w.kill_time = n_kill_time if watchdogs: return "\n".join(watchdogs) else: return 'No active watchdogs found'
def schedule_recipe_on_system(recipe, system): log.debug('Assigning recipe %s to system %s', recipe.id, system.fqdn) recipe.resource = SystemResource(system=system) # Reserving the system may fail here if someone stole it out from # underneath us, but that is fine... recipe.resource.allocate() recipe.schedule() recipe.createRepo() recipe.recipeset.lab_controller = system.lab_controller recipe.systems = [] # Create the watchdog without an Expire time. log.debug("Created watchdog for recipe id: %s and system: %s" % (recipe.id, system)) recipe.watchdog = Watchdog() log.info("recipe ID %s moved from Queued to Scheduled" % recipe.id) for guestrecipe in recipe.guests: guestrecipe.resource = GuestResource() guestrecipe.resource.allocate() guestrecipe.schedule() guestrecipe.createRepo() guestrecipe.watchdog = Watchdog() log.info('recipe ID %s guest %s moved from Queued to Scheduled', recipe.id, guestrecipe.id)
def remove_labcontroller(labcontroller): """ Disables and marks a lab controller as removed. """ labcontroller.removed = datetime.utcnow() # de-associate systems systems = System.query.filter(System.lab_controller == labcontroller) System.record_bulk_activity(systems, user=identity.current.user, service=u'HTTP', action=u'Changed', field=u'lab_controller', old=labcontroller.fqdn, new=None) systems.update({'lab_controller_id': None}, synchronize_session=False) # cancel running recipes watchdogs = Watchdog.by_status(labcontroller=labcontroller, status='active') for w in watchdogs: w.recipe.recipeset.job.cancel(msg='LabController %s has been deleted' % labcontroller.fqdn) # remove distro trees distro_tree_assocs = LabControllerDistroTree.query\ .filter(LabControllerDistroTree.lab_controller == labcontroller)\ .join(LabControllerDistroTree.distro_tree) DistroTree.record_bulk_activity(distro_tree_assocs, user=identity.current.user, service=u'HTTP', action=u'Removed', field=u'lab_controller_assocs', old=labcontroller.fqdn, new=None) distro_tree_assocs.delete(synchronize_session=False) labcontroller.disabled = True labcontroller.record_activity(user=identity.current.user, service=u'HTTP', field=u'Disabled', action=u'Changed', old=unicode(False), new=unicode(True)) labcontroller.record_activity(user=identity.current.user, service=u'HTTP', field=u'Removed', action=u'Changed', old=unicode(False), new=unicode(True))
def mark_recipe_scheduled(recipe, start_time=None, system=None, fqdn=None, mac_address=None, lab_controller=None, virt=False, instance_id=None, **kwargs): recipe.process() recipe.queue() recipe.schedule() if not recipe.resource: if isinstance(recipe, MachineRecipe): if virt: if not lab_controller: lab_controller = create_labcontroller( fqdn=u'dummylab.example.invalid') if not instance_id: instance_id = uuid.uuid4() recipe.resource = VirtResource(instance_id, lab_controller) recipe.recipeset.lab_controller = lab_controller else: if not system: if not lab_controller: lab_controller = create_labcontroller( fqdn=u'dummylab.example.invalid') system = create_system(arch=recipe.arch.arch, fqdn=fqdn, lab_controller=lab_controller) recipe.resource = SystemResource(system=system) recipe.resource.allocate() recipe.resource.reservation.start_time = start_time or datetime.datetime.utcnow( ) recipe.recipeset.lab_controller = system.lab_controller elif isinstance(recipe, GuestRecipe): recipe.resource = GuestResource() recipe.resource.allocate() if mac_address is not None: recipe.resource.mac_address = netaddr.EUI( mac_address, dialect=mac_unix_padded_dialect) if not recipe.distro_tree.url_in_lab(recipe.recipeset.lab_controller): add_distro_tree_to_lab(recipe.distro_tree, recipe.recipeset.lab_controller) recipe.watchdog = Watchdog() log.debug('Marked %s as scheduled with system %s', recipe.t_id, recipe.resource.fqdn)
def remove_labcontroller(labcontroller): """ Disables and marks a lab controller as removed. """ labcontroller.removed = datetime.utcnow() systems = System.query.filter(System.lab_controller == labcontroller) # Record systems set to status=broken. Trigger any event listener listening # for status changes. for sys in systems: sys.mark_broken('Lab controller de-associated') # de-associate systems System.record_bulk_activity(systems, user=identity.current.user, service=u'HTTP', action=u'Changed', field=u'Lab Controller', old=labcontroller.fqdn, new=None) systems.update({'lab_controller_id': None}, synchronize_session=False) # cancel running recipes watchdogs = Watchdog.by_status(labcontroller=labcontroller, status='active') for w in watchdogs: w.recipe.recipeset.job.cancel(msg='Lab controller %s has been deleted' % labcontroller.fqdn) # remove distro trees distro_tree_assocs = LabControllerDistroTree.query\ .filter(LabControllerDistroTree.lab_controller == labcontroller)\ .join(LabControllerDistroTree.distro_tree) DistroTree.record_bulk_activity( distro_tree_assocs, user=identity.current.user, service=u'HTTP', action=u'Removed', field=u'lab_controller_assocs', old=labcontroller.fqdn, new=None) distro_tree_assocs.delete(synchronize_session=False) labcontroller.disabled = True labcontroller.record_activity( user=identity.current.user, service=u'HTTP', field=u'Disabled', action=u'Changed', old=unicode(False), new=unicode(True)) labcontroller.record_activity( user=identity.current.user, service=u'HTTP', field=u'Removed', action=u'Changed', old=unicode(False), new=unicode(True))
def watchdogs(self, status='active',lc=None): """ Return all active/expired tasks for this lab controller The lab controllers login with host/fqdn """ # TODO work on logic that determines whether or not originator # was qpid or kobo ? if lc is None: try: labcontroller = identity.current.user.lab_controller except AttributeError: raise BX(_('No lab controller passed in and not currently logged in')) if not labcontroller: raise BX(_(u'Invalid login: %s, must log in as a lab controller' % identity.current.user)) else: try: labcontroller = LabController.by_name(lc) except InvalidRequestError: raise BX(_(u'Invalid lab controller: %s' % lc)) return [dict(recipe_id = w.recipe.id, system = w.recipe.resource.fqdn) for w in Watchdog.by_status(labcontroller, status)]
def test_recovers_leaked_system(self): # recipe is cancelled but system has still been assigned recipe = data_setup.create_recipe() data_setup.create_job_for_recipes([recipe]) recipe._abort_cancel(TaskStatus.cancelled) recipe.recipeset.job.update_status() system = data_setup.create_system() recipe.resource = SystemResource(system=system) recipe.resource.allocate() recipe.watchdog = Watchdog() session.flush() self.assert_(system.open_reservation is not None) self.assert_(system.open_reservation is recipe.resource.reservation) self.assert_(system.user is not None) self.assert_(recipe.watchdog is not None) recipe.recipeset.job.update_status() session.flush() session.expire_all() self.assertEquals(system.open_reservation, None) self.assertEquals(system.user, None) self.assertEquals(recipe.watchdog, None)
def index(self, *args, **kw): query = Watchdog.by_status(status=u'active')\ .join(Watchdog.recipe, Recipe.recipeset, RecipeSet.job)\ .order_by(Job.id)\ .options( joinedload_all(Watchdog.recipe, Recipe.recipeset, RecipeSet.job), joinedload_all(Watchdog.recipe, Recipe.recipeset, RecipeSet.lab_controller), joinedload_all(Watchdog.recipetask, RecipeTask.task)) col = myPaginateDataGrid.Column fields = [col(name='job_id', getter=lambda x: x.recipe.recipeset.job.link, title="Job ID"), col(name='system_name', getter=lambda x: x.recipe.resource.link, title="System"), col(name='lab_controller', getter=lambda x: x.recipe.recipeset.lab_controller, title="Lab Controller"), col(name='task_name', getter=lambda x: x.recipetask.name_markup if x.recipetask is not None else None, title="Task Name"), col(name='kill_time', getter=lambda x: x.kill_time, title="Kill Time", options=dict(datetime=True))] watchdog_grid = myPaginateDataGrid(fields=fields) return dict(title="Watchdogs", grid=watchdog_grid, search_bar=None, list=query)
def schedule_queued_recipe(recipe_id, guest_recipe_id=None): log.debug('Selecting a system for recipe %s', recipe_id) guest_recipe = aliased(Recipe) guest_distros_map = aliased(LabControllerDistroTree) guest_labcontroller = aliased(LabController) # This query will return all the systems that a recipe is # able to run on. A system is deemed eligible if: # * If the recipe's distro tree is available to the system's lab controller # * The system is available (see the filter criteria). # * If it's a host recipe, then the system needs to be on a lab controller # that can access the distro tree of both the host recipe, # and the guest recipe. systems = System.query.join(System.queued_recipes) \ .outerjoin(System.cpu) \ .join(Recipe.recipeset, RecipeSet.job) \ .join(System.lab_controller, LabController._distro_trees)\ .join((DistroTree, and_(LabControllerDistroTree.distro_tree_id == DistroTree.id, Recipe.distro_tree_id == DistroTree.id)))\ .outerjoin((machine_guest_map, Recipe.id == machine_guest_map.c.machine_recipe_id))\ .outerjoin((guest_recipe, machine_guest_map.c.guest_recipe_id == guest_recipe.id ))\ .outerjoin((guest_distros_map, guest_recipe.distro_tree_id == guest_distros_map.distro_tree_id))\ .outerjoin((guest_labcontroller, guest_distros_map.lab_controller_id == guest_labcontroller.id))\ .filter(Recipe.id == recipe_id) \ .filter(or_(guest_recipe.id == guest_recipe_id, guest_recipe.id == None))\ .filter(and_(System.user == None, or_(guest_distros_map.id == None, and_(guest_distros_map.id != None, guest_labcontroller.id == LabController.id, ), ), LabController.disabled == False, or_(System.loan_id == None, System.loan_id == Job.owner_id, ), ), # and ) # We reapply this filter here in case a peer recipe has locked the recipe # set in to a particular lab controller earlier in this scheduling pass recipe = MachineRecipe.by_id(recipe_id) if recipe.recipeset.lab_controller: systems = systems.filter( System.lab_controller == recipe.recipeset.lab_controller) # Something earlier in this pass meant we can't schedule this recipe # right now after all. We'll try again next pass. if not systems.count(): return # Order systems by owner, then Group, finally shared for everyone. # FIXME Make this configurable, so that a user can specify their scheduling # Implemented order, still need to do pool # preference from the job. # <recipe> # <autopick order='sequence|random'> # <pool>owner</pool> # <pool>groups</pool> # <pool>public</pool> # </autopick> # </recipe> user = recipe.recipeset.job.owner if True: #FIXME if pools are defined add them here in the order requested. systems = System.scheduler_ordering(user, query=systems) if recipe.autopick_random: system = systems[random.randrange(0, systems.count())] else: system = systems.first() log.debug("System : %s is available for Recipe %s" % (system, recipe.id)) # Check to see if user still has proper permissions to use the system. # Remember the mapping of available systems could have happend hours or even # days ago and groups or loans could have been put in place since. if not recipe.candidate_systems().filter(System.id == system.id).first(): log.debug("System : %s recipe: %s no longer has access. removing" % (system, recipe.id)) recipe.systems.remove(system) return recipe.resource = SystemResource(system=system) # Reserving the system may fail here if someone stole it out from # underneath us, but that is fine... recipe.resource.allocate() recipe.schedule() recipe.createRepo() recipe.recipeset.lab_controller = system.lab_controller recipe.systems = [] # Create the watchdog without an Expire time. log.debug("Created watchdog for recipe id: %s and system: %s" % (recipe.id, system)) recipe.watchdog = Watchdog() log.info("recipe ID %s moved from Queued to Scheduled" % recipe.id) for guestrecipe in recipe.guests: guestrecipe.resource = GuestResource() guestrecipe.resource.allocate() guestrecipe.schedule() guestrecipe.createRepo() guestrecipe.watchdog = Watchdog() log.info('recipe ID %s guest %s moved from Queued to Scheduled', recipe.id, guestrecipe.id)
def provision_virt_recipe(recipe_id): log.debug('Attempting to provision dynamic virt guest for recipe %s', recipe_id) session.begin() try: recipe = Recipe.by_id(recipe_id) job_owner = recipe.recipeset.job.owner manager = dynamic_virt.VirtManager(job_owner) available_flavors = manager.available_flavors() # We want them in order of smallest to largest, so that we can pick the # smallest flavor that satisfies the recipe's requirements. Sorting by RAM # is a decent approximation. possible_flavors = XmlHost.from_string(recipe.host_requires)\ .filter_openstack_flavors(available_flavors, manager.lab_controller) if not possible_flavors: log.info( 'No OpenStack flavors matched recipe %s, marking precluded', recipe.id) recipe.virt_status = RecipeVirtStatus.precluded return # cheapest flavor has the smallest disk and ram # id guarantees consistency of our results flavor = min(possible_flavors, key=lambda flavor: (flavor.ram, flavor.disk, flavor.id)) vm_name = '%srecipe-%s' % (ConfigItem.by_name( u'guest_name_prefix').current_value(u'beaker-'), recipe.id) log.debug('Creating VM named %s as flavor %s', vm_name, flavor) vm = manager.create_vm(vm_name, flavor) vm.instance_created = datetime.utcnow() try: recipe.createRepo() recipe.clear_candidate_systems() recipe.watchdog = Watchdog() recipe.resource = vm recipe.recipeset.lab_controller = manager.lab_controller recipe.virt_status = RecipeVirtStatus.succeeded recipe.schedule() log.info( "recipe ID %s moved from Queued to Scheduled by provision_virt_recipe", recipe.id) recipe.waiting() recipe.provision() log.info( "recipe ID %s moved from Scheduled to Waiting by provision_virt_recipe", recipe.id) except: exc_type, exc_value, exc_tb = sys.exc_info() try: manager.destroy_vm(vm) except Exception: log.exception( 'Failed to clean up VM %s during provision_virt_recipe, leaked!', vm.instance_id) # suppress this exception so the original one is not masked raise exc_type, exc_value, exc_tb session.commit() except Exception as e: log.exception('Error in provision_virt_recipe(%s)', recipe_id) session.rollback() # As an added precaution, let's try and avoid this recipe in future with session.begin(): recipe = Recipe.by_id(recipe_id) recipe.virt_status = RecipeVirtStatus.failed finally: session.close()