def update_db(self): self.logger.info('Updating local Beaker database..') for task_rpm in self.tasks_added: self.logger.debug('Adding %s'% task_rpm) with open(os.path.join(self.task_dir,task_rpm)) as f: try: session.begin() task = self.tasks.process_taskinfo(self.tasks.read_taskinfo(f)) old_rpm = task.rpm task.rpm = task_rpm session.commit() except Exception: session.rollback() session.close() self.logger.critical('Error adding task %s' % task_rpm) unlink_ignore(task_rpm) else: session.close() self.logger.debug('Successfully added %s' % task.rpm) if old_rpm: unlink_ignore(os.path.join(self.task_dir, old_rpm)) # Update task repo self.logger.info('Creating repodata..') Task.update_repo() return
def provision_virt_recipes(*args): recipes = MachineRecipe.query\ .join(Recipe.recipeset).join(RecipeSet.job)\ .filter(Job.dirty_version == Job.clean_version)\ .join(Recipe.distro_tree, DistroTree.lab_controller_assocs, LabController)\ .filter(Recipe.status == TaskStatus.queued)\ .filter(Recipe.virt_status == RecipeVirtStatus.possible)\ .filter(LabController.disabled == False)\ .filter(or_(RecipeSet.lab_controller == None, RecipeSet.lab_controller_id == LabController.id))\ .order_by(RecipeSet.priority.desc(), Recipe.id.asc()) if not recipes.count(): return False log.debug("Entering provision_virt_recipes") for recipe_id, in recipes.values(Recipe.id.distinct()): session.begin() try: provision_virt_recipe(recipe_id) session.commit() except Exception, e: log.exception('Error in provision_virt_recipe(%s)', recipe_id) session.rollback() # As an added precaution, let's try and avoid this recipe in future with session.begin(): recipe = Recipe.by_id(recipe_id) recipe.virt_status = RecipeVirtStatus.failed finally:
def save(self, task_rpm, *args, **kw): """ TurboGears method to upload task rpm package """ rpm_path = Task.get_rpm_path(task_rpm.filename) if not task_rpm.filename: flash(_(u'No task RPM specified')) redirect(url("./new")) # we do it here, since we do not want to proceed # any further if len(task_rpm.filename) > 255: flash(_(u"Task RPM name should be <= 255 characters")) redirect(url("./new")) if os.path.exists("%s" % rpm_path): flash( _(u'Failed to import because we already have %s' % task_rpm.filename)) redirect(url("./new")) try: def write_data(f): siphon(task_rpm.file, f) task = Task.update_task(task_rpm.filename, write_data) except Exception, err: session.rollback() log.exception('Failed to import %s', task_rpm.filename) flash(_(u'Failed to import task: %s' % err)) redirect(url("./new"))
def test_uploading_job_with_invalid_hostRequires_raises_exception(self): session.begin() try: xmljob = XmlJob(xmltramp.parse(''' <job> <whiteboard>job with invalid hostRequires</whiteboard> <recipeSet> <recipe> <distroRequires> <distro_name op="=" value="BlueShoeLinux5-5" /> </distroRequires> <hostRequires> <memory op=">=" value="500MB" /> </hostRequires> <task name="/distribution/install" role="STANDALONE"> <params/> </task> </recipe> </recipeSet> </job> ''')) self.assertRaises(BX, lambda: self.controller.process_xmljob(xmljob, self.user)) finally: session.rollback() session.close()
def provision_scheduled_recipesets(*args): """ if All recipes in a recipeSet are in Scheduled state then move them to Running. """ recipesets = RecipeSet.query.join(RecipeSet.job)\ .filter(and_(Job.dirty_version == Job.clean_version, Job.deleted == None))\ .filter(not_(RecipeSet.recipes.any( Recipe.status != TaskStatus.scheduled))) if not recipesets.count(): return False log.debug("Entering provision_scheduled_recipesets") for rs_id, in recipesets.values(RecipeSet.id): log.info("scheduled_recipesets: RS:%s" % rs_id) session.begin() try: provision_scheduled_recipeset(rs_id) session.commit() except exceptions.Exception: log.exception('Error in provision_scheduled_recipeset(%s)', rs_id) session.rollback() finally: session.close() log.debug("Exiting provision_scheduled_recipesets") return True
def abort_dead_recipes(*args): work_done = False with session.begin(): filters = [not_(DistroTree.lab_controller_assocs.any())] if _virt_enabled(): filters.append( and_(not_(Recipe.systems.any()), Recipe.virt_status != RecipeVirtStatus.possible)) else: filters.append(not_(Recipe.systems.any())) recipes = MachineRecipe.query\ .join(MachineRecipe.recipeset).join(RecipeSet.job)\ .filter(Job.dirty_version == Job.clean_version)\ .outerjoin(Recipe.distro_tree)\ .filter(Recipe.status == TaskStatus.queued)\ .filter(or_(*filters)) recipe_ids = [ recipe_id for recipe_id, in recipes.values(MachineRecipe.id) ] if recipe_ids: log.debug('Aborting dead recipes [%s ... %s] (%d total)', recipe_ids[0], recipe_ids[-1], len(recipe_ids)) for recipe_id in recipe_ids: session.begin() try: abort_dead_recipe(recipe_id) session.commit() except exceptions.Exception, e: log.exception('Error in abort_dead_recipe(%s)', recipe_id) session.rollback() finally:
def save(self, task_rpm, *args, **kw): """ TurboGears method to upload task rpm package """ rpm_path = Task.get_rpm_path(task_rpm.filename) if not task_rpm.filename: flash(_(u'No task RPM specified')) redirect(url("./new")) # we do it here, since we do not want to proceed # any further if len(task_rpm.filename) > 255: flash(_(u"Task RPM name should be <= 255 characters")) redirect(url("./new")) if os.path.exists("%s" % rpm_path): flash(_(u'Failed to import because we already have %s' % task_rpm.filename )) redirect(url("./new")) try: def write_data(f): siphon(task_rpm.file, f) task = Task.update_task(task_rpm.filename, write_data) except Exception, err: session.rollback() log.exception('Failed to import %s', task_rpm.filename) flash(_(u'Failed to import task: %s' % err)) redirect(url("./new"))
def genkey(self): username = turbogears.identity.current.user_name person = People.by_username(username) created = time.strftime("%Y-%m-%dT%H:%M:%S") hexctr = "%012x" % person.id publicname = hex2modhex(hexctr) internalname = gethexrand(12) aeskey = gethexrand(32) lockcode = gethexrand(12) try: new_ykksm = Ykksm(serialnr=person.id, publicname=publicname, created=created, internalname=internalname, aeskey=aeskey, lockcode=lockcode, creator=username) session.add(new_ykksm) session.flush() except IntegrityError: session.rollback() old_ykksm = session.query(Ykksm).filter_by(serialnr=person.id).all()[0] session.delete(old_ykksm) new_ykksm = Ykksm(serialnr=person.id, publicname=publicname, created=created, internalname=internalname, aeskey=aeskey, lockcode=lockcode, creator=username) old_ykksm = new_ykksm session.flush() try: old_ykval = session.query(Ykval).filter_by(yk_publicname=publicname).all()[0] session.delete(old_ykval) session.flush() except IndexError: # No old record? Maybe they never used their key pass string = "%s %s %s" % (publicname, internalname, aeskey) return dict(key=string)
def update_products(xml_file): dom = etree.parse(xml_file) xpath_string = "//cpe" cpes = dom.xpath(xpath_string) session.begin() try: to_add = {} dupe_errors = [] for cpe in cpes: cpe_text = cpe.text if cpe_text in to_add: dupe_errors.append(cpe_text) else: to_add[cpe_text] = 1 for cpe_to_add in to_add: try: prod = Product.by_name(u"%s" % cpe_to_add) except NoResultFound: session.add(Product(u"%s" % cpe_to_add)) continue session.commit() finally: session.rollback()
def abort_dead_recipes(*args): filters = [not_(DistroTree.lab_controller_assocs.any())] if _virt_enabled(): filters.append( and_(not_(Recipe.systems.any()), Recipe.virt_status != RecipeVirtStatus.possible)) else: filters.append(not_(Recipe.systems.any())) recipes = MachineRecipe.query\ .join(MachineRecipe.recipeset).join(RecipeSet.job)\ .filter(Job.dirty_version == Job.clean_version)\ .outerjoin(Recipe.distro_tree)\ .filter(Recipe.status == TaskStatus.queued)\ .filter(or_(*filters)) if not recipes.count(): return False log.debug("Entering abort_dead_recipes") for recipe_id, in recipes.values(MachineRecipe.id): session.begin() try: abort_dead_recipe(recipe_id) session.commit() except exceptions.Exception, e: log.exception('Error in abort_dead_recipe(%s)', recipe_id) session.rollback() finally:
def abort_dead_recipes(*args): filters = [not_(DistroTree.lab_controller_assocs.any())] if _virt_enabled(): filters.append(and_(not_(Recipe.systems.any()), Recipe.virt_status != RecipeVirtStatus.possible)) else: filters.append(not_(Recipe.systems.any())) recipes = MachineRecipe.query\ .join(MachineRecipe.recipeset).join(RecipeSet.job)\ .filter(Job.dirty_version == Job.clean_version)\ .outerjoin(Recipe.distro_tree)\ .filter(Recipe.status == TaskStatus.queued)\ .filter(or_(*filters)) if not recipes.count(): return False log.debug("Entering abort_dead_recipes") for recipe_id, in recipes.values(MachineRecipe.id): session.begin() try: abort_dead_recipe(recipe_id) session.commit() except exceptions.Exception, e: log.exception('Error in abort_dead_recipe(%s)', recipe_id) session.rollback() finally:
def test_uploading_job_with_invalid_hostRequires_raises_exception(self): session.begin() try: xmljob = XmlJob( xmltramp.parse(''' <job> <whiteboard>job with invalid hostRequires</whiteboard> <recipeSet> <recipe> <distroRequires> <distro_name op="=" value="BlueShoeLinux5-5" /> </distroRequires> <hostRequires> <memory op=">=" value="500MB" /> </hostRequires> <task name="/distribution/install" role="STANDALONE"> <params/> </task> </recipe> </recipeSet> </job> ''')) self.assertRaises( BX, lambda: self.controller.process_xmljob(xmljob, self.user)) finally: session.rollback()
def provision_virt_recipe(recipe_id): log.debug('Attempting to provision dynamic virt guest for recipe %s', recipe_id) session.begin() try: recipe = Recipe.by_id(recipe_id) manager = dynamic_virt.VirtManager(recipe.recipeset.job.owner) available_flavors = manager.available_flavors() # We want them in order of smallest to largest, so that we can pick the # smallest flavor that satisfies the recipe's requirements. Sorting by RAM # is a decent approximation. possible_flavors = XmlHost.from_string(recipe.host_requires)\ .filter_openstack_flavors(available_flavors, manager.lab_controller) if not possible_flavors: log.debug('No OpenStack flavors matched recipe %s, marking precluded', recipe.id) recipe.virt_status = RecipeVirtStatus.precluded return possible_flavors = sorted(possible_flavors, key=lambda flavor: flavor.ram) flavor = possible_flavors[0] vm_name = '%srecipe-%s' % ( ConfigItem.by_name(u'guest_name_prefix').current_value(u'beaker-'), recipe.id) # FIXME can we control use of virtio? #virtio_possible = True #if self.recipe.distro_tree.distro.osversion.osmajor.osmajor == "RedHatEnterpriseLinux3": # virtio_possible = False vm = manager.create_vm(vm_name, flavor) vm.instance_created = datetime.utcnow() try: recipe.createRepo() recipe.systems = [] recipe.watchdog = Watchdog() recipe.resource = vm recipe.recipeset.lab_controller = manager.lab_controller recipe.virt_status = RecipeVirtStatus.succeeded recipe.schedule() log.info("recipe ID %s moved from Queued to Scheduled by provision_virt_recipe" % recipe.id) recipe.waiting() recipe.provision() log.info("recipe ID %s moved from Scheduled to Waiting by provision_virt_recipe" % recipe.id) except: exc_type, exc_value, exc_tb = sys.exc_info() try: manager.destroy_vm(vm) except Exception: log.exception('Failed to clean up vm %s ' 'during provision_virt_recipe, leaked!', vm.instance_id) # suppress this exception so the original one is not masked raise exc_type, exc_value, exc_tb session.commit() except Exception, e: log.exception('Error in provision_virt_recipe(%s)', recipe_id) session.rollback() # As an added precaution, let's try and avoid this recipe in future with session.begin(): recipe = Recipe.by_id(recipe_id) recipe.virt_status = RecipeVirtStatus.failed
def provision_virt_recipe(recipe_id): log.debug('Attempting to provision dynamic virt guest for recipe %s', recipe_id) session.begin() try: recipe = Recipe.by_id(recipe_id) manager = dynamic_virt.VirtManager(recipe.recipeset.job.owner) available_flavors = manager.available_flavors() # We want them in order of smallest to largest, so that we can pick the # smallest flavor that satisfies the recipe's requirements. Sorting by RAM # is a decent approximation. possible_flavors = XmlHost.from_string(recipe.host_requires)\ .filter_openstack_flavors(available_flavors, manager.lab_controller) if not possible_flavors: log.debug('No OpenStack flavors matched recipe %s, marking precluded', recipe.id) recipe.virt_status = RecipeVirtStatus.precluded return possible_flavors = sorted(possible_flavors, key=lambda flavor: flavor.ram) flavor = possible_flavors[0] vm_name = '%srecipe-%s' % ( ConfigItem.by_name(u'guest_name_prefix').current_value(u'beaker-'), recipe.id) # FIXME can we control use of virtio? #virtio_possible = True #if self.recipe.distro_tree.distro.osversion.osmajor.osmajor == "RedHatEnterpriseLinux3": # virtio_possible = False vm = manager.create_vm(vm_name, flavor) vm.instance_created = datetime.utcnow() try: recipe.createRepo() recipe.clear_candidate_systems() recipe.watchdog = Watchdog() recipe.resource = vm recipe.recipeset.lab_controller = manager.lab_controller recipe.virt_status = RecipeVirtStatus.succeeded recipe.schedule() log.info("recipe ID %s moved from Queued to Scheduled by provision_virt_recipe" % recipe.id) recipe.waiting() recipe.provision() log.info("recipe ID %s moved from Scheduled to Waiting by provision_virt_recipe" % recipe.id) except: exc_type, exc_value, exc_tb = sys.exc_info() try: manager.destroy_vm(vm) except Exception: log.exception('Failed to clean up vm %s ' 'during provision_virt_recipe, leaked!', vm.instance_id) # suppress this exception so the original one is not masked raise exc_type, exc_value, exc_tb session.commit() except Exception, e: log.exception('Error in provision_virt_recipe(%s)', recipe_id) session.rollback() # As an added precaution, let's try and avoid this recipe in future with session.begin(): recipe = Recipe.by_id(recipe_id) recipe.virt_status = RecipeVirtStatus.failed
def close_session(exception=None): try: if session.is_active: log.warn('Session active when tearing down app context, rolling back') session.rollback() session.close() except Exception, e: # log and suppress log.exception('Error closing session when tearing down app context')
def commit_or_rollback_session(response): # Matches behaviour of TG's sa_rwt: commit on success or redirect, # roll back on error. if session.is_active: if response.status_code >= 200 and response.status_code < 400: session.commit() else: log.debug('Rolling back for %s response', response.status_code) session.rollback() return response
def disable_from_ui(self, t_id, *args, **kw): to_return = dict(t_id=t_id) try: self._disable(t_id) to_return['success'] = True except Exception, e: log.exception('Unable to disable task:%s' % t_id) to_return['success'] = False to_return['err_msg'] = unicode(e) session.rollback()
def disable_from_ui(self, t_id, *args, **kw): to_return = dict( t_id = t_id ) try: self._disable(t_id) to_return['success'] = True except Exception, e: log.exception('Unable to disable task:%s' % t_id) to_return['success'] = False to_return['err_msg'] = unicode(e) session.rollback()
def _mass_branch_worker(self, to_branch, devel_branch, pkgs, author_name): '''Main worker for mass branching. :arg to_branch: Branch to put new PackageListings on :arg devel_branch: Branch for devel, where we're branching from :arg pkgs: List of packages to branch :arg author_name: username of person making the branch :returns: List of packages which had failures while trying to branch This method forks and invokes the branching in a child. That prevents ballooning of memory when all the packages are processed in the main server process (note: separate threads don't seem to help.) ''' in_pipe, out_pipe = os.pipe() pid = os.fork() if pid: # Controller os.close(out_pipe) in_pipe = os.fdopen(in_pipe, 'r') raw_unbranched = in_pipe.read() retcode = os.waitpid(pid, 0) else: # Child os.close(in_pipe) out_pipe = os.fdopen(out_pipe, 'w') unbranched = [] for pkg in pkgs: if pkg['package_name'] in devel_branch.listings2: # clone the package from the devel branch try: devel_branch.listings2[pkg['package_name'] ].clone(to_branch.branchname, author_name) except InvalidRequestError: # Exceptions will be handled later. pass else: # Success, get us to the next package continue # If we get to here we had an error cloning this package unbranched.append(pkg['package_name']) # Commit the changes try: session.flush() #pylint:disable-msg=E1101 except SQLError, e: # If we have an error committing we lose this whole block session.rollback() #pylint:disable-msg=E1101 unbranched = pkgs # Child prints a \n separated list of unbranched packages out_pipe.write('\n'.join(unbranched)) sys.exit(0)
def run(self): session.begin() recipe = Job.by_id(job.id).recipesets[0].recipes[0] assert recipe.status == TaskStatus.queued self.ready_evt.set() self.continue_evt.wait() try: beakerd.schedule_queued_recipe(recipe.id) assert False, 'should raise' except StaleTaskStatusException: pass # expected session.rollback()
def _mass_branch_worker(self, to_branch, devel_branch, pkgs, author_name): '''Main worker for mass branching. :arg to_branch: Branch to put new PackageListings on :arg devel_branch: Branch for devel, where we're branching from :arg pkgs: List of packages to branch :arg author_name: username of person making the branch :returns: List of packages which had failures while trying to branch This method forks and invokes the branching in a child. That prevents ballooning of memory when all the packages are processed in the main server process (note: separate threads don't seem to help.) ''' in_pipe, out_pipe = os.pipe() pid = os.fork() if pid: # Controller os.close(out_pipe) in_pipe = os.fdopen(in_pipe, 'r') raw_unbranched = in_pipe.read() retcode = os.waitpid(pid, 0) else: # Child os.close(in_pipe) out_pipe = os.fdopen(out_pipe, 'w') unbranched = [] for pkg in pkgs: if pkg['package_name'] in devel_branch.listings2: # clone the package from the devel branch try: devel_branch.listings2[pkg['package_name']].clone( to_branch.branchname, author_name) except InvalidRequestError: # Exceptions will be handled later. pass else: # Success, get us to the next package continue # If we get to here we had an error cloning this package unbranched.append(pkg['package_name']) # Commit the changes try: session.flush() #pylint:disable-msg=E1101 except SQLError, e: # If we have an error committing we lose this whole block session.rollback() #pylint:disable-msg=E1101 unbranched = pkgs # Child prints a \n separated list of unbranched packages out_pipe.write('\n'.join(unbranched)) sys.exit(0)
def RPC2(self, *args, **kw): params, method = xmlrpclib.loads(cherrypy.request.body.read(), use_datetime=True) start = datetime.utcnow() try: if method == "RPC2": # prevent recursion raise AssertionError("method cannot be 'RPC2'") response = self.process_rpc(method, params) response = xmlrpclib.dumps((response,), methodresponse=1, allow_none=True) except identity.IdentityFailure, e: session.rollback() response = xmlrpclib.dumps(xmlrpclib.Fault(1, "%s: %s" % (e.__class__, str(e))))
def delete_jobs(self, jobs=None, tag=None, complete_days=None, family=None, dryrun=False, product=None): """ delete_jobs will mark the job to be deleted To select jobs by id, pass an array for the *jobs* argument. Elements of the array must be strings of the form ``'J:123'``. Alternatively, pass some combination of the *tag*, *complete_days*, or *family* arguments to select jobs for deletion. These arguments behave as per the :meth:`jobs.list` method. If *dryrun* is True, deletions will be reported but nothing will be modified. Admins are not be able to delete jobs which are not owned by themselves by using the tag, complete_days etc kwargs, instead, they should do that via the *jobs* argument. """ if jobs: #Turn them into job objects if not isinstance(jobs, list): jobs = [jobs] jobs_to_try_to_del = [] for j_id in jobs: job = TaskBase.get_by_t_id(j_id) if not isinstance(job, Job): raise BeakerException('Incorrect task type passed %s' % j_id) if not job.can_delete(identity.current.user): raise BeakerException( "You don't have permission to delete job %s" % j_id) jobs_to_try_to_del.append(job) delete_jobs_kw = dict(jobs=jobs_to_try_to_del) else: # only allow people to delete their own jobs while using these kwargs delete_jobs_kw = dict( query=Job.find_jobs(tag=tag, complete_days=complete_days, family=family, product=product, owner=identity.current.user.user_name)) deleted_jobs = Job.delete_jobs(**delete_jobs_kw) msg = 'Jobs deleted' if dryrun: session.rollback() msg = 'Dryrun only. %s' % (msg) return '%s: %s' % (msg, [j.t_id for j in deleted_jobs])
def run(self): session.begin() recipeset = Job.by_id(job.id).recipesets[0] assert recipeset.status == TaskStatus.processed self.ready_evt.set() self.continue_evt.wait() try: beakerd.queue_processed_recipeset(recipeset.id) assert False, 'should raise' except StaleTaskStatusException: pass # expected session.rollback()
def update_dirty_jobs(): work_done = False dirty_jobs = Job.query.filter(Job.dirty_version != Job.clean_version) for job_id, in dirty_jobs.values(Job.id): session.begin() try: update_dirty_job(job_id) session.commit() except Exception, e: log.exception('Error in update_dirty_job(%s)', job_id) session.rollback() finally:
def RPC2(self, *args, **kw): params, method = xmlrpclib.loads(cherrypy.request.body.read(), use_datetime=True) start = datetime.utcnow() try: if method == "RPC2": # prevent recursion raise AssertionError("method cannot be 'RPC2'") response = self.process_rpc(method,params) response = xmlrpclib.dumps((response,), methodresponse=1, allow_none=True) except identity.IdentityFailure, e: session.rollback() response = xmlrpclib.dumps(xmlrpclib.Fault(1,"%s: %s" % (e.__class__, str(e))))
def update_dirty_jobs(): dirty_jobs = Job.query.filter(Job.dirty_version != Job.clean_version) if not dirty_jobs.count(): return False log.debug("Entering update_dirty_jobs") for job_id, in dirty_jobs.values(Job.id): session.begin() try: update_dirty_job(job_id) session.commit() except Exception, e: log.exception('Error in update_dirty_job(%s)', job_id) session.rollback() finally:
def provision_scheduled_recipeset(recipeset_id): recipeset = RecipeSet.by_id(recipeset_id) # Go through each recipe in the recipeSet for recipe in recipeset.recipes: log.debug('Provisioning recipe %s in RS:%s', recipe.id, recipeset_id) try: recipe.waiting() recipe.provision() except Exception as e: log.exception("Failed to provision recipeid %s", recipe.id) session.rollback() session.begin() recipe.recipeset.abort(u"Failed to provision recipeid %s, %s" % (recipe.id, e)) return
def process_new_recipes(*args): work_done = False recipes = MachineRecipe.query\ .join(MachineRecipe.recipeset).join(RecipeSet.job)\ .filter(Recipe.status == TaskStatus.new) for recipe_id, in recipes.values(MachineRecipe.id): session.begin() try: process_new_recipe(recipe_id) session.commit() except Exception, e: log.exception('Error in process_new_recipe(%s)', recipe_id) session.rollback() finally:
def schedule_pending_systems(): work_done = False systems = System.query\ .join(System.lab_controller)\ .filter(LabController.disabled == False)\ .filter(System.scheduler_status == SystemSchedulerStatus.pending) for system_id, in systems.values(System.id): session.begin() try: schedule_pending_system(system_id) session.commit() except Exception, e: log.exception('Error in schedule_pending_system(%s)', system_id) session.rollback() finally:
def queue_processed_recipesets(*args): work_done = False recipesets = RecipeSet.query.join(RecipeSet.job)\ .filter(and_(Job.dirty_version == Job.clean_version, Job.deleted == None))\ .filter(not_(RecipeSet.recipes.any( Recipe.status != TaskStatus.processed))) for rs_id, in recipesets.values(RecipeSet.id): session.begin() try: queue_processed_recipeset(rs_id) session.commit() except Exception, e: log.exception('Error in queue_processed_recipeset(%s)', rs_id) session.rollback() finally:
def RPC2(self, *args, **kw): params, method = xmlrpclib.loads(cherrypy.request.body.read(), use_datetime=True) start = datetime.utcnow() try: if method == "RPC2": # prevent recursion raise AssertionError("method cannot be 'RPC2'") response = self.process_rpc(method,params) response = xmlrpclib.dumps((response,), methodresponse=1, allow_none=True) except IdentityFailure, e: session.rollback() # IdentityFailure constructor fiddles with the response code, # so let's set it back cherrypy.response.status = 200 identity_errors = get_identity_errors() response = xmlrpclib.dumps(xmlrpclib.Fault(1,"%s: %s" % (e.__class__,','.join(identity_errors))))
def queue_processed_recipesets(*args): work_done = False recipesets = RecipeSet.query.join(RecipeSet.job)\ .filter(not_(Job.is_deleted))\ .filter(not_(RecipeSet.recipes.any( Recipe.status != TaskStatus.processed)))\ .order_by(RecipeSet.priority.desc())\ .order_by(RecipeSet.id) for rs_id, in recipesets.values(RecipeSet.id): session.begin() try: queue_processed_recipeset(rs_id) session.commit() except Exception, e: log.exception('Error in queue_processed_recipeset(%s)', rs_id) session.rollback() finally:
def RPC2(self, *args, **kw): params, method = xmlrpclib.loads(cherrypy.request.body.read(), use_datetime=True) # pylint:disable=no-member if str(method).startswith('auth.'): log.debug('Handling %s', str(method)) else: log.debug('Handling %s %s', str(method), str(params)[0:50]) start = datetime.utcnow() try: if method == "RPC2": # prevent recursion raise AssertionError("method cannot be 'RPC2'") response = self.process_rpc(method,params) response = xmlrpclib.dumps((response,), methodresponse=1, allow_none=True) session.flush() except identity.IdentityFailure, e: session.rollback() response = xmlrpclib.dumps(xmlrpclib.Fault(1,"%s: %s" % (e.__class__, str(e))))
def update_dirty_jobs(): work_done = False with session.begin(): dirty_jobs = Job.query.filter(Job.is_dirty) job_ids = [job_id for job_id, in dirty_jobs.values(Job.id)] if job_ids: log.debug('Updating dirty jobs [%s ... %s] (%d total)', job_ids[0], job_ids[-1], len(job_ids)) for job_id in job_ids: session.begin() try: update_dirty_job(job_id) session.commit() except Exception, e: log.exception('Error in update_dirty_job(%s)', job_id) session.rollback() finally:
def cleanup_visits(keep_days): from turbogears.database import session from bkr.server.model import metadata from sqlalchemy import not_, select session.begin() try: visit = metadata.tables['visit'] visit_identity = metadata.tables['visit_identity'] session.execute(visit.delete(visit.c.expiry < (datetime.datetime.now() - datetime.timedelta(days=keep_days)))) # XXX we could use CASCADE DELETE for this session.execute(visit_identity.delete( not_(visit_identity.c.visit_key.in_(select([visit.c.visit_key]))))) session.commit() except: session.rollback() raise
def __updateConnInst(self): """Update the current connector instance with current date/time to avoid duplicate task while requeueing """ try: session.begin() connector_instance = model.ConnectorInstance.query().get(self.task.connector_instance_id) instance_data = json.loads(connector_instance.instance_data) instance_data['uri'] = 'http://trigger.datasift.com/?' + datetime.now().strftime('%Y%m%d%H%M%S%f') connector_instance.instance_data = json.dumps(instance_data) session.save_or_update(connector_instance) session.commit() session.close() except: log.exception(traceback.format_exc()) log.exception(self.log_msg('exception while updating twitter streaming random URL')) session.rollback() session.close()
def queue_processed_recipesets(*args): recipesets = RecipeSet.query.join(RecipeSet.job)\ .filter(and_(Job.dirty_version == Job.clean_version, Job.deleted == None))\ .filter(not_(RecipeSet.recipes.any( Recipe.status != TaskStatus.processed))) if not recipesets.count(): return False log.debug("Entering queue_processed_recipesets") for rs_id, in recipesets.values(RecipeSet.id): session.begin() try: queue_processed_recipeset(rs_id) session.commit() except Exception, e: log.exception('Error in queue_processed_recipeset(%s)', rs_id) session.rollback() finally:
def process_new_recipes(*args): recipes = MachineRecipe.query\ .join(MachineRecipe.recipeset).join(RecipeSet.job)\ .filter(Job.dirty_version == Job.clean_version)\ .filter(Recipe.status == TaskStatus.new) if not recipes.count(): return False log.debug("Entering process_new_recipes") for recipe_id, in recipes.values(MachineRecipe.id): session.begin() try: process_new_recipe(recipe_id) session.commit() except Exception, e: log.exception('Error in process_new_recipe(%s)', recipe_id) session.rollback() finally:
def save(self, group_id=None, display_name=None, group_name=None, ldap=False, root_password=None, **kwargs): user = identity.current.user if ldap and not user.is_admin(): flash(_(u'Only admins can create LDAP groups')) redirect('mine') try: group = Group.by_id(group_id) except DatabaseLookupError: flash(_(u"Group %s does not exist." % group_id)) redirect('mine') try: Group.by_name(group_name) except NoResultFound: pass else: if group_name != group.group_name: flash( _(u'Failed to update group %s: Group name already exists: %s' % (group.group_name, group_name))) redirect('mine') if not group.can_edit(user): flash(_(u'You are not an owner of group %s' % group)) redirect('../groups/mine') try: group.set_name(user, u'WEBUI', group_name) group.set_display_name(user, u'WEBUI', display_name) group.ldap = ldap group.set_root_password(user, u'WEBUI', root_password) except BeakerException, err: session.rollback() flash(_(u'Failed to update group %s: %s' % (group.group_name, err))) redirect('.')
def doerr(self, id, dorb=0): """Test controller that raises an error. This controller will raise an exception after trying to create a new User. This new user should not be in the database at the end because the rollback should make it disapear. dorb is a flag to check that if a rollback occurs inside the controller code, the exception does not cause problems. """ u = User() u.user_id = int(id) session.add(u) session.flush() if int(dorb) == 1: session.rollback() raise Exception('test')
def schedule_pending_systems(): work_done = False with session.begin(): systems = System.query\ .join(System.lab_controller)\ .filter(LabController.disabled == False)\ .filter(System.scheduler_status == SystemSchedulerStatus.pending) system_ids = [system_id for system_id, in systems.values(System.id)] if system_ids: log.debug('Scheduling pending systems (%d total)', len(system_ids)) for system_id in system_ids: session.begin() try: schedule_pending_system(system_id) session.commit() except Exception, e: log.exception('Error in schedule_pending_system(%s)', system_id) session.rollback() finally:
def queue_processed_recipesets(*args): work_done = False with session.begin(): recipesets = RecipeSet.by_recipe_status(TaskStatus.processed)\ .order_by(RecipeSet.priority.desc())\ .order_by(RecipeSet.id) recipeset_ids = [rs_id for rs_id, in recipesets.values(RecipeSet.id)] if recipeset_ids: log.debug('Queuing processed recipe sets [%s ... %s] (%d total)', recipeset_ids[0], recipeset_ids[-1], len(recipeset_ids)) for rs_id in recipeset_ids: session.begin() try: queue_processed_recipeset(rs_id) session.commit() except Exception, e: log.exception('Error in queue_processed_recipeset(%s)', rs_id) session.rollback() finally:
def sync_tasks(self, urls_to_sync): """Syncs remote tasks to the local task library. sync_local_tasks() downloads tasks in batches and syncs them to the local task library. If the operation fails at some point any batches that have already been processed will be preserved. """ def write_data_from_url(task_url): def _write_data_from_url(f): siphon(urllib2.urlopen(task_url), f) f.flush() return _write_data_from_url urls_to_sync.sort() tasks_and_writes = [] for task_url in urls_to_sync: task_rpm_name = os.path.split(task_url)[1] tasks_and_writes.append(( task_rpm_name, write_data_from_url(task_url), )) # We section the batch processing up to allow other processes # that may be queueing for the flock to have access, and to limit # wastage of time if an error occurs total_number_of_rpms = len(tasks_and_writes) rpms_synced = 0 while rpms_synced < total_number_of_rpms: session.begin() try: tasks_and_writes_current_batch = \ tasks_and_writes[rpms_synced:rpms_synced+self.batch_size] self.tasklib.update_tasks(tasks_and_writes_current_batch) except Exception, e: session.rollback() session.close() self.logger.exception('Error syncing tasks. Got error %s' % (unicode(e))) break session.commit() self.logger.debug('Synced %s tasks' % len(tasks_and_writes_current_batch)) rpms_synced += self.batch_size
def process_new_recipes(*args): work_done = False with session.begin(): recipes = MachineRecipe.query\ .join(MachineRecipe.recipeset).join(RecipeSet.job)\ .filter(Recipe.status == TaskStatus.new) recipe_ids = [recipe_id for recipe_id, in recipes.values(MachineRecipe.id)] if recipe_ids: log.debug('Processing new recipes [%s ... %s] (%d total)', recipe_ids[0], recipe_ids[-1], len(recipe_ids)) for recipe_id in recipe_ids: session.begin() try: process_new_recipe(recipe_id) session.commit() except Exception, e: log.exception('Error in process_new_recipe(%s)', recipe_id) session.rollback() finally:
def delete_jobs(self, jobs=None, tag=None, complete_days=None, family=None, dryrun=False, product=None): """ delete_jobs will mark the job to be deleted To select jobs by id, pass an array for the *jobs* argument. Elements of the array must be strings of the form ``'J:123'``. Alternatively, pass some combination of the *tag*, *complete_days*, or *family* arguments to select jobs for deletion. These arguments behave as per the :meth:`jobs.list` method. If *dryrun* is True, deletions will be reported but nothing will be modified. Admins are not be able to delete jobs which are not owned by themselves by using the tag, complete_days etc kwargs, instead, they should do that via the *jobs* argument. """ if jobs: #Turn them into job objects if not isinstance(jobs,list): jobs = [jobs] jobs_to_try_to_del = [] for j_id in jobs: job = TaskBase.get_by_t_id(j_id) if not isinstance(job,Job): raise BeakerException('Incorrect task type passed %s' % j_id ) if not job.can_delete(identity.current.user): raise BeakerException("You don't have permission to delete job %s" % j_id) jobs_to_try_to_del.append(job) delete_jobs_kw = dict(jobs=jobs_to_try_to_del) else: # only allow people to delete their own jobs while using these kwargs delete_jobs_kw = dict(query=Job.find_jobs(tag=tag, complete_days=complete_days, family=family, product=product, owner=identity.current.user.user_name)) deleted_jobs = Job.delete_jobs(**delete_jobs_kw) msg = 'Jobs deleted' if dryrun: session.rollback() msg = 'Dryrun only. %s' % (msg) return '%s: %s' % (msg, [j.t_id for j in deleted_jobs])
def abort_dead_recipes(*args): work_done = False with session.begin(): filters = [not_(DistroTree.lab_controller_assocs.any())] if _virt_enabled(): filters.append( and_(not_(Recipe.systems.any()), Recipe.virt_status != RecipeVirtStatus.possible)) else: filters.append(not_(Recipe.systems.any())) # Following query is looking for recipes stuck in Queued state. # This may be caused by no longer valid distribution in Lab Controller # or no machines available. # However, we have to account that custom distribution can be # used and this distribution is not stored in Database at all. recipes = MachineRecipe.query\ .join(MachineRecipe.recipeset).join(RecipeSet.job)\ .filter(not_(Job.is_dirty))\ .outerjoin(Recipe.distro_tree)\ .outerjoin(Recipe.systems) \ .filter(Recipe.status == TaskStatus.queued)\ .filter(or_(DistroTree.id.isnot(None), System.status == SystemStatus.broken)) \ .filter(or_(*filters)) recipe_ids = [ recipe_id for recipe_id, in recipes.values(MachineRecipe.id) ] if recipe_ids: log.debug('Aborting dead recipes [%s ... %s] (%d total)', recipe_ids[0], recipe_ids[-1], len(recipe_ids)) for recipe_id in recipe_ids: session.begin() try: abort_dead_recipe(recipe_id) session.commit() except exceptions.Exception as e: log.exception('Error in abort_dead_recipe(%s)', recipe_id) session.rollback() finally: session.close() work_done = True return work_done
def really_cancel(self, id, msg=None): """ Confirm cancel job """ try: job = Job.by_id(id) except InvalidRequestError: flash(_(u"Invalid job id %s" % id)) redirect(".") if not job.can_cancel(identity.current.user): flash(_(u"You don't have permission to cancel job id %s" % id)) redirect(".") try: job.cancel(msg) except StaleTaskStatusException, e: log.warn(str(e)) session.rollback() flash(_(u"Could not cancel job id %s. Please try later" % id)) redirect(".")