def list_tagged(tag): koji = get_session() builds = koji.listTagged(tag) nvrs = [build['nvr'] for build in builds] print "Fetched %d builds tagged with %s" % ( len(builds), tag) return nvrs
def setUp(self): fd, self.db_filename = tempfile.mkstemp(prefix="bodhi-testing-", suffix=".db") db_path = "sqlite:///%s" % self.db_filename # The BUILD_ID environment variable is set by Jenkins and allows us to # detect if # we are running the tests in jenkins or not # https://wiki.jenkins-ci.org/display/JENKINS/Building+a+software+project#Buildingasoftwareproject-below if os.environ.get("BUILD_ID"): faitout = "http://209.132.184.152/faitout/" try: import requests req = requests.get("%s/new" % faitout) if req.status_code == 200: db_path = req.text print "Using faitout at: %s" % db_path except: pass engine = create_engine(db_path) Base.metadata.create_all(engine) self.db_factory = transactional_session_maker(engine) with self.db_factory() as session: populate(session) assert session.query(Update).count() == 1 self.koji = buildsys.get_session() self.koji.clear() # clear out our dev introspection self.msg = makemsg() self.tempdir = tempfile.mkdtemp("bodhi") self.masher = Masher(FakeHub(), db_factory=self.db_factory, mash_dir=self.tempdir)
def compare_builds(testing_build, stable_build, untag, tag): if stable_build['package_name'] == testing_build['package_name']: if rpm.labelCompare((str(testing_build['epoch']), testing_build['version'], testing_build['release']), (str(stable_build['epoch']), stable_build['version'], stable_build['release'])) < 0: print "%s is older than %s" % (testing_build['nvr'], stable_build['nvr']) try: build = PackageBuild.byNvr(testing_build['nvr']) for update in build.updates: if update.status != 'stable': print "%s not stable in bodhi!" % update.title raise SQLObjectNotFound else: pass # TODO: do the untagging? except SQLObjectNotFound: if untag: print "Untagging via koji" koji = get_session() koji.untagBuild(tag, testing_build['nvr'], force=True) else: print "Need to untag koji build %s" % testing_build['nvr']
def compare_builds(testing_build, stable_build, untag, tag): if stable_build['package_name'] == testing_build['package_name']: if rpm.labelCompare( (str(testing_build['epoch']), testing_build['version'], testing_build['release']), (str(stable_build['epoch']), stable_build['version'], stable_build['release'])) < 0: print "%s is older than %s" % (testing_build['nvr'], stable_build['nvr']) try: build = PackageBuild.byNvr(testing_build['nvr']) for update in build.updates: if update.status != 'testing': print "%s not testing in bodhi!" % update.title raise SQLObjectNotFound else: if untag: print "Obsoleting via bodhi" update.obsolete(newer=stable_build['nvr']) else: print "Need to obsolete via bodhi" except SQLObjectNotFound: if untag: print "Untagging via koji" koji = get_session() koji.untagBuild(tag, testing_build['nvr'], force=True) else: print "Need to untag koji build %s" % testing_build['nvr']
def compare_builds(testing_build, stable_build, untag, tag): if stable_build["package_name"] == testing_build["package_name"]: if ( rpm.labelCompare( (str(testing_build["epoch"]), testing_build["version"], testing_build["release"]), (str(stable_build["epoch"]), stable_build["version"], stable_build["release"]), ) < 0 ): print "%s is older than %s" % (testing_build["nvr"], stable_build["nvr"]) try: build = PackageBuild.byNvr(testing_build["nvr"]) for update in build.updates: if update.status != "testing": print "%s not testing in bodhi!" % update.title raise SQLObjectNotFound else: if untag: print "Obsoleting via bodhi" update.obsolete(newer=stable_build["nvr"]) else: print "Need to obsolete via bodhi" except SQLObjectNotFound: if untag: print "Untagging via koji" koji = get_session() koji.untagBuild(tag, testing_build["nvr"], force=True) else: print "Need to untag koji build %s" % testing_build["nvr"]
def setUp(self): fd, self.db_filename = tempfile.mkstemp(prefix='bodhi-testing-', suffix='.db') db_path = 'sqlite:///%s' % self.db_filename # The BUILD_ID environment variable is set by Jenkins and allows us to # detect if # we are running the tests in jenkins or not # https://wiki.jenkins-ci.org/display/JENKINS/Building+a+software+project#Buildingasoftwareproject-below if os.environ.get('BUILD_ID'): faitout = 'http://209.132.184.152/faitout/' try: import requests req = requests.get('%s/new' % faitout) if req.status_code == 200: db_path = req.text print 'Using faitout at: %s' % db_path except: pass engine = create_engine(db_path) Base.metadata.create_all(engine) self.db_factory = transactional_session_maker(engine) with self.db_factory() as session: populate(session) assert session.query(Update).count() == 1 self.koji = buildsys.get_session() self.koji.clear() # clear out our dev introspection self.msg = makemsg() self.tempdir = tempfile.mkdtemp('bodhi') self.masher = Masher(FakeHub(), db_factory=self.db_factory, mash_dir=self.tempdir)
def __init__(self, id, updates, repos=None, resume=False): """ Initialize a new MashTask thread. @param updates: a list of PackageUpdate objects that we want to push @param repos: a list of repositories to compose updates{,-testing) @param resume: resume this set of updates based on the assumption that we have already tagged them appropriately, and that all we need to do is mash the repositories, close bugs, and send out update notices. """ Thread.__init__(self) repos = repos and set(repos) or set() log.debug("MashTask(%d, [%d updates], %s, %s)" % (id, len(updates), repos, resume)) self.id = id self.tag = None self.updates = set() if updates and isinstance(updates[0], basestring): updates = map(PackageUpdate.byTitle, updates) for update in updates: if update.status == "obsolete": log.warning("Skipping obsolete update %s" % update.title) continue if not update.request: log.warning("Skipping update without request: %s" % update.title) continue self.updates.add(update) if not resume: update.comment( "This update is currently being pushed to the %s %s updates repository." % (update.release.long_name, update.request), author="bodhi", email=False, ) if self.updates: up = self.updates.pop() self.updates.add(up) # eg: MASHING-FEDORA, MASHING-FEDORA-EPEL self.mash_lock_id = up.release.id_prefix else: self.mash_lock_id = "UNKNOWN" self.koji = buildsys.get_session() # which repos do we want to compose? (updates|updates-testing) self.repos = repos self.success = False self.cmd = "mash -o %s -c " + config.get("mash_conf") + " -f %s " self.actions = [] # [(title, current_tag, new_tag), ...] self.mashing = False # are we currently mashing? self.moving = False # are we currently moving build tags? self.log = None # filename that we wrote mash output to self.mashed_repos = {} # { repo: mashed_dir } self.errors = [] self.safe = True self.genmd = False self.resume = resume self.testing_digest = {} self.composed_repos = [] # previously mashed repos from our MASHING lock self._lock() self._find_repos()
def __init__(self, repo, cacheduinfo=None): self.tag = get_repo_tag(repo) self.repo = repo self.doc = None self.updates = set() self.builds = {} self._from = config.get('bodhi_email') self.koji = get_session() self._create_document() self._fetch_updates() missing_ids = [] if cacheduinfo and exists(cacheduinfo): log.debug("Loading cached updateinfo.xml.gz") umd = UpdateMetadata() umd.add(cacheduinfo) existing_ids = set([up['update_id'] for up in umd.get_notices()]) # Generate metadata for any new builds for update in self.updates: if update.updateid: self.add_update(update) log.debug('Adding new update notice: %s' % update.title) if update.updateid in existing_ids: existing_ids.remove(update.updateid) else: missing_ids.append(update.title) # Add all relevant notices from the metadata to this document for notice in umd.get_notices(): if notice['update_id'] in existing_ids: log.debug("Adding existing notice: %s" % notice['title']) self._add_notice(notice) existing_ids.remove(notice['update_id']) else: if notice['type'] == 'security': log.debug("Adding existing security notice: %s" % notice['title']) self._add_notice(notice) else: log.debug("Removing %s from updateinfo" % notice['title']) else: log.debug("Generating new updateinfo.xml") for update in self.updates: if update.updateid: self.add_update(update) else: missing_ids.append(update.title) # Add *all* security updates # TODO: only the most recent #for update in PackageUpdate.select(PackageUpdate.q.type=='security'): # self.add_update(update) if missing_ids: log.error("%d updates with missing ID!" % len(missing_ids)) log.debug(missing_ids)
def test_unpush_build(self): eq_(len(self.obj.builds), 1) b = self.obj.builds[0] release = self.obj.release koji = buildsys.get_session() koji.clear() koji.__tagged__[b.nvr] = [release.testing_tag, release.pending_testing_tag] self.obj.builds[0].unpush(koji) eq_(koji.__moved__, [(u'dist-f11-updates-testing', u'dist-f11-updates-candidate', u'TurboGears-1.0.8-3.fc11')]) eq_(koji.__untag__, [(u'dist-f11-updates-testing-pending', u'TurboGears-1.0.8-3.fc11')])
def __init__(self, repo, cacheduinfo=None): self.tag = get_repo_tag(repo) self.repo = repo self.doc = None self.updates = set() self.builds = {} self._from = config.get('bodhi_email') self.koji = get_session() self._create_document() self._fetch_updates() missing_ids = [] if cacheduinfo and exists(cacheduinfo): log.debug("Loading cached updateinfo.xml.gz") umd = UpdateMetadata() umd.add(cacheduinfo) # Generate metadata for any new builds for update in self.updates: for build in update.builds: if not umd.get_notice(build.nvr): if update.updateid: self.add_update(update) break else: missing_ids.append(update.title) # Add all relevant notices from the metadata to this document ids = [ update.updateid for update in self.updates if update.updateid ] for notice in umd.get_notices(): if notice['update_id'] in ids or notice['type'] == 'security': self._add_notice(notice) else: log.debug("Removing %s from updateinfo" % notice['title']) else: log.debug("Generating new updateinfo.xml") for update in self.updates: if update.updateid: self.add_update(update) else: missing_ids.append(update.title) # Add *all* security updates # TODO: only the most recent #for update in PackageUpdate.select(PackageUpdate.q.type=='security'): # self.add_update(update) if missing_ids: log.error("%d updates with missing ID!" % len(missing_ids)) log.debug(missing_ids)
def __init__(self, release, request, db, path): self.repo = path log.debug('repo = %r' % self.repo) self.request = request if request is UpdateRequest.stable: self.tag = release.stable_tag else: self.tag = release.testing_tag self.repo_path = os.path.join(self.repo, self.tag) self.db = db self.updates = set() self.builds = {} self.missing_ids = [] self._from = config.get('bodhi_email') self.koji = get_session() self._fetch_updates() self.uinfo = cr.UpdateInfo() self.hash_type = cr.SHA256 self.comp_type = cr.XZ if release.id_prefix == u'FEDORA-EPEL': # yum on py2.4 doesn't support sha256 (#1080373) if 'el5' in self.repo or '5E' in self.repo: self.hash_type = cr.SHA1 self.comp_type = cr.GZ else: # FIXME: I'm not sure which versions of RHEL support xz metadata # compression, so use the lowest common denominator for now. self.comp_type = cr.BZ2 # Load from the cache if it exists self.cached_repodata = os.path.join(self.repo, '..', self.tag + '.repocache', 'repodata/') if os.path.isdir(self.cached_repodata): self._load_cached_updateinfo() else: log.debug("Generating new updateinfo.xml") self.uinfo = cr.UpdateInfo() for update in self.updates: if update.alias: self.add_update(update) else: self.missing_ids.append(update.title) if self.missing_ids: log.error("%d updates with missing ID: %r" % (len(self.missing_ids), self.missing_ids))
def _fetch_candidate_builds(self, pkg): """ Return all candidate builds for a given package """ matches = {} koji = get_session() koji.multicall = True for tag in [r.candidate_tag for r in Release.select()]: koji.getLatestBuilds(tag, package=pkg) results = koji.multiCall() for result in results: for entries in result: for entry in entries: matches[entry['nvr']] = entry['completion_time'] return [build[0] for build in sorted(matches.items(), key=itemgetter(1), reverse=True)]
def __init__(self, release, request, db, path): self.repo = path log.debug('repo = %r' % self.repo) self.request = request if request is UpdateRequest.stable: self.tag = release.stable_tag else: self.tag = release.testing_tag self.repo_path = os.path.join(self.repo, self.tag) self.db = db self.updates = set() self.builds = {} self.missing_ids = [] self._from = config.get('bodhi_email') self.koji = get_session() self._fetch_updates() self.uinfo = cr.UpdateInfo() self.hash_type = cr.SHA256 self.comp_type = cr.XZ if release.id_prefix == u'FEDORA-EPEL': # yum on py2.4 doesn't support sha256 (#1080373) if 'el5' in self.repo or '5E' in self.repo: self.hash_type = cr.SHA1 self.comp_type = cr.GZ else: # FIXME: I'm not sure which versions of RHEL support xz metadata # compression, so use the lowest common denominator for now. self.comp_type = cr.BZ2 # Load from the cache if it exists self.cached_repodata = os.path.join(self.repo, '..', self.tag + '.repocache', 'repodata/') if os.path.isdir(self.cached_repodata): self._load_cached_updateinfo() else: log.debug("Generating new updateinfo.xml") self.uinfo = cr.UpdateInfo() for update in self.updates: if update.alias: self.add_update(update) else: self.missing_ids.append(update.title) if self.missing_ids: log.error("%d updates with missing ID: %r" % ( len(self.missing_ids), self.missing_ids))
def test_set_request_untested_stable(self): """ Ensure that we can't submit an update for stable if it hasn't met the minimum testing requirements. """ req = DummyRequest(user=DummyUser()) req.errors = cornice.Errors() req.koji = buildsys.get_session() eq_(self.obj.status, UpdateStatus.pending) try: self.obj.set_request(self.db, UpdateRequest.stable, req.user.name) assert False except BodhiException, e: pass
def __init__(self, id, updates, repos=None, resume=False): """ Initialize a new MashTask thread. @param updates: a list of PackageUpdate objects that we want to push @param repos: a list of repositories to compose updates{,-testing) @param resume: resume this set of updates based on the assumption that we have already tagged them appropriately, and that all we need to do is mash the repositories, close bugs, and send out update notices. """ Thread.__init__(self) repos = repos and set(repos) or set() log.debug("MashTask(%d, %s, %s, %s)" % (id, updates, repos, resume)) self.id = id self.tag = None self.updates = set() if updates and isinstance(updates[0], basestring): updates = map(PackageUpdate.byTitle, updates) for update in updates: if update.status == 'obsolete': log.warning("Skipping obsolete update %s" % update.title) continue if not update.request: log.warning('Skipping update without request: %s' %update.title) continue self.updates.add(update) if self.updates: up = self.updates.pop() self.updates.add(up) # eg: MASHING-FEDORA, MASHING-FEDORA-EPEL self.mash_lock_id = up.release.id_prefix else: self.mash_lock_id = 'UNKNOWN' self.koji = buildsys.get_session() # which repos do we want to compose? (updates|updates-testing) self.repos = repos self.success = False self.cmd = 'mash -o %s -c ' + config.get('mash_conf') + ' -f %s ' self.actions = [] # [(title, current_tag, new_tag), ...] self.mashing = False # are we currently mashing? self.moving = False # are we currently moving build tags? self.log = None # filename that we wrote mash output to self.mashed_repos = {} # { repo: mashed_dir } self.errors = [] self.safe = True self.genmd = False self.resume = resume self.testing_digest = {} self.composed_repos = [] # previously mashed repos from our MASHING lock self._lock() self._find_repos()
def _fetch_candidate_builds(self, pkg): """ Return all candidate builds for a given package """ matches = {} koji = get_session() koji.multicall = True for tag in [r.candidate_tag for r in Release.select()]: koji.listTagged(tag, package=pkg) results = koji.multiCall() for result in results: for entries in result: for entry in entries: matches[entry['nvr']] = entry['completion_time'] return [build[0] for build in sorted(matches.items(), key=itemgetter(1), reverse=True)]
def test_unpush_build(self): eq_(len(self.obj.builds), 1) b = self.obj.builds[0] release = self.obj.release koji = buildsys.get_session() koji.clear() koji.__tagged__[b.nvr] = [ release.testing_tag, release.pending_testing_tag ] self.obj.builds[0].unpush(koji) eq_(koji.__moved__, [(u'dist-f11-updates-testing', u'dist-f11-updates-candidate', u'TurboGears-1.0.8-3.fc11')]) eq_(koji.__untag__, [ (u'dist-f11-updates-testing-pending', u'TurboGears-1.0.8-3.fc11') ])
def clean_stable_builds(untag=False): koji = get_session() for release in Release.select(): latest_stable_builds = koji.listTagged(release.stable_tag, latest=True) latest_stable_nvrs = [build['nvr'] for build in latest_stable_builds] print "Fetched %d latest stable builds tagged with %s" % ( len(latest_stable_builds), release.stable_tag) stable_builds = koji.listTagged(release.stable_tag) stable_nvrs = [build['nvr'] for build in stable_builds] print "Fetched %d stable builds tagged with %s" % (len(stable_builds), release.stable_tag) for latest_build in latest_stable_builds: for build in stable_builds: if build['nvr'] == latest_build['nvr']: continue compare_builds(latest_build, build, untag, release.stable_tag)
def clean_stable_builds(untag=False): koji = get_session() for release in Release.select(): latest_stable_builds = koji.listTagged(release.stable_tag, latest=True) latest_stable_nvrs = [build['nvr'] for build in latest_stable_builds] print "Fetched %d latest stable builds tagged with %s" % ( len(latest_stable_builds), release.stable_tag) stable_builds = koji.listTagged(release.stable_tag) stable_nvrs = [build['nvr'] for build in stable_builds] print "Fetched %d stable builds tagged with %s" % ( len(stable_builds), release.stable_tag) for latest_build in latest_stable_builds: for build in stable_builds: if build['nvr'] == latest_build['nvr']: continue compare_builds(latest_build, build, untag, release.stable_tag)
def clean_testing_builds(untag=False): koji = get_session() for release in Release.select(): stable_builds = koji.listTagged(release.stable_tag, latest=True) stable_nvrs = [build["nvr"] for build in stable_builds] print "Fetched %d builds tagged with %s" % (len(stable_builds), release.stable_tag) testing_builds = koji.listTagged(release.testing_tag, latest=True) print "Fetched %d builds tagged with %s" % (len(testing_builds), release.testing_tag) testing_nvrs = [build["nvr"] for build in testing_builds] for testing_build in testing_builds: for build in testing_builds: compare_builds(testing_build, build, untag, release.testing_tag) for build in stable_builds: compare_builds(testing_build, build, untag, release.testing_tag) # Find testing updates that aren't in the list of latest builds for update in PackageUpdate.select( AND( PackageUpdate.q.releaseID == release.id, PackageUpdate.q.status == "testing", PackageUpdate.q.request == None, ) ): for build in update.builds: if build.nvr not in testing_nvrs: latest_testing = None latest_stable = None for testing in testing_nvrs: if testing.startswith(build.package.name + "-"): latest_testing = testing break for stable in stable_nvrs: if stable.startswith(build.package.name + "-"): latest_stable = stable break if latest_testing: koji_build = koji.getBuild(build.nvr) latest_build = koji.getBuild(latest_testing) if rpm.labelCompare(build_evr(koji_build), build_evr(latest_build)) < 0: print "%s in testing, latest_testing = %s, latest_stable = %s" % ( update.title, latest_testing, latest_stable, ) if untag: print "Obsoleting %s" % update.title update.obsolete(newer=latest_testing)
def save(self, builds, notes, expiration=None, *args, **kw): log.debug(repr(locals())) last_release = None # for our koji wait-repo example if expiration: if datetime.utcnow() > expiration: flash('Cannot set an expiration in the past') if request_format() == 'json': return dict() raise redirect('/override/new') try: koji = get_session() except Exception, e: flash('Unable to connect to Koji') if request_format() == 'json': return dict() raise redirect('/override/new')
def clean_testing_builds(untag=False): koji = get_session() for release in Release.select(): stable_builds = koji.listTagged(release.stable_tag, latest=True) stable_nvrs = [build['nvr'] for build in stable_builds] print "Fetched %d builds tagged with %s" % (len(stable_builds), release.stable_tag) testing_builds = koji.listTagged(release.testing_tag, latest=True) print "Fetched %d builds tagged with %s" % (len(testing_builds), release.testing_tag) testing_nvrs = [build['nvr'] for build in testing_builds] for testing_build in testing_builds: for build in testing_builds: compare_builds(testing_build, build, untag, release.testing_tag) for build in stable_builds: compare_builds(testing_build, build, untag, release.testing_tag) # Find testing updates that aren't in the list of latest builds for update in PackageUpdate.select( AND(PackageUpdate.q.releaseID == release.id, PackageUpdate.q.status == 'testing', PackageUpdate.q.request == None)): for build in update.builds: if build.nvr not in testing_nvrs: latest_testing = None latest_stable = None for testing in testing_nvrs: if testing.startswith(build.package.name + '-'): latest_testing = testing break for stable in stable_nvrs: if stable.startswith(build.package.name + '-'): latest_stable = stable break if latest_testing: koji_build = koji.getBuild(build.nvr) latest_build = koji.getBuild(latest_testing) if rpm.labelCompare(build_evr(koji_build), build_evr(latest_build)) < 0: print "%s in testing, latest_testing = %s, latest_stable = %s" % ( update.title, latest_testing, latest_stable) if untag: print "Obsoleting %s" % update.title update.obsolete(newer=latest_testing)
def main(argv=sys.argv): if len(argv) != 2: usage(argv) config_uri = argv[1] setup_logging(config_uri) log = logging.getLogger(__name__) settings = get_appsettings(config_uri) engine = engine_from_config(settings, 'sqlalchemy.') Session = scoped_session( sessionmaker(extension=ZopeTransactionExtension())) Session.configure(bind=engine) db = Session() koji = buildsys.get_session() one_day = timedelta(days=1) now = datetime.utcnow() with transaction.manager: for release in db.query(Release).filter_by( state=ReleaseState.pending).all(): log.info(release.name) for update in db.query(Update).filter_by( release=release, status=UpdateStatus.stable).all(): assert update.date_stable, update.title if now - update.date_stable > one_day: for build in update.builds: tags = build.get_tags() stable_tag = release.dist_tag testing_tag = release.testing_tag pending_testing_tag = Release.pending_testing_tag if stable_tag not in tags: log.error('%s not tagged as stable %s' % (build.nvr, tags)) continue if testing_tag in tags: log.info('Removing %s from %s' % (testing_tag, build.nvr)) koji.untagBuild(testing_tag, build.nvr) if pending_testing_tag in tags: log.info('Removing %s from %s' % (pending_testing_tag, build.nvr)) koji.untagBuild(pending_testing_tag, build.nvr)
def test_set_request_stable_after_week_in_testing(self, publish): req = DummyRequest() req.errors = cornice.Errors() req.koji = buildsys.get_session() req.user = model.User(name='bob') self.obj.status = UpdateStatus.testing self.obj.request = None # Pretend it's been in testing for a week self.obj.comment( self.db, u'This update has been pushed to testing.', author=u'bodhi') self.obj.date_testing = self.obj.comments[-1].timestamp - timedelta(days=7) eq_(self.obj.days_in_testing, 7) eq_(self.obj.meets_testing_requirements, True) self.obj.set_request(self.db, UpdateRequest.stable, req) eq_(self.obj.request, UpdateRequest.stable) eq_(len(req.errors), 0) publish.assert_called_once_with( topic='update.request.stable', msg=mock.ANY)
def main(argv=sys.argv): if len(argv) != 2: usage(argv) config_uri = argv[1] setup_logging(config_uri) log = logging.getLogger(__name__) settings = get_appsettings(config_uri) engine = engine_from_config(settings, 'sqlalchemy.') Session = scoped_session(sessionmaker(extension=ZopeTransactionExtension())) Session.configure(bind=engine) db = Session() koji = buildsys.get_session() one_day = timedelta(days=1) now = datetime.utcnow() with transaction.manager: for release in db.query(Release).filter_by( state=ReleaseState.pending).all(): log.info(release.name) for update in db.query(Update).filter_by( release=release, status=UpdateStatus.stable).all(): assert update.date_stable, update.title if now - update.date_stable > one_day: for build in update.builds: tags = build.get_tags() stable_tag = release.dist_tag testing_tag = release.testing_tag pending_testing_tag = Release.pending_testing_tag if stable_tag not in tags: log.error('%s not tagged as stable %s' % (build.nvr, tags)) continue if testing_tag in tags: log.info('Removing %s from %s' % (testing_tag, build.nvr)) koji.untagBuild(testing_tag, build.nvr) if pending_testing_tag in tags: log.info('Removing %s from %s' % (pending_testing_tag, build.nvr)) koji.untagBuild(pending_testing_tag, build.nvr)
def test_set_meets_then_met_requirements(self, publish): req = DummyRequest() req.errors = cornice.Errors() req.koji = buildsys.get_session() req.user = model.User(name='bob') self.obj.status = UpdateStatus.testing self.obj.request = None # Pretend it's been in testing for a week self.obj.comment( self.db, u'This update has been pushed to testing.', author=u'bodhi') self.obj.date_testing = self.obj.comments[-1].timestamp - timedelta(days=7) eq_(self.obj.days_in_testing, 7) eq_(self.obj.meets_testing_requirements, True) eq_(self.obj.met_testing_requirements, False) text = config.get('testing_approval_msg') % self.obj.days_in_testing self.obj.comment(self.db, text, author=u'bodhi') eq_(self.obj.meets_testing_requirements, True) eq_(self.obj.met_testing_requirements, True)
def test_set_request_stable_after_week_in_testing(self, publish): req = DummyRequest() req.errors = cornice.Errors() req.koji = buildsys.get_session() req.user = model.User(name='bob') self.obj.status = UpdateStatus.testing self.obj.request = None # Pretend it's been in testing for a week self.obj.comment(self.db, u'This update has been pushed to testing.', author=u'bodhi') self.obj.date_testing = self.obj.comments[-1].timestamp - timedelta( days=7) eq_(self.obj.days_in_testing, 7) eq_(self.obj.meets_testing_requirements, True) self.obj.set_request(self.db, UpdateRequest.stable, req) eq_(self.obj.request, UpdateRequest.stable) eq_(len(req.errors), 0) publish.assert_called_once_with(topic='update.request.stable', msg=mock.ANY)
def clean_pending_tags(): """ Clean up any stray pending tags """ koji = get_session() for release in Release.select(): log.info("Finding all stray pending-testing builds...") if release.name.startswith('E'): continue tag = release.pending_testing_tag tagged = [build['nvr'] for build in koji.listTagged(tag)] for nvr in tagged: try: build = PackageBuild.byNvr(nvr) for update in build.updates: if update.status in ('testing', 'stable', 'obsolete'): log.info("%s %s" % (nvr, update.status)) log.info("Untagging %s" % nvr) koji.untagBuild(tag, nvr, force=True) except SQLObjectNotFound: log.info("Can't find build for %s" % nvr) log.info("Untagging %s" % nvr) koji.untagBuild(tag, nvr, force=True) log.info("Finding all stray pending-stable builds...") tag = release.pending_stable_tag tagged = [build['nvr'] for build in koji.listTagged(tag)] for nvr in tagged: try: build = PackageBuild.byNvr(nvr) for update in build.updates: if update.status in ('pending', 'obsolete', 'stable'): log.info("%s %s" % (nvr, update.status)) log.info("Untagging %s" % nvr) koji.untagBuild(tag, nvr, force=True) except SQLObjectNotFound: log.info("Untagging %s" % nvr) koji.untagBuild(tag, nvr, force=True)
def clean_pending_tags(): """ Clean up any stray pending tags """ koji = get_session() for release in Release.select(): log.info("Finding all stray pending-testing builds...") if release.name.startswith('EL'): continue tag = release.pending_testing_tag tagged = [build['nvr'] for build in koji.listTagged(tag)] for nvr in tagged: try: build = PackageBuild.byNvr(nvr) for update in build.updates: if update.status in ('testing', 'stable', 'obsolete'): log.info("%s %s" % (nvr, update.status)) log.info("Untagging %s" % nvr) koji.untagBuild(tag, nvr, force=True) except SQLObjectNotFound: log.info("Can't find build for %s" % nvr) log.info("Untagging %s" % nvr) koji.untagBuild(tag, nvr, force=True) log.info("Finding all stray pending-stable builds...") tag = release.pending_stable_tag tagged = [build['nvr'] for build in koji.listTagged(tag)] for nvr in tagged: try: build = PackageBuild.byNvr(nvr) for update in build.updates: if update.status in ('pending', 'obsolete', 'stable'): log.info("%s %s" % (nvr, update.status)) log.info("Untagging %s" % nvr) koji.untagBuild(tag, nvr, force=True) except SQLObjectNotFound: log.info("Untagging %s" % nvr) koji.untagBuild(tag, nvr, force=True)
def test_set_meets_then_met_requirements(self, publish): req = DummyRequest() req.errors = cornice.Errors() req.koji = buildsys.get_session() req.user = model.User(name='bob') self.obj.status = UpdateStatus.testing self.obj.request = None # Pretend it's been in testing for a week self.obj.comment(self.db, u'This update has been pushed to testing.', author=u'bodhi') self.obj.date_testing = self.obj.comments[-1].timestamp - timedelta( days=7) eq_(self.obj.days_in_testing, 7) eq_(self.obj.meets_testing_requirements, True) eq_(self.obj.met_testing_requirements, False) text = config.get('testing_approval_msg') % self.obj.days_in_testing self.obj.comment(self.db, text, author=u'bodhi') eq_(self.obj.meets_testing_requirements, True) eq_(self.obj.met_testing_requirements, True)
def test_extended_metadata_updating_with_edited_updates(self): testutil.capture_log(['bodhi.metadata']) # grab the name of a build in updates-testing, and create it in our db koji = get_session() builds = koji.listTagged('dist-f13-updates-testing', latest=True) # Create all of the necessary database entries release = Release(name='F13', long_name='Fedora 13', id_prefix='FEDORA', dist_tag='dist-f13') package = Package(name=builds[0]['package_name']) update = PackageUpdate(title=builds[0]['nvr'], release=release, submitter=builds[0]['owner_name'], status='testing', notes='foobar', type='bugfix') build = PackageBuild(nvr=builds[0]['nvr'], package=package) update.addPackageBuild(build) update.assign_id() ## Initialize our temporary repo temprepo = join(tempfile.mkdtemp('bodhi'), 'f13-updates-testing') print "Inserting updateinfo into temprepo: %s" % temprepo mkmetadatadir(join(temprepo, 'i386')) repodata = join(temprepo, 'i386', 'repodata') assert exists(join(repodata, 'repomd.xml')) ## Generate the XML md = ExtendedMetadata(temprepo) ## Insert the updateinfo.xml into the repository md.insert_updateinfo() updateinfo = self.__verify_updateinfo(repodata) ## Read an verify the updateinfo.xml.gz uinfo = UpdateMetadata() uinfo.add(updateinfo) notice = uinfo.get_notice(('mutt', '1.5.14', '1.fc13')) assert not notice notice = uinfo.get_notice(get_nvr(update.title)) assert notice assert notice['status'] == update.status assert notice['updated'] == update.date_modified assert notice['from'] == str(config.get('bodhi_email')) assert notice['description'] == update.notes assert notice['issued'] is not None assert notice['update_id'] == update.updateid assert notice['title'] == update.title assert notice['release'] == update.release.long_name ## Edit the update and bump the build revision nvr = 'TurboGears-1.0.2.2-3.fc7' newbuild = PackageBuild(nvr=nvr, package=package) update.removePackageBuild(build) update.addPackageBuild(newbuild) update.title = nvr update.date_modified = datetime.utcnow() # Pretend -2 was unpushed assert len(koji.__untag__) == 0 koji.__untag__.append('TurboGears-1.0.2.2-2.fc7') ## Test out updateinfo.xml updating via our ExtendedMetadata md = ExtendedMetadata(temprepo, updateinfo) md.insert_updateinfo() updateinfo = self.__verify_updateinfo(repodata) ## Read an verify the updateinfo.xml.gz uinfo = UpdateMetadata() uinfo.add(updateinfo) print(testutil.get_log()) notice = uinfo.get_notice(('TurboGears', '1.0.2.2', '2.fc7')) assert not notice, "Old TG notice did not get pruned: %s" % notice notice = uinfo.get_notice(('TurboGears', '1.0.2.2', '3.fc7')) assert notice, uinfo assert notice['title'] == update.title num_notices = len(uinfo.get_notices()) assert num_notices == 1, num_notices ## Clean up shutil.rmtree(temprepo) del(koji.__untag__[:])
def tearDown(self): self.db.close() get_session().clear() shutil.rmtree(self.tempdir)
from turbogears import expose, controllers, identity, config, flash from bodhi.widgets import NewUpdateForm from bodhi.buildsys import get_session from bodhi.model import Release from bodhi.util import url log = logging.getLogger(__name__) update_form = NewUpdateForm() packages = [] # Load the list of packages from koji if config.get('buildsystem') == 'koji': try: koji = get_session() packages = [pkg['package_name'] for pkg in koji.listPackages()] log.debug("%d packages loaded from koji" % len(packages)) except Exception, e: log.exception(e) log.error("There was a problem loading the package list from koji") else: # Resort to looking on the filesystem for the package names try: packages = os.listdir(config.get('build_dir')) except (OSError, TypeError): log.warning("Warning: build_dir either invalid or not set in app.cfg") class NewUpdateController(controllers.Controller):
def test_session(self): from bodhi.buildsys import get_session session = get_session() assert session
def test_extended_metadata_updating_with_old_stable_security(self): testutil.capture_log(['bodhi.metadata']) koji = get_session() del (koji.__untag__[:]) assert not koji.__untag__, koji.__untag__ builds = koji.listTagged('dist-f7-updates', latest=True) # Create all of the necessary database entries release = Release(name='F17', long_name='Fedora 17', id_prefix='FEDORA', dist_tag='dist-f17') package = Package(name='TurboGears') update = PackageUpdate(title='TurboGears-1.0.2.2-2.fc7', release=release, submitter=builds[0]['owner_name'], status='stable', notes='foobar', type='security') build = PackageBuild(nvr='TurboGears-1.0.2.2-2.fc7', package=package) update.addPackageBuild(build) update.assign_id() assert update.updateid ## Initialize our temporary repo temprepo = join(tempfile.mkdtemp('bodhi'), 'f7-updates') print "Inserting updateinfo into temprepo: %s" % temprepo mkmetadatadir(join(temprepo, 'i386')) repodata = join(temprepo, 'i386', 'repodata') assert exists(join(repodata, 'repomd.xml')) ## Generate the XML md = ExtendedMetadata(temprepo) ## Insert the updateinfo.xml into the repository md.insert_updateinfo() updateinfo = self.__verify_updateinfo(repodata) # Create a new non-security update for the same package newbuild = 'TurboGears-1.0.2.2-3.fc7' update = PackageUpdate(title=newbuild, release=release, submitter=builds[0]['owner_name'], status='stable', notes='foobar', type='enhancement') build = PackageBuild(nvr=newbuild, package=package) update.addPackageBuild(build) update.assign_id() koji.__untag__.append('TurboGears-1.0.2.2-2.fc7') ## Test out updateinfo.xml updating via our ExtendedMetadata md = ExtendedMetadata(temprepo, updateinfo) md.insert_updateinfo() updateinfo = self.__verify_updateinfo(repodata) ## Read an verify the updateinfo.xml.gz uinfo = UpdateMetadata() uinfo.add(updateinfo) print(testutil.get_log()) assert len(uinfo.get_notices()) == 2, len(uinfo.get_notices()) assert uinfo.get_notice(get_nvr('TurboGears-1.0.2.2-2.fc7')) notice = uinfo.get_notice(get_nvr(update.title)) assert notice, 'Cannot find update for %r' % get_nvr(update.title) assert notice['status'] == update.status assert notice['updated'] == update.date_modified assert notice['from'] == str(config.get('bodhi_email')) assert notice['description'] == update.notes assert notice['issued'] is not None assert notice['update_id'] == update.updateid ## Clean up shutil.rmtree(temprepo) del (koji.__untag__[:])
def work(self): self.koji = buildsys.get_session() self.release = self.db.query(Release)\ .filter_by(name=self.release).one() self.id = getattr(self.release, '%s_tag' % self.request.value) # Set our thread's "name" so it shows up nicely in the logs. # https://docs.python.org/2/library/threading.html#thread-objects self.name = self.id # For 'pending' branched releases, we only want to perform repo-related # tasks for testing updates. For stable updates, we should just add the # dist_tag and do everything else other than mashing/updateinfo, since # the nightly build-branched cron job mashes for us. self.skip_mash = False if (self.release.state is ReleaseState.pending and self.request is UpdateRequest.stable): self.skip_mash = True self.log.info('Running MasherThread(%s)' % self.id) self.init_state() if not self.resume: self.init_path() notifications.publish( topic="mashtask.mashing", msg=dict(repo=self.id, updates=self.state['updates']), force=True, ) try: if self.resume: self.load_state() else: self.save_state() self.load_updates() self.verify_updates() if self.request is UpdateRequest.stable: self.perform_gating() self.determine_and_perform_tag_actions() self.update_security_bugs() self.expire_buildroot_overrides() self.remove_pending_tags() self.update_comps() if self.resume and self.path in self.state['completed_repos']: self.log.info('Skipping completed repo: %s', self.path) self.complete_requests() # We still need to generate the testing digest, since it's stored in memory self.generate_testing_digest() else: if not self.skip_mash: mash_thread = self.mash() # Things we can do while we're mashing self.complete_requests() self.generate_testing_digest() if not self.skip_mash: uinfo = self.generate_updateinfo() self.wait_for_mash(mash_thread) uinfo.insert_updateinfo() uinfo.insert_pkgtags() uinfo.cache_repodata() # Compose OSTrees from our freshly mashed repos if config.get('compose_atomic_trees'): self.compose_atomic_trees() if not self.skip_mash: self.sanity_check_repo() self.stage_repo() # Wait for the repo to hit the master mirror self.wait_for_sync() # Send fedmsg notifications self.send_notifications() # Update bugzillas self.modify_bugs() # Add comments to updates self.status_comments() # Announce stable updates to the mailing list self.send_stable_announcements() # Email updates-testing digest self.send_testing_digest() self.success = True self.remove_state() self.unlock_updates() self.check_all_karma_thresholds() except: self.log.exception('Exception in MasherThread(%s)' % self.id) self.save_state() raise finally: self.finish(self.success)
def __init__(self, repo, cacheduinfo=None): self.tag = get_repo_tag(repo) self.repo = repo self.doc = None self.updates = set() self.builds = {} self._from = config.get('bodhi_email') self.koji = get_session() self._create_document() self._fetch_updates() missing_ids = [] if cacheduinfo and exists(cacheduinfo): log.debug("Loading cached updateinfo.xml.gz") umd = UpdateMetadata() umd.add(cacheduinfo) # Drop the old cached updateinfo.xml.gz, it's unneeded now os.unlink(cacheduinfo) existing_ids = set([up['update_id'] for up in umd.get_notices()]) seen_ids = set() from_cache = set() # Generate metadata for any new builds for update in self.updates: if update.updateid: seen_ids.add(update.updateid) if update.updateid in existing_ids: notice = umd.get_notice(update.title) if not notice: log.warn( '%s ID in cache but notice cannot be found' % (update.title)) self.add_update(update) continue if notice['updated']: if datetime.strptime(notice['updated'], '%Y-%m-%d %H:%M:%S' ) < update.date_modified: log.debug( 'Update modified, generating new notice: %s' % update.title) self.add_update(update) else: log.debug('Loading updated %s from cache' % update.title) from_cache.add(update.updateid) elif update.date_modified: log.debug( 'Update modified, generating new notice: %s' % update.title) self.add_update(update) else: log.debug('Loading %s from cache' % update.title) from_cache.add(update.updateid) else: log.debug('Adding new update notice: %s' % update.title) self.add_update(update) else: missing_ids.append(update.title) # Add all relevant notices from the cache to this document for notice in umd.get_notices(): if notice['update_id'] in from_cache: log.debug("Keeping existing notice: %s" % notice['title']) self._add_notice(notice) else: # Keep all security notices in the stable repo if 'testing' not in self.repo: if notice['type'] == 'security': if notice['update_id'] not in seen_ids: log.debug( "Keeping existing security notice: %s" % notice['title']) self._add_notice(notice) else: log.debug('%s already added?' % notice['title']) else: log.debug('Purging cached stable notice %s' % notice['title']) else: log.debug('Purging cached testing update %s' % notice['title']) # Clean metadata generation else: log.debug("Generating new updateinfo.xml") for update in self.updates: if update.updateid: self.add_update(update) else: missing_ids.append(update.title) # Add *all* security updates # TODO: only the most recent #for update in PackageUpdate.select(PackageUpdate.q.type=='security'): # self.add_update(update) if missing_ids: log.error("%d updates with missing ID!" % len(missing_ids)) log.debug(missing_ids)
def main(): load_config() __connection__ = hub = PackageHub("bodhi") koji = get_session() tasks = [] broke = set() # Clean up any stray pending tags for release in Release.select(): print "Finding all pending-testing builds..." if release.name.startswith('EL'): continue tag = release.pending_testing_tag tagged = [build['nvr'] for build in koji.listTagged(tag)] for nvr in tagged: try: build = PackageBuild.byNvr(nvr) for update in build.updates: if update.status in ('testing', 'stable', 'obsolete'): print "%s %s" % (nvr, update.status) if '--fix' in sys.argv: print "Untagging %s" % nvr koji.untagBuild(tag, nvr, force=True) except SQLObjectNotFound: print "Can't find build for %s" % nvr if '--fix' in sys.argv: print "Untagging %s" % nvr koji.untagBuild(tag, nvr, force=True) tag = release.pending_stable_tag tagged = [build['nvr'] for build in koji.listTagged(tag)] for nvr in tagged: try: build = PackageBuild.byNvr(nvr) for update in build.updates: if update.status in ('pending', 'obsolete'): print "%s %s" % (nvr, update.status) if '--fix' in sys.argv: print "Untagging %s" % nvr koji.untagBuild(tag, nvr, force=True) except SQLObjectNotFound: print "Can't find build for %s" % nvr if '--fix' in sys.argv: print "Untagging %s" % nvr koji.untagBuild(tag, nvr, force=True) # Check for testing updates that aren't tagged properly for update in PackageUpdate.select(PackageUpdate.q.status=='testing'): dest_tag = update.release.testing_tag for build in update.builds: tags = [tag['name'] for tag in koji.listTags(build=build.nvr)] if dest_tag not in tags: print "%s marked as testing, but tagged with %s" % (build.nvr, tags) if '--fix' in sys.argv: broke.add((tags[0], dest_tag, build.nvr)) # Check all candidate updates to see if they are in a different bodhi state for release in Release.select(): tag = release.candidate_tag tagged = [build['nvr'] for build in koji.listTagged(tag, latest=True)] for nvr in tagged: try: build = PackageBuild.byNvr(nvr) for update in build.updates: if update.status in ('testing', 'stable'): print "%s %s but tagged as %s" % (nvr, update.status, tag) if '--fix' in sys.argv: dest = release.testing_tag if update.status == 'stable': dest = release.stable_tag elif update.status == 'obsolete': dest = release.candidate_tag broke.add((tag, dest, nvr)) except SQLObjectNotFound: pass # Make sure that all builds in koji tagged as an update exist # in bodhi, and are in the expect state. for release in Release.select(): for tag in (release.testing_tag, release.stable_tag): tagged = [build['nvr'] for build in koji.listTagged(tag, latest=True)] for nvr in tagged: try: build = PackageBuild.byNvr(nvr) except SQLObjectNotFound: print "PackageUpdate(%s) not found!" % nvr continue if not len(build.updates): print "PackageBuild(%s) has no updates" % (build.nvr) status = 'testing' in tag and 'testing' or 'stable' for update in build.updates: if update.status != status: print "%s is %s in bodhi but tagged as %s in koji" % ( update.title, update.status, tag) if '--fix' in sys.argv: dest = release.testing_tag if update.status == 'stable': dest = release.stable_tag elif update.status == 'obsolete': dest = release.candidate_tag for b in update.builds: broke.add((tag, dest, b.nvr)) if broke: print " ** Fixing broken tags! **" koji.multicall = True for tag, dest, build in broke: print "Moving %s from %s to %s" % (build, tag, dest) koji.moveBuild(tag, dest, build, force=True) print "Running koji.multiCall()" results = koji.multiCall() success = False print "Waiting for tasks" bad_tasks = wait_for_tasks([task[0] for task in results]) if bad_tasks == 0: success = True if success: print "Tags successfully moved!" else: print "Error moving tags!" print "bad_tasks = %r" % bad_tasks
def list_tagged(tag): koji = get_session() builds = koji.listTagged(tag) nvrs = [build['nvr'] for build in builds] print "Fetched %d builds tagged with %s" % (len(builds), tag) return nvrs
def __init__(self, repo, cacheduinfo=None): self.tag = get_repo_tag(repo) self.repo = repo self.doc = None self.updates = set() self.builds = {} self._from = config.get('bodhi_email') self.koji = get_session() self._create_document() self._fetch_updates() missing_ids = [] if cacheduinfo and exists(cacheduinfo): log.debug("Loading cached updateinfo.xml.gz") umd = UpdateMetadata() umd.add(cacheduinfo) existing_ids = set([up['update_id'] for up in umd.get_notices()]) seen_ids = set() from_cache = set() # Generate metadata for any new builds for update in self.updates: if update.updateid: seen_ids.add(update.updateid) if update.updateid in existing_ids: notice = umd.get_notice(update.title) if not notice: log.warn('%s ID in cache but notice cannot be found' % (update.title)) self.add_update(update) continue if notice['updated']: if datetime.strptime(notice['updated'], '%Y-%m-%d %H:%M:%S') < update.date_modified: log.debug('Update modified, generating new notice: %s' % update.title) self.add_update(update) else: log.debug('Loading updated %s from cache' % update.title) from_cache.append(update.updateid) elif update.date_modified: log.debug('Update modified, generating new notice: %s' % update.title) self.add_update(update) else: log.debug('Loading %s from cache' % update.title) from_cache.add(update.updateid) else: log.debug('Adding new update notice: %s' % update.title) self.add_update(update) else: missing_ids.append(update.title) # Add all relevant notices from the cache to this document for notice in umd.get_notices(): if notice['update_id'] in from_cache: log.debug("Keeping existing notice: %s" % notice['title']) self._add_notice(notice) else: # Keep all security notices in the stable repo if 'testing' not in self.repo: if notice['type'] == 'security': if notice['update_id'] not in seen_ids: log.debug("Keeping existing security notice: %s" % notice['title']) self._add_notice(notice) else: log.debug('%s already added?' % notice['title']) else: log.debug('Purging cached stable notice %s' % notice['title']) else: log.debug('Purging cached testing update %s' % notice['title']) # Clean metadata generation else: log.debug("Generating new updateinfo.xml") for update in self.updates: if update.updateid: self.add_update(update) else: missing_ids.append(update.title) # Add *all* security updates # TODO: only the most recent #for update in PackageUpdate.select(PackageUpdate.q.type=='security'): # self.add_update(update) if missing_ids: log.error("%d updates with missing ID!" % len(missing_ids)) log.debug(missing_ids)
def test_extended_metadata(self): # grab the name of a build in updates-testing, and create it in our db koji = get_session() builds = koji.listTagged('dist-f13-updates-testing', latest=True) # Create all of the necessary database entries release = Release(name='F13', long_name='Fedora 13', id_prefix='FEDORA', dist_tag='dist-f13') package = Package(name=builds[0]['package_name']) update = PackageUpdate(title=builds[0]['nvr'], release=release, submitter=builds[0]['owner_name'], status='testing', notes='foobar', type='bugfix') build = PackageBuild(nvr=builds[0]['nvr'], package=package) update.addPackageBuild(build) bug = Bugzilla(bz_id=1) update.addBugzilla(bug) cve = CVE(cve_id="CVE-2007-0000") update.addCVE(cve) update.assign_id() print update ## Initialize our temporary repo temprepo = join(tempfile.mkdtemp('bodhi'), 'f13-updates-testing') print "Inserting updateinfo into temprepo: %s" % temprepo mkmetadatadir(join(temprepo, 'i386')) repodata = join(temprepo, 'i386', 'repodata') assert exists(join(repodata, 'repomd.xml')) ## Generate the XML md = ExtendedMetadata(temprepo) ## Insert the updateinfo.xml into the repository md.insert_updateinfo() updateinfo = self.__verify_updateinfo(repodata) ## Read an verify the updateinfo.xml.gz uinfo = UpdateMetadata() uinfo.add(updateinfo) notice = uinfo.get_notice(('mutt', '1.5.14', '1.fc13')) assert not notice notice = uinfo.get_notice(get_nvr(update.title)) assert notice assert notice['status'] == update.status assert notice['updated'] == update.date_modified assert notice['from'] == str(config.get('bodhi_email')) assert notice['description'] == update.notes assert notice['issued'] is not None assert notice['update_id'] == update.updateid assert notice['epoch'] is None cve = notice['references'][0] assert cve['type'] == 'cve' assert cve['href'] == update.cves[0].get_url() assert cve['id'] == update.cves[0].cve_id bug = notice['references'][1] assert bug['href'] == update.bugs[0].get_url() assert bug['id'] == '1' assert bug['type'] == 'bugzilla' # FC6's yum update metadata parser doesn't know about some stuff from yum import __version__ if __version__ >= '3.0.6': assert notice['title'] == update.title assert notice['release'] == update.release.long_name assert cve['title'] is None ## Clean up shutil.rmtree(temprepo)
def test_extended_metadata_updating_with_old_stable_security(self): testutil.capture_log(['bodhi.metadata']) koji = get_session() del(koji.__untag__[:]) assert not koji.__untag__, koji.__untag__ builds = koji.listTagged('dist-f7-updates', latest=True) # Create all of the necessary database entries release = Release(name='F17', long_name='Fedora 17', id_prefix='FEDORA', dist_tag='dist-f17') package = Package(name='TurboGears') update = PackageUpdate(title='TurboGears-1.0.2.2-2.fc7', release=release, submitter=builds[0]['owner_name'], status='stable', notes='foobar', type='security') build = PackageBuild(nvr='TurboGears-1.0.2.2-2.fc7', package=package) update.addPackageBuild(build) update.assign_id() assert update.updateid ## Initialize our temporary repo temprepo = join(tempfile.mkdtemp('bodhi'), 'f7-updates') print "Inserting updateinfo into temprepo: %s" % temprepo mkmetadatadir(join(temprepo, 'i386')) repodata = join(temprepo, 'i386', 'repodata') assert exists(join(repodata, 'repomd.xml')) ## Generate the XML md = ExtendedMetadata(temprepo) ## Insert the updateinfo.xml into the repository md.insert_updateinfo() updateinfo = self.__verify_updateinfo(repodata) # Create a new non-security update for the same package newbuild = 'TurboGears-1.0.2.2-3.fc7' update = PackageUpdate(title=newbuild, release=release, submitter=builds[0]['owner_name'], status='stable', notes='foobar', type='enhancement') build = PackageBuild(nvr=newbuild, package=package) update.addPackageBuild(build) update.assign_id() koji.__untag__.append('TurboGears-1.0.2.2-2.fc7') ## Test out updateinfo.xml updating via our ExtendedMetadata md = ExtendedMetadata(temprepo, updateinfo) md.insert_updateinfo() updateinfo = self.__verify_updateinfo(repodata) ## Read an verify the updateinfo.xml.gz uinfo = UpdateMetadata() uinfo.add(updateinfo) print(testutil.get_log()) assert len(uinfo.get_notices()) == 2, len(uinfo.get_notices()) assert uinfo.get_notice(get_nvr('TurboGears-1.0.2.2-2.fc7')) notice = uinfo.get_notice(get_nvr(update.title)) assert notice, 'Cannot find update for %r' % get_nvr(update.title) assert notice['status'] == update.status assert notice['updated'] == update.date_modified assert notice['from'] == str(config.get('bodhi_email')) assert notice['description'] == update.notes assert notice['issued'] is not None assert notice['update_id'] == update.updateid ## Clean up shutil.rmtree(temprepo) del(koji.__untag__[:])
def tearDown(self): DBSession.remove() get_session().clear() shutil.rmtree(self.tempdir)
def get_all_packages(): """ Get a list of all packages in Koji """ log.debug('Fetching list of all packages...') koji = buildsys.get_session() return [pkg['package_name'] for pkg in koji.listPackages()]
def test_extended_metadata_updating(self): # grab the name of a build in updates-testing, and create it in our db koji = get_session() builds = koji.listTagged('dist-f13-updates-testing', latest=True) # Create all of the necessary database entries release = Release(name='F13', long_name='Fedora 13', id_prefix='FEDORA', dist_tag='dist-f13') package = Package(name=builds[0]['package_name']) update = PackageUpdate(title=builds[0]['nvr'], release=release, submitter=builds[0]['owner_name'], status='testing', notes='foobar', type='bugfix') build = PackageBuild(nvr=builds[0]['nvr'], package=package) update.addPackageBuild(build) bug = Bugzilla(bz_id=1) bug.title = u'test bug' update.addBugzilla(bug) cve = CVE(cve_id="CVE-2007-0000") update.addCVE(cve) update.assign_id() ## Initialize our temporary repo temprepo = join(tempfile.mkdtemp('bodhi'), 'f7-updates-testing') print "Inserting updateinfo into temprepo: %s" % temprepo mkmetadatadir(join(temprepo, 'i386')) repodata = join(temprepo, 'i386', 'repodata') assert exists(join(repodata, 'repomd.xml')) ## Generate the XML md = ExtendedMetadata(temprepo) ## Insert the updateinfo.xml into the repository md.insert_updateinfo() updateinfo = join(repodata, 'updateinfo.xml.gz') assert exists(updateinfo) ## Read an verify the updateinfo.xml.gz uinfo = UpdateMetadata() uinfo.add(updateinfo) notice = uinfo.get_notice(('mutt', '1.5.14', '1.fc13')) assert not notice notice = uinfo.get_notice(get_nvr(update.title)) assert notice assert notice['status'] == update.status assert notice['updated'] == update.date_modified assert notice['from'] == str(config.get('bodhi_email')) assert notice['description'] == update.notes assert notice['issued'] != None assert notice['update_id'] == update.updateid cve = notice['references'][0] assert cve['type'] == 'cve' assert cve['href'] == update.cves[0].get_url() assert cve['id'] == update.cves[0].cve_id bug = notice['references'][1] assert bug['href'] == update.bugs[0].get_url() assert bug['id'] == '1' assert bug['type'] == 'bugzilla' assert bug['title'] == 'test bug' # FC6's yum update metadata parser doesn't know about some stuff from yum import __version__ if __version__ >= '3.0.6': assert notice['title'] == update.title assert notice['release'] == update.release.long_name assert cve['title'] == None ## Test out updateinfo.xml updating via our ExtendedMetadata md = ExtendedMetadata(temprepo, updateinfo) md.insert_updateinfo() updateinfo = join(repodata, 'updateinfo.xml.gz') assert exists(updateinfo) ## Read an verify the updateinfo.xml.gz uinfo = UpdateMetadata() uinfo.add(updateinfo) notice = uinfo.get_notice(('mutt', '1.5.14', '1.fc13')) assert not notice notice = uinfo.get_notice(get_nvr(update.title)) assert notice assert notice['status'] == update.status assert notice['updated'] == update.date_modified assert notice['from'] == str(config.get('bodhi_email')) assert notice['description'] == update.notes assert notice['issued'] != None assert notice['update_id'] == update.updateid cve = notice['references'][0] assert cve['type'] == 'cve' assert cve['href'] == update.cves[0].get_url() assert cve['id'] == update.cves[0].cve_id bug = notice['references'][1] assert bug['href'] == update.bugs[0].get_url() assert bug['id'] == '1' assert bug['type'] == 'bugzilla' assert bug['title'] == 'test bug', bug # FC6's yum update metadata parser doesn't know about some stuff from yum import __version__ if __version__ >= '3.0.6': assert notice['title'] == update.title assert notice['release'] == update.release.long_name assert cve['title'] == None ## Clean up shutil.rmtree(temprepo)
def main(): load_config() __connection__ = hub = PackageHub("bodhi") koji = get_session() tasks = [] broke = set() # Clean up any stray pending tags for release in Release.select(): print "Finding all pending-testing builds..." if release.name.startswith('EL'): continue tag = release.pending_testing_tag tagged = [build['nvr'] for build in koji.listTagged(tag)] for nvr in tagged: try: build = PackageBuild.byNvr(nvr) for update in build.updates: if update.status in ('testing', 'stable', 'obsolete'): print "%s %s" % (nvr, update.status) if '--fix' in sys.argv: print "Untagging %s" % nvr koji.untagBuild(tag, nvr, force=True) except SQLObjectNotFound: print "Can't find build for %s" % nvr if '--fix' in sys.argv: print "Untagging %s" % nvr koji.untagBuild(tag, nvr, force=True) tag = release.pending_stable_tag tagged = [build['nvr'] for build in koji.listTagged(tag)] for nvr in tagged: try: build = PackageBuild.byNvr(nvr) for update in build.updates: if update.status in ('pending', 'obsolete', 'stable'): print "%s %s" % (nvr, update.status) if '--fix' in sys.argv: print "Untagging %s" % nvr koji.untagBuild(tag, nvr, force=True) except SQLObjectNotFound: print "Can't find build for %s" % nvr if '--fix' in sys.argv: print "Untagging %s" % nvr koji.untagBuild(tag, nvr, force=True) # Check for testing updates that aren't tagged properly for update in PackageUpdate.select(PackageUpdate.q.status == 'testing'): dest_tag = update.release.testing_tag for build in update.builds: tags = [tag['name'] for tag in koji.listTags(build=build.nvr)] if dest_tag not in tags: print "%s marked as testing, but tagged with %s" % (build.nvr, tags) if '--fix' in sys.argv: broke.add((tags[0], dest_tag, build.nvr)) # Check all candidate updates to see if they are in a different bodhi state for release in Release.select(): tag = release.candidate_tag tagged = [build['nvr'] for build in koji.listTagged(tag, latest=True)] for nvr in tagged: try: build = PackageBuild.byNvr(nvr) for update in build.updates: if update.status in ('testing', 'stable'): print "%s %s but tagged as %s" % (nvr, update.status, tag) if '--fix' in sys.argv: dest = release.testing_tag if update.status == 'stable': dest = release.stable_tag elif update.status == 'obsolete': dest = release.candidate_tag broke.add((tag, dest, nvr)) except SQLObjectNotFound: pass # Make sure that all builds in koji tagged as an update exist # in bodhi, and are in the expect state. for release in Release.select(): for tag in (release.testing_tag, release.stable_tag): tagged = [ build['nvr'] for build in koji.listTagged(tag, latest=True) ] for nvr in tagged: try: build = PackageBuild.byNvr(nvr) except SQLObjectNotFound: print "PackageUpdate(%s) not found!" % nvr continue if not len(build.updates): print "PackageBuild(%s) has no updates" % (build.nvr) status = 'testing' in tag and 'testing' or 'stable' for update in build.updates: if update.status != status: print "%s is %s in bodhi but tagged as %s in koji" % ( update.title, update.status, tag) if '--fix' in sys.argv: dest = release.testing_tag if update.status == 'stable': dest = release.stable_tag elif update.status == 'obsolete': dest = release.candidate_tag for b in update.builds: broke.add((tag, dest, b.nvr)) if broke: print " ** Fixing broken tags! **" koji.multicall = True for tag, dest, build in broke: print "Moving %s from %s to %s" % (build, tag, dest) koji.moveBuild(tag, dest, build, force=True) print "Running koji.multiCall()" results = koji.multiCall() success = False print "Waiting for tasks" bad_tasks = wait_for_tasks([task[0] for task in results]) if bad_tasks == 0: success = True if success: print "Tags successfully moved!" else: print "Error moving tags!" print "bad_tasks = %r" % bad_tasks
def test_extended_metadata_updating_with_edited_updates(self): testutil.capture_log(['bodhi.metadata']) # grab the name of a build in updates-testing, and create it in our db koji = get_session() builds = koji.listTagged('dist-f13-updates-testing', latest=True) # Create all of the necessary database entries release = Release(name='F13', long_name='Fedora 13', id_prefix='FEDORA', dist_tag='dist-f13') package = Package(name=builds[0]['package_name']) update = PackageUpdate(title=builds[0]['nvr'], release=release, submitter=builds[0]['owner_name'], status='testing', notes='foobar', type='bugfix') build = PackageBuild(nvr=builds[0]['nvr'], package=package) update.addPackageBuild(build) update.assign_id() ## Initialize our temporary repo temprepo = join(tempfile.mkdtemp('bodhi'), 'f13-updates-testing') print "Inserting updateinfo into temprepo: %s" % temprepo mkmetadatadir(join(temprepo, 'i386')) repodata = join(temprepo, 'i386', 'repodata') assert exists(join(repodata, 'repomd.xml')) ## Generate the XML md = ExtendedMetadata(temprepo) ## Insert the updateinfo.xml into the repository md.insert_updateinfo() updateinfo = self.__verify_updateinfo(repodata) ## Read an verify the updateinfo.xml.gz uinfo = UpdateMetadata() uinfo.add(updateinfo) notice = uinfo.get_notice(('mutt', '1.5.14', '1.fc13')) assert not notice notice = uinfo.get_notice(get_nvr(update.title)) assert notice assert notice['status'] == update.status assert notice['updated'] == update.date_modified assert notice['from'] == str(config.get('bodhi_email')) assert notice['description'] == update.notes assert notice['issued'] is not None assert notice['update_id'] == update.updateid assert notice['title'] == update.title assert notice['release'] == update.release.long_name ## Edit the update and bump the build revision nvr = 'TurboGears-1.0.2.2-3.fc7' newbuild = PackageBuild(nvr=nvr, package=package) update.removePackageBuild(build) update.addPackageBuild(newbuild) update.title = nvr update.date_modified = datetime.utcnow() # Pretend -2 was unpushed assert len(koji.__untag__) == 0 koji.__untag__.append('TurboGears-1.0.2.2-2.fc7') ## Test out updateinfo.xml updating via our ExtendedMetadata md = ExtendedMetadata(temprepo, updateinfo) md.insert_updateinfo() updateinfo = self.__verify_updateinfo(repodata) ## Read an verify the updateinfo.xml.gz uinfo = UpdateMetadata() uinfo.add(updateinfo) print(testutil.get_log()) notice = uinfo.get_notice(('TurboGears', '1.0.2.2', '2.fc7')) assert not notice, "Old TG notice did not get pruned: %s" % notice notice = uinfo.get_notice(('TurboGears', '1.0.2.2', '3.fc7')) assert notice, uinfo assert notice['title'] == update.title num_notices = len(uinfo.get_notices()) assert num_notices == 1, num_notices ## Clean up shutil.rmtree(temprepo) del (koji.__untag__[:])