def test_security_update_priority_testing(self, publish, *args): with self.db_factory() as db: up = db.query(Update).one() up.type = UpdateType.security up.request = UpdateRequest.testing user = db.query(User).first() # Create a security update for a different release release = Release( name=u'F18', long_name=u'Fedora 18', id_prefix=u'FEDORA', version=u'18', dist_tag=u'f18', stable_tag=u'f18-updates', testing_tag=u'f18-updates-testing', candidate_tag=u'f18-updates-candidate', pending_testing_tag=u'f18-updates-testing-pending', pending_stable_tag=u'f18-updates-pending', override_tag=u'f18-override', branch=u'f18') db.add(release) build = Build(nvr=u'bodhi-2.0-1.fc18', release=release, package=up.builds[0].package) db.add(build) update = Update( title=u'bodhi-2.0-1.fc18', builds=[build], user=user, status=UpdateStatus.testing, request=UpdateRequest.stable, notes=u'Useful details!', release=release) update.type = UpdateType.enhancement db.add(update) # Wipe out the tag cache so it picks up our new release Release._tag_cache = None self.msg['body']['msg']['updates'] += ['bodhi-2.0-1.fc18'] self.masher.consume(self.msg) # Ensure that F17 updates-testing runs before F18 calls = publish.mock_calls self.assertEquals(calls[1], mock.call( msg={'repo': u'f17-updates-testing', 'updates': [u'bodhi-2.0-1.fc17']}, force=True, topic='mashtask.mashing')) self.assertEquals(calls[3], mock.call( msg={'success': True, 'repo': 'f17-updates-testing'}, force=True, topic='mashtask.complete')) self.assertEquals(calls[4], mock.call( msg={'repo': u'f18-updates', 'updates': [u'bodhi-2.0-1.fc18']}, force=True, topic='mashtask.mashing')) self.assertEquals(calls[-1], mock.call( msg={'success': True, 'repo': 'f18-updates'}, force=True, topic='mashtask.complete'))
def consume(self, message): msg = message["body"]["msg"] topic = message["topic"] alias = msg["update"].get("alias") log.info("Updates Handler handling %s, %s" % (alias, topic)) # Go to sleep for a second to try and avoid a race condition # https://github.com/fedora-infra/bodhi/issues/458 time.sleep(1) if not alias: log.error("Update Handler got update with no " "alias %s." % pprint.pformat(msg)) return with self.db_factory() as session: update = Update.get(alias, session) if not update: raise BodhiException("Couldn't find alias %r in DB" % alias) if topic.endswith("update.edit"): bugs = [Bug.get(idx, session) for idx in msg["new_bugs"]] # Sanity check for bug in bugs: assert bug in update.bugs elif topic.endswith("update.request.testing"): bugs = update.bugs else: raise NotImplementedError("Should never get here.") self.work_on_bugs(session, update, bugs) self.fetch_test_cases(session, update) log.info("Updates Handler done with %s, %s" % (alias, topic))
def test_obsolete_older_updates(self, publish, *args): otherbuild = 'bodhi-2.0-2.fc17' oldbuild = None self.msg['body']['msg']['updates'].insert(0, otherbuild) with self.db_factory() as session: # Put the older update into testing oldupdate = session.query(Update).one() oldbuild = oldupdate.builds[0].nvr oldupdate.status = UpdateStatus.testing oldupdate.request = None oldupdate.locked = False # Create a newer build build = Build(nvr=otherbuild, package=oldupdate.builds[0].package) session.add(build) update = Update(title=otherbuild, builds=[build], type=UpdateType.bugfix, request=UpdateRequest.testing, notes=u'second update', user=oldupdate.user, release=oldupdate.release) session.add(update) session.flush() self.masher.consume(self.msg) with self.db_factory() as session: # Ensure that the older update got obsoleted up = session.query(Update).filter_by(title=oldbuild).one() self.assertEquals(up.status, UpdateStatus.obsolete) self.assertEquals(up.request, None) # The latest update should be in testing up = session.query(Update).filter_by(title=otherbuild).one() self.assertEquals(up.status, UpdateStatus.testing) self.assertEquals(up.request, None)
def test_tag_ordering(self, publish, *args): """ Test pushing an batch of updates with multiple builds for the same package. Ensure that the latest version is tagged last. """ otherbuild = 'bodhi-2.0-2.fc17' self.msg['body']['msg']['updates'].insert(0, otherbuild) with self.db_factory() as session: firstupdate = session.query(Update).filter_by( title=self.msg['body']['msg']['updates'][1]).one() build = Build(nvr=otherbuild, package=firstupdate.builds[0].package) session.add(build) update = Update(title=otherbuild, builds=[build], type=UpdateType.bugfix, request=UpdateRequest.testing, notes=u'second update', user=firstupdate.user, release=firstupdate.release) session.add(update) session.flush() # Start the push self.masher.consume(self.msg) # Ensure that fedmsg was called 5 times self.assertEquals(len(publish.call_args_list), 5) # Also, ensure we reported success publish.assert_called_with(topic="mashtask.complete", msg=dict(success=True, repo='f17-updates-testing'), force=True) # Ensure our two updates were moved self.assertEquals(len(self.koji.__moved__), 2) self.assertEquals(len(self.koji.__added__), 0) # Ensure the most recent version is tagged last in order to be the 'koji latest-pkg' self.assertEquals(self.koji.__moved__[0], (u'f17-updates-candidate', u'f17-updates-testing', u'bodhi-2.0-1.fc17')) self.assertEquals(self.koji.__moved__[1], (u'f17-updates-candidate', u'f17-updates-testing', u'bodhi-2.0-2.fc17'))
def test_list_comments_by_update_no_comments(self): update = Update( title=u'bodhi-2.0-200.fc17', #builds=[build], #user=user, request=UpdateRequest.testing, type=UpdateType.enhancement, notes=u'Useful details!', #release=release, date_submitted=datetime(1984, 11, 02), requirements=u'rpmlint', stable_karma=3, unstable_karma=-3, ) self.db.add(update) self.db.flush() res = self.app.get('/comments/', {"updates": "bodhi-2.0-200.fc17"}) body = res.json_body self.assertEquals(len(body['comments']), 0)
def consume(self, message): msg = message['body']['msg'] topic = message['topic'] alias = msg['update'].get('alias') log.info("Updates Handler handling %s, %s" % (alias, topic)) # Go to sleep for a second to try and avoid a race condition # https://github.com/fedora-infra/bodhi/issues/458 time.sleep(1) if not alias: log.error("Update Handler got update with no " "alias %s." % pprint.pformat(msg)) return with self.db_factory() as session: update = Update.get(alias, session) if not update: raise BodhiException("Couldn't find alias %r in DB" % alias) if topic.endswith('update.edit'): bugs = [Bug.get(idx, session) for idx in msg['new_bugs']] # Sanity check for bug in bugs: assert bug in update.bugs elif topic.endswith('update.request.testing'): bugs = update.bugs else: raise NotImplementedError("Should never get here.") self.work_on_bugs(session, update, bugs) self.fetch_test_cases(session, update) log.info("Updates Handler done with %s, %s" % (alias, topic))
def setUp(self, *args, **kwargs): super(TestCommentsService, self).setUp(*args, **kwargs) # Add a second update owned by somebody else so we can test karma # policy stuff user2 = User(name=u'lmacken') self.db.flush() self.db.add(user2) release = self.db.query(Release).filter_by(name=u'F17').one() update = Update( title=someone_elses_update, #builds=[build], user=user2, request=UpdateRequest.testing, type=UpdateType.enhancement, notes=u'Useful details!', release=release, date_submitted=datetime(1984, 11, 02), requirements=u'rpmlint', stable_karma=3, unstable_karma=-3, ) self.db.add(update) self.db.flush()
def test_metadata_updating_with_old_stable_security(self): update = self.db.query(Update).one() update.request = None update.type = UpdateType.security update.status = UpdateStatus.stable update.date_pushed = datetime.utcnow() DevBuildsys.__tagged__[update.title] = ['f17-updates'] repo = join(self.tempdir, 'f17-updates') mkmetadatadir(join(repo, 'f17-updates', 'i386')) self.repodata = join(repo, 'f17-updates', 'i386', 'repodata') # Generate the XML md = ExtendedMetadata(update.release, UpdateRequest.stable, self.db, repo) # Insert the updateinfo.xml into the repository md.insert_updateinfo() md.cache_repodata() updateinfo = self._verify_updateinfo(self.repodata) # Read an verify the updateinfo.xml.gz uinfo = createrepo_c.UpdateInfo(updateinfo) notice = self.get_notice(uinfo, update.title) self.assertIsNotNone(notice) # Create a new non-security update for the same package newbuild = 'bodhi-2.0-2.fc17' pkg = self.db.query(Package).filter_by(name=u'bodhi').one() build = Build(nvr=newbuild, package=pkg) self.db.add(build) self.db.flush() newupdate = Update(title=newbuild, type=UpdateType.enhancement, status=UpdateStatus.stable, request=None, release=update.release, builds=[build], notes=u'x') newupdate.assign_alias() self.db.add(newupdate) self.db.flush() # Untag the old security build DevBuildsys.__untag__.append(update.title) DevBuildsys.__tagged__[newupdate.title] = [newupdate.release.stable_tag] buildrpms = DevBuildsys.__rpms__[0].copy() buildrpms['nvr'] = 'bodhi-2.0-2.fc17' buildrpms['release'] = '2.fc17' DevBuildsys.__rpms__.append(buildrpms) # Re-initialize our temporary repo shutil.rmtree(repo) os.mkdir(repo) mkmetadatadir(join(repo, 'f17-updates', 'i386')) md = ExtendedMetadata(update.release, UpdateRequest.stable, self.db, repo) md.insert_updateinfo() updateinfo = self._verify_updateinfo(self.repodata) # Read an verify the updateinfo.xml.gz uinfo = createrepo_c.UpdateInfo(updateinfo) self.assertEquals(len(uinfo.updates), 2) notice = self.get_notice(uinfo, 'bodhi-2.0-1.fc17') self.assertIsNotNone(notice) notice = self.get_notice(uinfo, 'bodhi-2.0-2.fc17') self.assertIsNotNone(notice)
def test_security_updates_parallel(self, publish, *args): with self.db_factory() as db: up = db.query(Update).one() up.type = UpdateType.security up.status = UpdateStatus.testing up.request = UpdateRequest.stable user = db.query(User).first() # Create a security update for a different release release = Release( name=u"F18", long_name=u"Fedora 18", id_prefix=u"FEDORA", version=u"18", dist_tag=u"f18", stable_tag=u"f18-updates", testing_tag=u"f18-updates-testing", candidate_tag=u"f18-updates-candidate", pending_testing_tag=u"f18-updates-testing-pending", pending_stable_tag=u"f18-updates-pending", override_tag=u"f18-override", branch=u"f18", ) db.add(release) build = Build(nvr=u"bodhi-2.0-1.fc18", release=release, package=up.builds[0].package) db.add(build) update = Update( title=u"bodhi-2.0-1.fc18", builds=[build], user=user, status=UpdateStatus.testing, request=UpdateRequest.stable, notes=u"Useful details!", release=release, ) update.type = UpdateType.security db.add(update) # Wipe out the tag cache so it picks up our new release Release._tag_cache = None self.msg["body"]["msg"]["updates"] += ["bodhi-2.0-1.fc18"] self.masher.consume(self.msg) # Ensure that F18 and F17 run in parallel calls = publish.mock_calls if calls[1] == mock.call( msg={"repo": u"f18-updates", "updates": [u"bodhi-2.0-1.fc18"]}, force=True, topic="mashtask.mashing" ): self.assertEquals( calls[2], mock.call( msg={"repo": u"f17-updates", "updates": [u"bodhi-2.0-1.fc17"]}, force=True, topic="mashtask.mashing" ), ) elif calls[1] == self.assertEquals( calls[1], mock.call( msg={"repo": u"f17-updates", "updates": [u"bodhi-2.0-1.fc17"]}, force=True, topic="mashtask.mashing" ), ): self.assertEquals( calls[2], mock.call( msg={"repo": u"f18-updates", "updates": [u"bodhi-2.0-1.fc18"]}, force=True, topic="mashtask.mashing" ), )
def populate(db): user = User(name=u'guest') db.add(user) anonymous = User(name=u'anonymous') db.add(anonymous) provenpackager = Group(name=u'provenpackager') db.add(provenpackager) packager = Group(name=u'packager') db.add(packager) db.flush() user.groups.append(packager) release = Release( name=u'F17', long_name=u'Fedora 17', id_prefix=u'FEDORA', version=u'17', dist_tag=u'f17', stable_tag=u'f17-updates', testing_tag=u'f17-updates-testing', candidate_tag=u'f17-updates-candidate', pending_testing_tag=u'f17-updates-testing-pending', pending_stable_tag=u'f17-updates-pending', override_tag=u'f17-override', branch=u'f17') db.add(release) pkg = Package(name=u'bodhi') db.add(pkg) user.packages.append(pkg) build = Build(nvr=u'bodhi-2.0-1.fc17', release=release, package=pkg) db.add(build) testcase = TestCase(name=u'Wat') db.add(testcase) pkg.test_cases.append(testcase) update = Update( title=u'bodhi-2.0-1.fc17', builds=[build], user=user, request=UpdateRequest.testing, notes=u'Useful details!', release=release, date_submitted=datetime(1984, 11, 02), requirements=u'rpmlint', stable_karma=3, unstable_karma=-3, ) update.type = UpdateType.bugfix bug = Bug(bug_id=12345) db.add(bug) update.bugs.append(bug) cve = CVE(cve_id=u"CVE-1985-0110") db.add(cve) update.cves.append(cve) comment = Comment(karma=1, text=u"wow. amaze.") db.add(comment) comment.user = user update.comments.append(comment) update.karma = 1 comment = Comment(karma=0, text=u"srsly. pretty good.", anonymous=True) comment.user = anonymous db.add(comment) update.comments.append(comment) with mock.patch(target='uuid.uuid4', return_value='wat'): update.assign_alias() db.add(update) expiration_date = datetime.utcnow() expiration_date = expiration_date + timedelta(days=1) override = BuildrootOverride(build=build, submitter=user, notes=u'blah blah blah', expiration_date=expiration_date) db.add(override) db.flush()
def new_update(request): """ Save an update. This entails either creating a new update, or editing an existing one. To edit an existing update, the update's original title must be specified in the ``edited`` parameter. """ data = request.validated log.debug('validated = %s' % data) # This has already been validated at this point, but we need to ditch # it since the models don't care about a csrf argument. data.pop('csrf_token') caveats = [] try: releases = set() builds = [] # Create the Package and Build entities for nvr in data['builds']: name, version, release = request.buildinfo[nvr]['nvr'] package = request.db.query(Package).filter_by(name=name).first() if not package: package = Package(name=name) request.db.add(package) request.db.flush() build = Build.get(nvr, request.db) if build is None: log.debug("Adding nvr %s", nvr) build = Build(nvr=nvr, package=package) request.db.add(build) request.db.flush() build.package = package build.release = request.buildinfo[build.nvr]['release'] builds.append(build) releases.add(request.buildinfo[build.nvr]['release']) if data.get('edited'): log.info('Editing update: %s' % data['edited']) assert len(releases) == 1, "Updates may not span multiple releases" data['release'] = list(releases)[0] data['builds'] = [b.nvr for b in builds] result, _caveats = Update.edit(request, data) caveats.extend(_caveats) else: if len(releases) > 1: caveats.append({ 'name': 'releases', 'description': 'Your update is being split ' 'into %i, one for each release.' % len(releases) }) updates = [] for release in releases: _data = copy.copy(data) # Copy it because .new(..) mutates it _data['builds'] = [b for b in builds if b.release == release] _data['release'] = release log.info('Creating new update: %r' % _data['builds']) result, _caveats = Update.new(request, _data) log.debug('update = %r' % result) updates.append(result) caveats.extend(_caveats) if len(releases) > 1: result = dict(updates=updates) except LockedUpdateException as e: log.warn(str(e)) request.errors.add('body', 'builds', "%s" % str(e)) return except Exception as e: log.exception('Failed to create update') request.errors.add( 'body', 'builds', 'Unable to create update. %s' % str(e)) return # Obsolete older updates for three different cases... # editing an update, submitting a new single update, submitting multiple. if isinstance(result, dict): updates = result['updates'] else: updates = [result] for update in updates: try: caveats.extend(update.obsolete_older_updates(request)) except Exception as e: caveats.append({ 'name': 'update', 'description': 'Problem obsoleting older updates: %s' % str(e), }) if not isinstance(result, dict): result = result.__json__() result['caveats'] = caveats return result
def test_metadata_updating_with_old_testing_security(self): update = self.db.query(Update).one() update.request = None update.type = UpdateType.security update.status = UpdateStatus.testing update.date_pushed = datetime.utcnow() DevBuildsys.__tagged__[update.title] = ["f17-updates-testing"] # Generate the XML md = ExtendedMetadata(update.release, UpdateRequest.testing, self.db, self.temprepo) # Insert the updateinfo.xml into the repository md.insert_updateinfo() md.cache_repodata() updateinfo = self._verify_updateinfo(self.repodata) # Read an verify the updateinfo.xml.gz uinfo = createrepo_c.UpdateInfo(updateinfo) notice = self.get_notice(uinfo, update.title) self.assertIsNotNone(notice) # Create a new non-security update for the same package newbuild = "bodhi-2.0-2.fc17" pkg = self.db.query(Package).filter_by(name=u"bodhi").one() build = Build(nvr=newbuild, package=pkg) self.db.add(build) self.db.flush() newupdate = Update( title=newbuild, type=UpdateType.enhancement, status=UpdateStatus.testing, request=None, release=update.release, builds=[build], notes=u"x", ) newupdate.assign_alias() self.db.add(newupdate) self.db.flush() # Untag the old security build del (DevBuildsys.__tagged__[update.title]) DevBuildsys.__untag__.append(update.title) DevBuildsys.__tagged__[newupdate.title] = [newupdate.release.testing_tag] buildrpms = DevBuildsys.__rpms__[0].copy() buildrpms["nvr"] = "bodhi-2.0-2.fc17" buildrpms["release"] = "2.fc17" DevBuildsys.__rpms__.append(buildrpms) del (DevBuildsys.__rpms__[0]) # Re-initialize our temporary repo shutil.rmtree(self.temprepo) os.mkdir(self.temprepo) mkmetadatadir(join(self.temprepo, "f17-updates-testing", "i386")) md = ExtendedMetadata(update.release, UpdateRequest.testing, self.db, self.temprepo) md.insert_updateinfo() updateinfo = self._verify_updateinfo(self.repodata) # Read an verify the updateinfo.xml.gz uinfo = createrepo_c.UpdateInfo(updateinfo) self.assertEquals(len(uinfo.updates), 1) notice = self.get_notice(uinfo, "bodhi-2.0-1.fc17") self.assertIsNone(notice) notice = self.get_notice(uinfo, "bodhi-2.0-2.fc17") self.assertIsNotNone(notice)
def populate(db): user = User(name=u'guest') db.add(user) anonymous = User(name=u'anonymous') db.add(anonymous) provenpackager = Group(name=u'provenpackager') db.add(provenpackager) packager = Group(name=u'packager') db.add(packager) db.flush() user.groups.append(packager) release = Release(name=u'F17', long_name=u'Fedora 17', id_prefix=u'FEDORA', version=u'17', dist_tag=u'f17', stable_tag=u'f17-updates', testing_tag=u'f17-updates-testing', candidate_tag=u'f17-updates-candidate', pending_testing_tag=u'f17-updates-testing-pending', pending_stable_tag=u'f17-updates-pending', override_tag=u'f17-override', branch=u'f17') db.add(release) pkg = Package(name=u'bodhi') db.add(pkg) user.packages.append(pkg) build = Build(nvr=u'bodhi-2.0-1.fc17', release=release, package=pkg) db.add(build) testcase = TestCase(name=u'Wat') db.add(testcase) pkg.test_cases.append(testcase) update = Update( title=u'bodhi-2.0-1.fc17', builds=[build], user=user, request=UpdateRequest.testing, notes=u'Useful details!', release=release, date_submitted=datetime(1984, 11, 02), requirements=u'rpmlint', stable_karma=3, unstable_karma=-3, ) update.type = UpdateType.bugfix bug = Bug(bug_id=12345) db.add(bug) update.bugs.append(bug) cve = CVE(cve_id=u"CVE-1985-0110") db.add(cve) update.cves.append(cve) comment = Comment(karma=1, text=u"wow. amaze.") db.add(comment) comment.user = user update.comments.append(comment) update.karma = 1 comment = Comment(karma=0, text=u"srsly. pretty good.", anonymous=True) comment.user = anonymous db.add(comment) update.comments.append(comment) with mock.patch(target='uuid.uuid4', return_value='wat'): update.assign_alias() db.add(update) expiration_date = datetime.utcnow() expiration_date = expiration_date + timedelta(days=1) override = BuildrootOverride(build=build, submitter=user, notes=u'blah blah blah', expiration_date=expiration_date) db.add(override) db.flush()
def load_sqlalchemy_db(): print "\nLoading pickled database %s" % sys.argv[2] db = file(sys.argv[2], 'r') data = pickle.load(db) import transaction from bodhi.models import Base from bodhi.models import Release, Update, Build, Comment, User, Bug, CVE from bodhi.models import Package, Group from bodhi.models import UpdateType, UpdateStatus, UpdateRequest from sqlalchemy import create_engine from sqlalchemy.orm.exc import NoResultFound # Caches for quick lookup releases = {} packages = {} users = {} critpath = {} aliases = [] engine = bodhi.config['sqlalchemy.url'] Session = scoped_session( sessionmaker(extension=ZopeTransactionExtension())) Session.configure(bind=engine) db = Session() # Allow filtering of releases to load whitelist = [] if '--release' in sys.argv: for r in sys.argv[sys.argv.index('--release') + 1].split(','): whitelist.append(r) print('whitelist = %r' % whitelist) # Legacy format was just a list of update dictionaries # Now we'll pull things out into an organized dictionary: # {'updates': [], 'releases': []} if isinstance(data, dict): for release in data['releases']: try: db.query(Release).filter_by(name=release['name']).one() except NoResultFound: del (release['metrics']) del (release['locked']) r = Release(**release) r.stable_tag = "%s-updates" % r.dist_tag r.testing_tag = "%s-testing" % r.stable_tag r.candidate_tag = "%s-candidate" % r.stable_tag r.pending_testing_tag = "%s-pending" % r.testing_tag r.pending_stable_tag = "%s-pending" % r.stable_tag r.override_tag = "%s-override" % r.dist_tag db.add(r) data = data['updates'] progress = ProgressBar(widgets=[SimpleProgress(), Percentage(), Bar()]) for u in progress(data): try: release = releases[u['release'][0]] except KeyError: try: release = db.query(Release).filter_by( name=u['release'][0]).one() except NoResultFound: release = Release(name=u['release'][0], long_name=u['release'][1], id_prefix=u['release'][2], dist_tag=u['release'][3]) db.add(release) releases[u['release'][0]] = release if whitelist: if release.name in whitelist: critpath[release.name] = get_critpath_pkgs( release.name.lower()) print('%s critpath packages for %s' % (len(critpath[release.name]), release.name)) else: critpath[release.name] = get_critpath_pkgs( release.name.lower()) print('%s critpath packages for %s' % (len(critpath[release.name]), release.name)) if whitelist and release.name not in whitelist: continue ## Backwards compatbility request = u['request'] if u['request'] == 'move': u['request'] = 'stable' elif u['request'] == 'push': u['request'] = 'testing' elif u['request'] == 'unpush': u['request'] = 'obsolete' if u['approved'] not in (True, False): u['approved'] = None if u.has_key('update_id'): u['updateid'] = u['update_id'] u['alias'] = u['update_id'] if u['alias']: split = u['alias'].split('-') year, id = split[-2:] aliases.append((int(year), int(id))) if not u.has_key('date_modified'): u['date_modified'] = None # Port to new enum types if u['request']: if u['request'] == 'stable': u['request'] = UpdateRequest.stable elif u['request'] == 'testing': u['request'] = UpdateRequest.testing else: raise Exception("Unknown request: %s" % u['request']) if u['type'] == 'bugfix': u['type'] = UpdateType.bugfix elif u['type'] == 'newpackage': u['type'] = UpdateType.newpackage elif u['type'] == 'enhancement': u['type'] = UpdateType.enhancement elif u['type'] == 'security': u['type'] = UpdateType.security else: raise Exception("Unknown type: %r" % u['type']) if u['status'] == 'pending': u['status'] = UpdateStatus.pending elif u['status'] == 'testing': u['status'] = UpdateStatus.testing elif u['status'] == 'obsolete': u['status'] = UpdateStatus.obsolete elif u['status'] == 'stable': u['status'] = UpdateStatus.stable elif u['status'] == 'unpushed': u['status'] = UpdateStatus.unpushed else: raise Exception("Unknown status: %r" % u['status']) try: update = db.query(Update).filter_by(title=u['title']).one() continue except NoResultFound: update = Update( title=u['title'], date_submitted=u['date_submitted'], date_pushed=u['date_pushed'], date_modified=u['date_modified'], release=release, old_updateid=u['updateid'], alias=u['updateid'], pushed=u['pushed'], notes=u['notes'], karma=u['karma'], type=u['type'], status=u['status'], request=u['request'], ) #approved=u['approved']) db.add(update) db.flush() try: user = users[u['submitter']] except KeyError: try: user = db.query(User).filter_by(name=u['submitter']).one() except NoResultFound: user = User(name=u['submitter']) db.add(user) db.flush() users[u['submitter']] = user user.updates.append(update) ## Create Package and Build objects for pkg, nvr in u['builds']: try: package = packages[pkg] except KeyError: try: package = db.query(Package).filter_by(name=pkg).one() except NoResultFound: package = Package(name=pkg) db.add(package) packages[pkg] = package if package.name in critpath[update.release.name]: update.critpath = True try: build = db.query(Build).filter_by(nvr=nvr).one() except NoResultFound: build = Build(nvr=nvr, package=package) db.add(build) update.builds.append(build) ## Create all Bugzilla objects for this update for bug_num, bug_title, security, parent in u['bugs']: try: bug = db.query(Bug).filter_by(bug_id=bug_num).one() except NoResultFound: bug = Bug(bug_id=bug_num, security=security, parent=parent, title=bug_title) db.add(bug) update.bugs.append(bug) ## Create all CVE objects for this update for cve_id in u['cves']: try: cve = db.query(CVE).filter_by(cve_id=cve_id).one() except NoResultFound: cve = CVE(cve_id=cve_id) db.add(cve) update.cves.append(cve) ## Create all Comments for this update for c in u['comments']: try: timestamp, author, text, karma, anonymous = c except ValueError: timestamp, author, text, karma = c anonymous = '@' in author comment = Comment(timestamp=timestamp, text=text, karma=karma, anonymous=anonymous) db.add(comment) db.flush() update.comments.append(comment) if anonymous: name = u'anonymous' else: name = author group = None if not anonymous and ' (' in name: split = name.split(' (') name = split[0] group = split[1][:-1] assert group, name try: user = users[name] except KeyError: try: user = db.query(User).filter_by(name=name).one() except NoResultFound: user = User(name=name) db.add(user) db.flush() users[name] = user comment.user = user if group: try: group = db.query(Group).filter_by(name=group).one() except NoResultFound: group = Group(name=group) db.add(group) db.flush() user.groups.append(group) db.flush() # Hack to get the Bodhi2 alias generator working with bodhi1 data. # The new generator assumes that the alias is assigned at submission time, as opposed to push time. year, id = max(aliases) print('Highest alias = %r %r' % (year, id)) up = db.query(Update).filter_by(alias=u'FEDORA-%s-%s' % (year, id)).one() print(up.title) up.date_submitted = up.date_pushed db.flush() transaction.commit() print("\nDatabase migration complete!") print(" * %d updates" % db.query(Update).count()) print(" * %d builds" % db.query(Build).count()) print(" * %d comments" % db.query(Comment).count()) print(" * %d users" % db.query(User).count()) print(" * %d bugs" % db.query(Bug).count()) print(" * %d CVEs" % db.query(CVE).count())
def load_sqlalchemy_db(): print "\nLoading pickled database %s" % sys.argv[2] db = file(sys.argv[2], 'r') data = pickle.load(db) import transaction from bodhi.models import Base from bodhi.models import Release, Update, Build, Comment, User, Bug, CVE from bodhi.models import Package, Group from bodhi.models import UpdateType, UpdateStatus, UpdateRequest from sqlalchemy import create_engine from sqlalchemy.orm.exc import NoResultFound # Caches for quick lookup releases = {} packages = {} users = {} critpath = {} aliases = [] engine = bodhi.config['sqlalchemy.url'] Session = scoped_session(sessionmaker(extension=ZopeTransactionExtension())) Session.configure(bind=engine) db = Session() # Allow filtering of releases to load whitelist = [] if '--release' in sys.argv: for r in sys.argv[sys.argv.index('--release') + 1].split(','): whitelist.append(r) print('whitelist = %r' % whitelist) # Legacy format was just a list of update dictionaries # Now we'll pull things out into an organized dictionary: # {'updates': [], 'releases': []} if isinstance(data, dict): for release in data['releases']: try: db.query(Release).filter_by(name=release['name']).one() except NoResultFound: del(release['metrics']) del(release['locked']) r = Release(**release) r.stable_tag = "%s-updates" % r.dist_tag r.testing_tag = "%s-testing" % r.stable_tag r.candidate_tag = "%s-candidate" % r.stable_tag r.pending_testing_tag = "%s-pending" % r.testing_tag r.pending_stable_tag = "%s-pending" % r.stable_tag r.override_tag = "%s-override" % r.dist_tag db.add(r) data = data['updates'] progress = ProgressBar(widgets=[SimpleProgress(), Percentage(), Bar()]) for u in progress(data): try: release = releases[u['release'][0]] except KeyError: try: release = db.query(Release).filter_by(name=u['release'][0]).one() except NoResultFound: release = Release(name=u['release'][0], long_name=u['release'][1], id_prefix=u['release'][2], dist_tag=u['release'][3]) db.add(release) releases[u['release'][0]] = release if whitelist: if release.name in whitelist: critpath[release.name] = get_critpath_pkgs(release.name.lower()) print('%s critpath packages for %s' % (len(critpath[release.name]), release.name)) else: critpath[release.name] = get_critpath_pkgs(release.name.lower()) print('%s critpath packages for %s' % (len(critpath[release.name]), release.name)) if whitelist and release.name not in whitelist: continue ## Backwards compatbility request = u['request'] if u['request'] == 'move': u['request'] = 'stable' elif u['request'] == 'push': u['request'] = 'testing' elif u['request'] == 'unpush': u['request'] = 'obsolete' if u['approved'] not in (True, False): u['approved'] = None if u.has_key('update_id'): u['updateid'] = u['update_id'] u['alias'] = u['update_id'] if u['alias']: split = u['alias'].split('-') year, id = split[-2:] aliases.append((int(year), int(id))) if not u.has_key('date_modified'): u['date_modified'] = None # Port to new enum types if u['request']: if u['request'] == 'stable': u['request'] = UpdateRequest.stable elif u['request'] == 'testing': u['request'] = UpdateRequest.testing else: raise Exception("Unknown request: %s" % u['request']) if u['type'] == 'bugfix': u['type'] = UpdateType.bugfix elif u['type'] == 'newpackage': u['type'] = UpdateType.newpackage elif u['type'] == 'enhancement': u['type'] = UpdateType.enhancement elif u['type'] == 'security': u['type'] = UpdateType.security else: raise Exception("Unknown type: %r" % u['type']) if u['status'] == 'pending': u['status'] = UpdateStatus.pending elif u['status'] == 'testing': u['status'] = UpdateStatus.testing elif u['status'] == 'obsolete': u['status'] = UpdateStatus.obsolete elif u['status'] == 'stable': u['status'] = UpdateStatus.stable elif u['status'] == 'unpushed': u['status'] = UpdateStatus.unpushed else: raise Exception("Unknown status: %r" % u['status']) try: update = db.query(Update).filter_by(title=u['title']).one() continue except NoResultFound: update = Update(title=u['title'], date_submitted=u['date_submitted'], date_pushed=u['date_pushed'], date_modified=u['date_modified'], release=release, old_updateid=u['updateid'], alias=u['updateid'], pushed=u['pushed'], notes=u['notes'], karma=u['karma'], type=u['type'], status=u['status'], request=u['request'], ) #approved=u['approved']) db.add(update) db.flush() try: user = users[u['submitter']] except KeyError: try: user = db.query(User).filter_by(name=u['submitter']).one() except NoResultFound: user = User(name=u['submitter']) db.add(user) db.flush() users[u['submitter']] = user user.updates.append(update) ## Create Package and Build objects for pkg, nvr in u['builds']: try: package = packages[pkg] except KeyError: try: package = db.query(Package).filter_by(name=pkg).one() except NoResultFound: package = Package(name=pkg) db.add(package) packages[pkg] = package if package.name in critpath[update.release.name]: update.critpath = True try: build = db.query(Build).filter_by(nvr=nvr).one() except NoResultFound: build = Build(nvr=nvr, package=package) db.add(build) update.builds.append(build) ## Create all Bugzilla objects for this update for bug_num, bug_title, security, parent in u['bugs']: try: bug = db.query(Bug).filter_by(bug_id=bug_num).one() except NoResultFound: bug = Bug(bug_id=bug_num, security=security, parent=parent, title=bug_title) db.add(bug) update.bugs.append(bug) ## Create all CVE objects for this update for cve_id in u['cves']: try: cve = db.query(CVE).filter_by(cve_id=cve_id).one() except NoResultFound: cve = CVE(cve_id=cve_id) db.add(cve) update.cves.append(cve) ## Create all Comments for this update for c in u['comments']: try: timestamp, author, text, karma, anonymous = c except ValueError: timestamp, author, text, karma = c anonymous = '@' in author comment = Comment(timestamp=timestamp, text=text, karma=karma, anonymous=anonymous) db.add(comment) db.flush() update.comments.append(comment) if anonymous: name = u'anonymous' else: name = author group = None if not anonymous and ' (' in name: split = name.split(' (') name = split[0] group = split[1][:-1] assert group, name try: user = users[name] except KeyError: try: user = db.query(User).filter_by(name=name).one() except NoResultFound: user = User(name=name) db.add(user) db.flush() users[name] = user comment.user = user if group: try: group = db.query(Group).filter_by(name=group).one() except NoResultFound: group = Group(name=group) db.add(group) db.flush() user.groups.append(group) db.flush() # Hack to get the Bodhi2 alias generator working with bodhi1 data. # The new generator assumes that the alias is assigned at submission time, as opposed to push time. year, id = max(aliases) print('Highest alias = %r %r' % (year, id)) up = db.query(Update).filter_by(alias=u'FEDORA-%s-%s' % (year, id)).one() print(up.title) up.date_submitted = up.date_pushed db.flush() transaction.commit() print("\nDatabase migration complete!") print(" * %d updates" % db.query(Update).count()) print(" * %d builds" % db.query(Build).count()) print(" * %d comments" % db.query(Comment).count()) print(" * %d users" % db.query(User).count()) print(" * %d bugs" % db.query(Bug).count()) print(" * %d CVEs" % db.query(CVE).count())
def test_security_update_priority(self, publish, *args): with self.db_factory() as db: up = db.query(Update).one() user = db.query(User).first() # Create a security update for a different release release = Release( name=u"F18", long_name=u"Fedora 18", id_prefix=u"FEDORA", version=u"18", dist_tag=u"f18", stable_tag=u"f18-updates", testing_tag=u"f18-updates-testing", candidate_tag=u"f18-updates-candidate", pending_testing_tag=u"f18-updates-testing-pending", pending_stable_tag=u"f18-updates-pending", override_tag=u"f18-override", branch=u"f18", ) db.add(release) build = Build(nvr=u"bodhi-2.0-1.fc18", release=release, package=up.builds[0].package) db.add(build) update = Update( title=u"bodhi-2.0-1.fc18", builds=[build], user=user, status=UpdateStatus.testing, request=UpdateRequest.stable, notes=u"Useful details!", release=release, ) update.type = UpdateType.security db.add(update) # Wipe out the tag cache so it picks up our new release Release._tag_cache = None self.msg["body"]["msg"]["updates"] += ["bodhi-2.0-1.fc18"] self.masher.consume(self.msg) # Ensure that F18 runs before F17 calls = publish.mock_calls # Order of fedmsgs at the the moment: # masher.start # mashing f18 # complete.stable (for each update) # errata.publish # mashtask.complete # mashing f17 # complete.testing # mashtask.complete self.assertEquals( calls[1], mock.call( force=True, msg={"repo": u"f18-updates", "updates": [u"bodhi-2.0-1.fc18"]}, topic="mashtask.mashing" ), ) self.assertEquals( calls[4], mock.call(force=True, msg={"success": True, "repo": "f18-updates"}, topic="mashtask.complete") ) self.assertEquals( calls[5], mock.call( force=True, msg={"repo": u"f17-updates-testing", "updates": [u"bodhi-2.0-1.fc17"]}, topic="mashtask.mashing", ), ) self.assertEquals( calls[-1], mock.call(force=True, msg={"success": True, "repo": "f17-updates-testing"}, topic="mashtask.complete"), )
def new_update(request): """ Save an update. This entails either creating a new update, or editing an existing one. To edit an existing update, the update's original title must be specified in the ``edited`` parameter. """ data = request.validated log.debug('validated = %s' % data) # This has already been validated at this point, but we need to ditch # it since the models don't care about a csrf argument. data.pop('csrf_token') caveats = [] try: releases = set() builds = [] # Create the Package and Build entities for nvr in data['builds']: name, version, release = request.buildinfo[nvr]['nvr'] package = request.db.query(Package).filter_by(name=name).first() if not package: package = Package(name=name) request.db.add(package) request.db.flush() build = Build.get(nvr, request.db) if build is None: log.debug("Adding nvr %s", nvr) build = Build(nvr=nvr, package=package) request.db.add(build) request.db.flush() build.package = package build.release = request.buildinfo[build.nvr]['release'] builds.append(build) releases.add(request.buildinfo[build.nvr]['release']) if data.get('edited'): log.info('Editing update: %s' % data['edited']) assert len(releases) == 1, "Updates may not span multiple releases" data['release'] = list(releases)[0] data['builds'] = [b.nvr for b in builds] result, _caveats = Update.edit(request, data) caveats.extend(_caveats) else: if len(releases) > 1: caveats.append({ 'name': 'releases', 'description': 'Your update is being split ' 'into %i, one for each release.' % len(releases) }) updates = [] for release in releases: _data = copy.copy(data) # Copy it because .new(..) mutates it _data['builds'] = [b for b in builds if b.release == release] _data['release'] = release log.info('Creating new update: %r' % _data['builds']) result, _caveats = Update.new(request, _data) log.debug('%s update created', result.title) updates.append(result) caveats.extend(_caveats) if len(releases) > 1: result = dict(updates=updates) except LockedUpdateException as e: log.warn(str(e)) request.errors.add('body', 'builds', "%s" % str(e)) return except Exception as e: log.exception('Failed to create update') request.errors.add('body', 'builds', 'Unable to create update. %s' % str(e)) return # Obsolete older updates for three different cases... # editing an update, submitting a new single update, submitting multiple. if isinstance(result, dict): updates = result['updates'] else: updates = [result] for update in updates: try: caveats.extend(update.obsolete_older_updates(request.db)) except Exception as e: caveats.append({ 'name': 'update', 'description': 'Problem obsoleting older updates: %s' % str(e), }) if not isinstance(result, dict): result = result.__json__() result['caveats'] = caveats return result
def test_metadata_updating_with_old_testing_security(self): update = self.db.query(Update).one() update.request = None update.type = UpdateType.security update.status = UpdateStatus.testing update.date_pushed = datetime.utcnow() DevBuildsys.__tagged__[update.title] = ['f17-updates-testing'] # Generate the XML md = ExtendedMetadata(update.release, UpdateRequest.testing, self.db, self.temprepo) # Insert the updateinfo.xml into the repository md.insert_updateinfo() md.cache_repodata() updateinfo = self._verify_updateinfo(self.repodata) # Read an verify the updateinfo.xml.gz uinfo = createrepo_c.UpdateInfo(updateinfo) notice = self.get_notice(uinfo, update.title) self.assertIsNotNone(notice) # Create a new non-security update for the same package newbuild = 'bodhi-2.0-2.fc17' pkg = self.db.query(Package).filter_by(name=u'bodhi').one() build = Build(nvr=newbuild, package=pkg) self.db.add(build) self.db.flush() newupdate = Update(title=newbuild, type=UpdateType.enhancement, status=UpdateStatus.testing, request=None, release=update.release, builds=[build], notes=u'x') newupdate.assign_alias() self.db.add(newupdate) self.db.flush() # Untag the old security build del (DevBuildsys.__tagged__[update.title]) DevBuildsys.__untag__.append(update.title) DevBuildsys.__tagged__[newupdate.title] = [ newupdate.release.testing_tag ] buildrpms = DevBuildsys.__rpms__[0].copy() buildrpms['nvr'] = 'bodhi-2.0-2.fc17' buildrpms['release'] = '2.fc17' DevBuildsys.__rpms__.append(buildrpms) del (DevBuildsys.__rpms__[0]) # Re-initialize our temporary repo shutil.rmtree(self.temprepo) os.mkdir(self.temprepo) mkmetadatadir(join(self.temprepo, 'f17-updates-testing', 'i386')) md = ExtendedMetadata(update.release, UpdateRequest.testing, self.db, self.temprepo) md.insert_updateinfo() updateinfo = self._verify_updateinfo(self.repodata) # Read an verify the updateinfo.xml.gz uinfo = createrepo_c.UpdateInfo(updateinfo) self.assertEquals(len(uinfo.updates), 1) notice = self.get_notice(uinfo, 'bodhi-2.0-1.fc17') self.assertIsNone(notice) notice = self.get_notice(uinfo, 'bodhi-2.0-2.fc17') self.assertIsNotNone(notice)