def list_tagged(tag):
koji = get_session()
builds = koji.listTagged(tag)
nvrs = [build['nvr'] for build in builds]
print "Fetched %d builds tagged with %s" % (
len(builds), tag)
return nvrs
def setUp(self):
fd, self.db_filename = tempfile.mkstemp(prefix="bodhi-testing-", suffix=".db")
db_path = "sqlite:///%s" % self.db_filename
# The BUILD_ID environment variable is set by Jenkins and allows us to
# detect if
# we are running the tests in jenkins or not
# https://wiki.jenkins-ci.org/display/JENKINS/Building+a+software+project#Buildingasoftwareproject-below
if os.environ.get("BUILD_ID"):
faitout = "http://209.132.184.152/faitout/"
try:
import requests
req = requests.get("%s/new" % faitout)
if req.status_code == 200:
db_path = req.text
print "Using faitout at: %s" % db_path
except:
pass
engine = create_engine(db_path)
Base.metadata.create_all(engine)
self.db_factory = transactional_session_maker(engine)
with self.db_factory() as session:
populate(session)
assert session.query(Update).count() == 1
self.koji = buildsys.get_session()
self.koji.clear() # clear out our dev introspection
self.msg = makemsg()
self.tempdir = tempfile.mkdtemp("bodhi")
self.masher = Masher(FakeHub(), db_factory=self.db_factory, mash_dir=self.tempdir)
def compare_builds(testing_build, stable_build, untag, tag):
if stable_build['package_name'] == testing_build['package_name']:
if rpm.labelCompare((str(testing_build['epoch']),
testing_build['version'],
testing_build['release']),
(str(stable_build['epoch']),
stable_build['version'],
stable_build['release'])) < 0:
print "%s is older than %s" % (testing_build['nvr'],
stable_build['nvr'])
try:
build = PackageBuild.byNvr(testing_build['nvr'])
for update in build.updates:
if update.status != 'stable':
print "%s not stable in bodhi!" % update.title
raise SQLObjectNotFound
else:
pass # TODO: do the untagging?
except SQLObjectNotFound:
if untag:
print "Untagging via koji"
koji = get_session()
koji.untagBuild(tag, testing_build['nvr'], force=True)
else:
print "Need to untag koji build %s" % testing_build['nvr']
def compare_builds(testing_build, stable_build, untag, tag):
if stable_build['package_name'] == testing_build['package_name']:
if rpm.labelCompare(
(str(testing_build['epoch']), testing_build['version'],
testing_build['release']),
(str(stable_build['epoch']), stable_build['version'],
stable_build['release'])) < 0:
print "%s is older than %s" % (testing_build['nvr'],
stable_build['nvr'])
try:
build = PackageBuild.byNvr(testing_build['nvr'])
for update in build.updates:
if update.status != 'testing':
print "%s not testing in bodhi!" % update.title
raise SQLObjectNotFound
else:
if untag:
print "Obsoleting via bodhi"
update.obsolete(newer=stable_build['nvr'])
else:
print "Need to obsolete via bodhi"
except SQLObjectNotFound:
if untag:
print "Untagging via koji"
koji = get_session()
koji.untagBuild(tag, testing_build['nvr'], force=True)
else:
print "Need to untag koji build %s" % testing_build['nvr']
def compare_builds(testing_build, stable_build, untag, tag):
if stable_build["package_name"] == testing_build["package_name"]:
if (
rpm.labelCompare(
(str(testing_build["epoch"]), testing_build["version"], testing_build["release"]),
(str(stable_build["epoch"]), stable_build["version"], stable_build["release"]),
)
< 0
):
print "%s is older than %s" % (testing_build["nvr"], stable_build["nvr"])
try:
build = PackageBuild.byNvr(testing_build["nvr"])
for update in build.updates:
if update.status != "testing":
print "%s not testing in bodhi!" % update.title
raise SQLObjectNotFound
else:
if untag:
print "Obsoleting via bodhi"
update.obsolete(newer=stable_build["nvr"])
else:
print "Need to obsolete via bodhi"
except SQLObjectNotFound:
if untag:
print "Untagging via koji"
koji = get_session()
koji.untagBuild(tag, testing_build["nvr"], force=True)
else:
print "Need to untag koji build %s" % testing_build["nvr"]
def setUp(self):
fd, self.db_filename = tempfile.mkstemp(prefix='bodhi-testing-',
suffix='.db')
db_path = 'sqlite:///%s' % self.db_filename
# The BUILD_ID environment variable is set by Jenkins and allows us to
# detect if
# we are running the tests in jenkins or not
# https://wiki.jenkins-ci.org/display/JENKINS/Building+a+software+project#Buildingasoftwareproject-below
if os.environ.get('BUILD_ID'):
faitout = 'http://209.132.184.152/faitout/'
try:
import requests
req = requests.get('%s/new' % faitout)
if req.status_code == 200:
db_path = req.text
print 'Using faitout at: %s' % db_path
except:
pass
engine = create_engine(db_path)
Base.metadata.create_all(engine)
self.db_factory = transactional_session_maker(engine)
with self.db_factory() as session:
populate(session)
assert session.query(Update).count() == 1
self.koji = buildsys.get_session()
self.koji.clear() # clear out our dev introspection
self.msg = makemsg()
self.tempdir = tempfile.mkdtemp('bodhi')
self.masher = Masher(FakeHub(),
db_factory=self.db_factory,
mash_dir=self.tempdir)
def __init__(self, id, updates, repos=None, resume=False):
""" Initialize a new MashTask thread.
@param updates: a list of PackageUpdate objects that we want to push
@param repos: a list of repositories to compose updates{,-testing)
@param resume: resume this set of updates based on the assumption that
we have already tagged them appropriately, and that all we need to do
is mash the repositories, close bugs, and send out update notices.
"""
Thread.__init__(self)
repos = repos and set(repos) or set()
log.debug("MashTask(%d, [%d updates], %s, %s)" % (id, len(updates), repos, resume))
self.id = id
self.tag = None
self.updates = set()
if updates and isinstance(updates[0], basestring):
updates = map(PackageUpdate.byTitle, updates)
for update in updates:
if update.status == "obsolete":
log.warning("Skipping obsolete update %s" % update.title)
continue
if not update.request:
log.warning("Skipping update without request: %s" % update.title)
continue
self.updates.add(update)
if not resume:
update.comment(
"This update is currently being pushed to the %s %s updates repository."
% (update.release.long_name, update.request),
author="bodhi",
email=False,
)
if self.updates:
up = self.updates.pop()
self.updates.add(up)
# eg: MASHING-FEDORA, MASHING-FEDORA-EPEL
self.mash_lock_id = up.release.id_prefix
else:
self.mash_lock_id = "UNKNOWN"
self.koji = buildsys.get_session()
# which repos do we want to compose? (updates|updates-testing)
self.repos = repos
self.success = False
self.cmd = "mash -o %s -c " + config.get("mash_conf") + " -f %s "
self.actions = [] # [(title, current_tag, new_tag), ...]
self.mashing = False # are we currently mashing?
self.moving = False # are we currently moving build tags?
self.log = None # filename that we wrote mash output to
self.mashed_repos = {} # { repo: mashed_dir }
self.errors = []
self.safe = True
self.genmd = False
self.resume = resume
self.testing_digest = {}
self.composed_repos = [] # previously mashed repos from our MASHING lock
self._lock()
self._find_repos()
def __init__(self, repo, cacheduinfo=None):
self.tag = get_repo_tag(repo)
self.repo = repo
self.doc = None
self.updates = set()
self.builds = {}
self._from = config.get('bodhi_email')
self.koji = get_session()
self._create_document()
self._fetch_updates()
missing_ids = []
if cacheduinfo and exists(cacheduinfo):
log.debug("Loading cached updateinfo.xml.gz")
umd = UpdateMetadata()
umd.add(cacheduinfo)
existing_ids = set([up['update_id'] for up in umd.get_notices()])
# Generate metadata for any new builds
for update in self.updates:
if update.updateid:
self.add_update(update)
log.debug('Adding new update notice: %s' % update.title)
if update.updateid in existing_ids:
existing_ids.remove(update.updateid)
else:
missing_ids.append(update.title)
# Add all relevant notices from the metadata to this document
for notice in umd.get_notices():
if notice['update_id'] in existing_ids:
log.debug("Adding existing notice: %s" % notice['title'])
self._add_notice(notice)
existing_ids.remove(notice['update_id'])
else:
if notice['type'] == 'security':
log.debug("Adding existing security notice: %s" %
notice['title'])
self._add_notice(notice)
else:
log.debug("Removing %s from updateinfo" % notice['title'])
else:
log.debug("Generating new updateinfo.xml")
for update in self.updates:
if update.updateid:
self.add_update(update)
else:
missing_ids.append(update.title)
# Add *all* security updates
# TODO: only the most recent
#for update in PackageUpdate.select(PackageUpdate.q.type=='security'):
# self.add_update(update)
if missing_ids:
log.error("%d updates with missing ID!" % len(missing_ids))
log.debug(missing_ids)
def test_unpush_build(self):
eq_(len(self.obj.builds), 1)
b = self.obj.builds[0]
release = self.obj.release
koji = buildsys.get_session()
koji.clear()
koji.__tagged__[b.nvr] = [release.testing_tag,
release.pending_testing_tag]
self.obj.builds[0].unpush(koji)
eq_(koji.__moved__, [(u'dist-f11-updates-testing', u'dist-f11-updates-candidate', u'TurboGears-1.0.8-3.fc11')])
eq_(koji.__untag__, [(u'dist-f11-updates-testing-pending', u'TurboGears-1.0.8-3.fc11')])
def __init__(self, repo, cacheduinfo=None):
self.tag = get_repo_tag(repo)
self.repo = repo
self.doc = None
self.updates = set()
self.builds = {}
self._from = config.get('bodhi_email')
self.koji = get_session()
self._create_document()
self._fetch_updates()
missing_ids = []
if cacheduinfo and exists(cacheduinfo):
log.debug("Loading cached updateinfo.xml.gz")
umd = UpdateMetadata()
umd.add(cacheduinfo)
# Generate metadata for any new builds
for update in self.updates:
for build in update.builds:
if not umd.get_notice(build.nvr):
if update.updateid:
self.add_update(update)
break
else:
missing_ids.append(update.title)
# Add all relevant notices from the metadata to this document
ids = [
update.updateid for update in self.updates if update.updateid
]
for notice in umd.get_notices():
if notice['update_id'] in ids or notice['type'] == 'security':
self._add_notice(notice)
else:
log.debug("Removing %s from updateinfo" % notice['title'])
else:
log.debug("Generating new updateinfo.xml")
for update in self.updates:
if update.updateid:
self.add_update(update)
else:
missing_ids.append(update.title)
# Add *all* security updates
# TODO: only the most recent
#for update in PackageUpdate.select(PackageUpdate.q.type=='security'):
# self.add_update(update)
if missing_ids:
log.error("%d updates with missing ID!" % len(missing_ids))
log.debug(missing_ids)
def __init__(self, release, request, db, path):
self.repo = path
log.debug('repo = %r' % self.repo)
self.request = request
if request is UpdateRequest.stable:
self.tag = release.stable_tag
else:
self.tag = release.testing_tag
self.repo_path = os.path.join(self.repo, self.tag)
self.db = db
self.updates = set()
self.builds = {}
self.missing_ids = []
self._from = config.get('bodhi_email')
self.koji = get_session()
self._fetch_updates()
self.uinfo = cr.UpdateInfo()
self.hash_type = cr.SHA256
self.comp_type = cr.XZ
if release.id_prefix == u'FEDORA-EPEL':
# yum on py2.4 doesn't support sha256 (#1080373)
if 'el5' in self.repo or '5E' in self.repo:
self.hash_type = cr.SHA1
self.comp_type = cr.GZ
else:
# FIXME: I'm not sure which versions of RHEL support xz metadata
# compression, so use the lowest common denominator for now.
self.comp_type = cr.BZ2
# Load from the cache if it exists
self.cached_repodata = os.path.join(self.repo, '..',
self.tag + '.repocache',
'repodata/')
if os.path.isdir(self.cached_repodata):
self._load_cached_updateinfo()
else:
log.debug("Generating new updateinfo.xml")
self.uinfo = cr.UpdateInfo()
for update in self.updates:
if update.alias:
self.add_update(update)
else:
self.missing_ids.append(update.title)
if self.missing_ids:
log.error("%d updates with missing ID: %r" %
(len(self.missing_ids), self.missing_ids))
def _fetch_candidate_builds(self, pkg):
""" Return all candidate builds for a given package """
matches = {}
koji = get_session()
koji.multicall = True
for tag in [r.candidate_tag for r in Release.select()]:
koji.getLatestBuilds(tag, package=pkg)
results = koji.multiCall()
for result in results:
for entries in result:
for entry in entries:
matches[entry['nvr']] = entry['completion_time']
return [build[0] for build in
sorted(matches.items(), key=itemgetter(1), reverse=True)]
def __init__(self, release, request, db, path):
self.repo = path
log.debug('repo = %r' % self.repo)
self.request = request
if request is UpdateRequest.stable:
self.tag = release.stable_tag
else:
self.tag = release.testing_tag
self.repo_path = os.path.join(self.repo, self.tag)
self.db = db
self.updates = set()
self.builds = {}
self.missing_ids = []
self._from = config.get('bodhi_email')
self.koji = get_session()
self._fetch_updates()
self.uinfo = cr.UpdateInfo()
self.hash_type = cr.SHA256
self.comp_type = cr.XZ
if release.id_prefix == u'FEDORA-EPEL':
# yum on py2.4 doesn't support sha256 (#1080373)
if 'el5' in self.repo or '5E' in self.repo:
self.hash_type = cr.SHA1
self.comp_type = cr.GZ
else:
# FIXME: I'm not sure which versions of RHEL support xz metadata
# compression, so use the lowest common denominator for now.
self.comp_type = cr.BZ2
# Load from the cache if it exists
self.cached_repodata = os.path.join(self.repo, '..', self.tag +
'.repocache', 'repodata/')
if os.path.isdir(self.cached_repodata):
self._load_cached_updateinfo()
else:
log.debug("Generating new updateinfo.xml")
self.uinfo = cr.UpdateInfo()
for update in self.updates:
if update.alias:
self.add_update(update)
else:
self.missing_ids.append(update.title)
if self.missing_ids:
log.error("%d updates with missing ID: %r" % (
len(self.missing_ids), self.missing_ids))
def test_set_request_untested_stable(self):
"""
Ensure that we can't submit an update for stable if it hasn't met the
minimum testing requirements.
"""
req = DummyRequest(user=DummyUser())
req.errors = cornice.Errors()
req.koji = buildsys.get_session()
eq_(self.obj.status, UpdateStatus.pending)
try:
self.obj.set_request(self.db, UpdateRequest.stable, req.user.name)
assert False
except BodhiException, e:
pass
def __init__(self, id, updates, repos=None, resume=False):
""" Initialize a new MashTask thread.
@param updates: a list of PackageUpdate objects that we want to push
@param repos: a list of repositories to compose updates{,-testing)
@param resume: resume this set of updates based on the assumption that
we have already tagged them appropriately, and that all we need to do
is mash the repositories, close bugs, and send out update notices.
"""
Thread.__init__(self)
repos = repos and set(repos) or set()
log.debug("MashTask(%d, %s, %s, %s)" % (id, updates, repos, resume))
self.id = id
self.tag = None
self.updates = set()
if updates and isinstance(updates[0], basestring):
updates = map(PackageUpdate.byTitle, updates)
for update in updates:
if update.status == 'obsolete':
log.warning("Skipping obsolete update %s" % update.title)
continue
if not update.request:
log.warning('Skipping update without request: %s' %update.title)
continue
self.updates.add(update)
if self.updates:
up = self.updates.pop()
self.updates.add(up)
# eg: MASHING-FEDORA, MASHING-FEDORA-EPEL
self.mash_lock_id = up.release.id_prefix
else:
self.mash_lock_id = 'UNKNOWN'
self.koji = buildsys.get_session()
# which repos do we want to compose? (updates|updates-testing)
self.repos = repos
self.success = False
self.cmd = 'mash -o %s -c ' + config.get('mash_conf') + ' -f %s '
self.actions = [] # [(title, current_tag, new_tag), ...]
self.mashing = False # are we currently mashing?
self.moving = False # are we currently moving build tags?
self.log = None # filename that we wrote mash output to
self.mashed_repos = {} # { repo: mashed_dir }
self.errors = []
self.safe = True
self.genmd = False
self.resume = resume
self.testing_digest = {}
self.composed_repos = [] # previously mashed repos from our MASHING lock
self._lock()
self._find_repos()
def test_set_request_untested_stable(self):
"""
Ensure that we can't submit an update for stable if it hasn't met the
minimum testing requirements.
"""
req = DummyRequest(user=DummyUser())
req.errors = cornice.Errors()
req.koji = buildsys.get_session()
eq_(self.obj.status, UpdateStatus.pending)
try:
self.obj.set_request(self.db, UpdateRequest.stable, req.user.name)
assert False
except BodhiException, e:
pass
def _fetch_candidate_builds(self, pkg):
""" Return all candidate builds for a given package """
matches = {}
koji = get_session()
koji.multicall = True
for tag in [r.candidate_tag for r in Release.select()]:
koji.listTagged(tag, package=pkg)
results = koji.multiCall()
for result in results:
for entries in result:
for entry in entries:
matches[entry['nvr']] = entry['completion_time']
return [build[0] for build in
sorted(matches.items(), key=itemgetter(1), reverse=True)]
def test_unpush_build(self):
eq_(len(self.obj.builds), 1)
b = self.obj.builds[0]
release = self.obj.release
koji = buildsys.get_session()
koji.clear()
koji.__tagged__[b.nvr] = [
release.testing_tag, release.pending_testing_tag
]
self.obj.builds[0].unpush(koji)
eq_(koji.__moved__,
[(u'dist-f11-updates-testing', u'dist-f11-updates-candidate',
u'TurboGears-1.0.8-3.fc11')])
eq_(koji.__untag__, [
(u'dist-f11-updates-testing-pending', u'TurboGears-1.0.8-3.fc11')
])
def clean_stable_builds(untag=False):
koji = get_session()
for release in Release.select():
latest_stable_builds = koji.listTagged(release.stable_tag, latest=True)
latest_stable_nvrs = [build['nvr'] for build in latest_stable_builds]
print "Fetched %d latest stable builds tagged with %s" % (
len(latest_stable_builds), release.stable_tag)
stable_builds = koji.listTagged(release.stable_tag)
stable_nvrs = [build['nvr'] for build in stable_builds]
print "Fetched %d stable builds tagged with %s" % (len(stable_builds),
release.stable_tag)
for latest_build in latest_stable_builds:
for build in stable_builds:
if build['nvr'] == latest_build['nvr']:
continue
compare_builds(latest_build, build, untag, release.stable_tag)
def clean_stable_builds(untag=False):
koji = get_session()
for release in Release.select():
latest_stable_builds = koji.listTagged(release.stable_tag, latest=True)
latest_stable_nvrs = [build['nvr'] for build in latest_stable_builds]
print "Fetched %d latest stable builds tagged with %s" % (
len(latest_stable_builds), release.stable_tag)
stable_builds = koji.listTagged(release.stable_tag)
stable_nvrs = [build['nvr'] for build in stable_builds]
print "Fetched %d stable builds tagged with %s" % (
len(stable_builds), release.stable_tag)
for latest_build in latest_stable_builds:
for build in stable_builds:
if build['nvr'] == latest_build['nvr']:
continue
compare_builds(latest_build, build, untag, release.stable_tag)
def clean_testing_builds(untag=False):
koji = get_session()
for release in Release.select():
stable_builds = koji.listTagged(release.stable_tag, latest=True)
stable_nvrs = [build["nvr"] for build in stable_builds]
print "Fetched %d builds tagged with %s" % (len(stable_builds), release.stable_tag)
testing_builds = koji.listTagged(release.testing_tag, latest=True)
print "Fetched %d builds tagged with %s" % (len(testing_builds), release.testing_tag)
testing_nvrs = [build["nvr"] for build in testing_builds]
for testing_build in testing_builds:
for build in testing_builds:
compare_builds(testing_build, build, untag, release.testing_tag)
for build in stable_builds:
compare_builds(testing_build, build, untag, release.testing_tag)
# Find testing updates that aren't in the list of latest builds
for update in PackageUpdate.select(
AND(
PackageUpdate.q.releaseID == release.id,
PackageUpdate.q.status == "testing",
PackageUpdate.q.request == None,
)
):
for build in update.builds:
if build.nvr not in testing_nvrs:
latest_testing = None
latest_stable = None
for testing in testing_nvrs:
if testing.startswith(build.package.name + "-"):
latest_testing = testing
break
for stable in stable_nvrs:
if stable.startswith(build.package.name + "-"):
latest_stable = stable
break
if latest_testing:
koji_build = koji.getBuild(build.nvr)
latest_build = koji.getBuild(latest_testing)
if rpm.labelCompare(build_evr(koji_build), build_evr(latest_build)) < 0:
print "%s in testing, latest_testing = %s, latest_stable = %s" % (
update.title,
latest_testing,
latest_stable,
)
if untag:
print "Obsoleting %s" % update.title
update.obsolete(newer=latest_testing)
def save(self, builds, notes, expiration=None, *args, **kw):
log.debug(repr(locals()))
last_release = None # for our koji wait-repo example
if expiration:
if datetime.utcnow() > expiration:
flash('Cannot set an expiration in the past')
if request_format() == 'json': return dict()
raise redirect('/override/new')
try:
koji = get_session()
except Exception, e:
flash('Unable to connect to Koji')
if request_format() == 'json':
return dict()
raise redirect('/override/new')
def clean_testing_builds(untag=False):
koji = get_session()
for release in Release.select():
stable_builds = koji.listTagged(release.stable_tag, latest=True)
stable_nvrs = [build['nvr'] for build in stable_builds]
print "Fetched %d builds tagged with %s" % (len(stable_builds),
release.stable_tag)
testing_builds = koji.listTagged(release.testing_tag, latest=True)
print "Fetched %d builds tagged with %s" % (len(testing_builds),
release.testing_tag)
testing_nvrs = [build['nvr'] for build in testing_builds]
for testing_build in testing_builds:
for build in testing_builds:
compare_builds(testing_build, build, untag,
release.testing_tag)
for build in stable_builds:
compare_builds(testing_build, build, untag,
release.testing_tag)
# Find testing updates that aren't in the list of latest builds
for update in PackageUpdate.select(
AND(PackageUpdate.q.releaseID == release.id,
PackageUpdate.q.status == 'testing',
PackageUpdate.q.request == None)):
for build in update.builds:
if build.nvr not in testing_nvrs:
latest_testing = None
latest_stable = None
for testing in testing_nvrs:
if testing.startswith(build.package.name + '-'):
latest_testing = testing
break
for stable in stable_nvrs:
if stable.startswith(build.package.name + '-'):
latest_stable = stable
break
if latest_testing:
koji_build = koji.getBuild(build.nvr)
latest_build = koji.getBuild(latest_testing)
if rpm.labelCompare(build_evr(koji_build),
build_evr(latest_build)) < 0:
print "%s in testing, latest_testing = %s, latest_stable = %s" % (
update.title, latest_testing, latest_stable)
if untag:
print "Obsoleting %s" % update.title
update.obsolete(newer=latest_testing)
def save(self, builds, notes, expiration=None, *args, **kw):
log.debug(repr(locals()))
last_release = None # for our koji wait-repo example
if expiration:
if datetime.utcnow() > expiration:
flash('Cannot set an expiration in the past')
if request_format() == 'json': return dict()
raise redirect('/override/new')
try:
koji = get_session()
except Exception, e:
flash('Unable to connect to Koji')
if request_format() == 'json':
return dict()
raise redirect('/override/new')
def main(argv=sys.argv):
if len(argv) != 2:
usage(argv)
config_uri = argv[1]
setup_logging(config_uri)
log = logging.getLogger(__name__)
settings = get_appsettings(config_uri)
engine = engine_from_config(settings, 'sqlalchemy.')
Session = scoped_session(
sessionmaker(extension=ZopeTransactionExtension()))
Session.configure(bind=engine)
db = Session()
koji = buildsys.get_session()
one_day = timedelta(days=1)
now = datetime.utcnow()
with transaction.manager:
for release in db.query(Release).filter_by(
state=ReleaseState.pending).all():
log.info(release.name)
for update in db.query(Update).filter_by(
release=release, status=UpdateStatus.stable).all():
assert update.date_stable, update.title
if now - update.date_stable > one_day:
for build in update.builds:
tags = build.get_tags()
stable_tag = release.dist_tag
testing_tag = release.testing_tag
pending_testing_tag = Release.pending_testing_tag
if stable_tag not in tags:
log.error('%s not tagged as stable %s' %
(build.nvr, tags))
continue
if testing_tag in tags:
log.info('Removing %s from %s' %
(testing_tag, build.nvr))
koji.untagBuild(testing_tag, build.nvr)
if pending_testing_tag in tags:
log.info('Removing %s from %s' %
(pending_testing_tag, build.nvr))
koji.untagBuild(pending_testing_tag, build.nvr)
def test_set_request_stable_after_week_in_testing(self, publish):
req = DummyRequest()
req.errors = cornice.Errors()
req.koji = buildsys.get_session()
req.user = model.User(name='bob')
self.obj.status = UpdateStatus.testing
self.obj.request = None
# Pretend it's been in testing for a week
self.obj.comment(
self.db, u'This update has been pushed to testing.', author=u'bodhi')
self.obj.date_testing = self.obj.comments[-1].timestamp - timedelta(days=7)
eq_(self.obj.days_in_testing, 7)
eq_(self.obj.meets_testing_requirements, True)
self.obj.set_request(self.db, UpdateRequest.stable, req)
eq_(self.obj.request, UpdateRequest.stable)
eq_(len(req.errors), 0)
publish.assert_called_once_with(
topic='update.request.stable', msg=mock.ANY)
def main(argv=sys.argv):
if len(argv) != 2:
usage(argv)
config_uri = argv[1]
setup_logging(config_uri)
log = logging.getLogger(__name__)
settings = get_appsettings(config_uri)
engine = engine_from_config(settings, 'sqlalchemy.')
Session = scoped_session(sessionmaker(extension=ZopeTransactionExtension()))
Session.configure(bind=engine)
db = Session()
koji = buildsys.get_session()
one_day = timedelta(days=1)
now = datetime.utcnow()
with transaction.manager:
for release in db.query(Release).filter_by(
state=ReleaseState.pending).all():
log.info(release.name)
for update in db.query(Update).filter_by(
release=release, status=UpdateStatus.stable).all():
assert update.date_stable, update.title
if now - update.date_stable > one_day:
for build in update.builds:
tags = build.get_tags()
stable_tag = release.dist_tag
testing_tag = release.testing_tag
pending_testing_tag = Release.pending_testing_tag
if stable_tag not in tags:
log.error('%s not tagged as stable %s' % (build.nvr, tags))
continue
if testing_tag in tags:
log.info('Removing %s from %s' % (testing_tag, build.nvr))
koji.untagBuild(testing_tag, build.nvr)
if pending_testing_tag in tags:
log.info('Removing %s from %s' % (pending_testing_tag, build.nvr))
koji.untagBuild(pending_testing_tag, build.nvr)
def test_set_meets_then_met_requirements(self, publish):
req = DummyRequest()
req.errors = cornice.Errors()
req.koji = buildsys.get_session()
req.user = model.User(name='bob')
self.obj.status = UpdateStatus.testing
self.obj.request = None
# Pretend it's been in testing for a week
self.obj.comment(
self.db, u'This update has been pushed to testing.', author=u'bodhi')
self.obj.date_testing = self.obj.comments[-1].timestamp - timedelta(days=7)
eq_(self.obj.days_in_testing, 7)
eq_(self.obj.meets_testing_requirements, True)
eq_(self.obj.met_testing_requirements, False)
text = config.get('testing_approval_msg') % self.obj.days_in_testing
self.obj.comment(self.db, text, author=u'bodhi')
eq_(self.obj.meets_testing_requirements, True)
eq_(self.obj.met_testing_requirements, True)
def test_set_request_stable_after_week_in_testing(self, publish):
req = DummyRequest()
req.errors = cornice.Errors()
req.koji = buildsys.get_session()
req.user = model.User(name='bob')
self.obj.status = UpdateStatus.testing
self.obj.request = None
# Pretend it's been in testing for a week
self.obj.comment(self.db,
u'This update has been pushed to testing.',
author=u'bodhi')
self.obj.date_testing = self.obj.comments[-1].timestamp - timedelta(
days=7)
eq_(self.obj.days_in_testing, 7)
eq_(self.obj.meets_testing_requirements, True)
self.obj.set_request(self.db, UpdateRequest.stable, req)
eq_(self.obj.request, UpdateRequest.stable)
eq_(len(req.errors), 0)
publish.assert_called_once_with(topic='update.request.stable',
msg=mock.ANY)
def clean_pending_tags():
""" Clean up any stray pending tags """
koji = get_session()
for release in Release.select():
log.info("Finding all stray pending-testing builds...")
if release.name.startswith('E'):
continue
tag = release.pending_testing_tag
tagged = [build['nvr'] for build in koji.listTagged(tag)]
for nvr in tagged:
try:
build = PackageBuild.byNvr(nvr)
for update in build.updates:
if update.status in ('testing', 'stable', 'obsolete'):
log.info("%s %s" % (nvr, update.status))
log.info("Untagging %s" % nvr)
koji.untagBuild(tag, nvr, force=True)
except SQLObjectNotFound:
log.info("Can't find build for %s" % nvr)
log.info("Untagging %s" % nvr)
koji.untagBuild(tag, nvr, force=True)
log.info("Finding all stray pending-stable builds...")
tag = release.pending_stable_tag
tagged = [build['nvr'] for build in koji.listTagged(tag)]
for nvr in tagged:
try:
build = PackageBuild.byNvr(nvr)
for update in build.updates:
if update.status in ('pending', 'obsolete', 'stable'):
log.info("%s %s" % (nvr, update.status))
log.info("Untagging %s" % nvr)
koji.untagBuild(tag, nvr, force=True)
except SQLObjectNotFound:
log.info("Untagging %s" % nvr)
koji.untagBuild(tag, nvr, force=True)
def clean_pending_tags():
""" Clean up any stray pending tags """
koji = get_session()
for release in Release.select():
log.info("Finding all stray pending-testing builds...")
if release.name.startswith('EL'):
continue
tag = release.pending_testing_tag
tagged = [build['nvr'] for build in koji.listTagged(tag)]
for nvr in tagged:
try:
build = PackageBuild.byNvr(nvr)
for update in build.updates:
if update.status in ('testing', 'stable', 'obsolete'):
log.info("%s %s" % (nvr, update.status))
log.info("Untagging %s" % nvr)
koji.untagBuild(tag, nvr, force=True)
except SQLObjectNotFound:
log.info("Can't find build for %s" % nvr)
log.info("Untagging %s" % nvr)
koji.untagBuild(tag, nvr, force=True)
log.info("Finding all stray pending-stable builds...")
tag = release.pending_stable_tag
tagged = [build['nvr'] for build in koji.listTagged(tag)]
for nvr in tagged:
try:
build = PackageBuild.byNvr(nvr)
for update in build.updates:
if update.status in ('pending', 'obsolete', 'stable'):
log.info("%s %s" % (nvr, update.status))
log.info("Untagging %s" % nvr)
koji.untagBuild(tag, nvr, force=True)
except SQLObjectNotFound:
log.info("Untagging %s" % nvr)
koji.untagBuild(tag, nvr, force=True)
def test_set_meets_then_met_requirements(self, publish):
req = DummyRequest()
req.errors = cornice.Errors()
req.koji = buildsys.get_session()
req.user = model.User(name='bob')
self.obj.status = UpdateStatus.testing
self.obj.request = None
# Pretend it's been in testing for a week
self.obj.comment(self.db,
u'This update has been pushed to testing.',
author=u'bodhi')
self.obj.date_testing = self.obj.comments[-1].timestamp - timedelta(
days=7)
eq_(self.obj.days_in_testing, 7)
eq_(self.obj.meets_testing_requirements, True)
eq_(self.obj.met_testing_requirements, False)
text = config.get('testing_approval_msg') % self.obj.days_in_testing
self.obj.comment(self.db, text, author=u'bodhi')
eq_(self.obj.meets_testing_requirements, True)
eq_(self.obj.met_testing_requirements, True)
def test_extended_metadata_updating_with_edited_updates(self):
testutil.capture_log(['bodhi.metadata'])
# grab the name of a build in updates-testing, and create it in our db
koji = get_session()
builds = koji.listTagged('dist-f13-updates-testing', latest=True)
# Create all of the necessary database entries
release = Release(name='F13', long_name='Fedora 13', id_prefix='FEDORA',
dist_tag='dist-f13')
package = Package(name=builds[0]['package_name'])
update = PackageUpdate(title=builds[0]['nvr'],
release=release,
submitter=builds[0]['owner_name'],
status='testing',
notes='foobar',
type='bugfix')
build = PackageBuild(nvr=builds[0]['nvr'], package=package)
update.addPackageBuild(build)
update.assign_id()
## Initialize our temporary repo
temprepo = join(tempfile.mkdtemp('bodhi'), 'f13-updates-testing')
print "Inserting updateinfo into temprepo: %s" % temprepo
mkmetadatadir(join(temprepo, 'i386'))
repodata = join(temprepo, 'i386', 'repodata')
assert exists(join(repodata, 'repomd.xml'))
## Generate the XML
md = ExtendedMetadata(temprepo)
## Insert the updateinfo.xml into the repository
md.insert_updateinfo()
updateinfo = self.__verify_updateinfo(repodata)
## Read an verify the updateinfo.xml.gz
uinfo = UpdateMetadata()
uinfo.add(updateinfo)
notice = uinfo.get_notice(('mutt', '1.5.14', '1.fc13'))
assert not notice
notice = uinfo.get_notice(get_nvr(update.title))
assert notice
assert notice['status'] == update.status
assert notice['updated'] == update.date_modified
assert notice['from'] == str(config.get('bodhi_email'))
assert notice['description'] == update.notes
assert notice['issued'] is not None
assert notice['update_id'] == update.updateid
assert notice['title'] == update.title
assert notice['release'] == update.release.long_name
## Edit the update and bump the build revision
nvr = 'TurboGears-1.0.2.2-3.fc7'
newbuild = PackageBuild(nvr=nvr, package=package)
update.removePackageBuild(build)
update.addPackageBuild(newbuild)
update.title = nvr
update.date_modified = datetime.utcnow()
# Pretend -2 was unpushed
assert len(koji.__untag__) == 0
koji.__untag__.append('TurboGears-1.0.2.2-2.fc7')
## Test out updateinfo.xml updating via our ExtendedMetadata
md = ExtendedMetadata(temprepo, updateinfo)
md.insert_updateinfo()
updateinfo = self.__verify_updateinfo(repodata)
## Read an verify the updateinfo.xml.gz
uinfo = UpdateMetadata()
uinfo.add(updateinfo)
print(testutil.get_log())
notice = uinfo.get_notice(('TurboGears', '1.0.2.2', '2.fc7'))
assert not notice, "Old TG notice did not get pruned: %s" % notice
notice = uinfo.get_notice(('TurboGears', '1.0.2.2', '3.fc7'))
assert notice, uinfo
assert notice['title'] == update.title
num_notices = len(uinfo.get_notices())
assert num_notices == 1, num_notices
## Clean up
shutil.rmtree(temprepo)
del(koji.__untag__[:])
def tearDown(self):
self.db.close()
get_session().clear()
shutil.rmtree(self.tempdir)
from turbogears import expose, controllers, identity, config, flash
from bodhi.widgets import NewUpdateForm
from bodhi.buildsys import get_session
from bodhi.model import Release
from bodhi.util import url
log = logging.getLogger(__name__)
update_form = NewUpdateForm()
packages = []
# Load the list of packages from koji
if config.get('buildsystem') == 'koji':
try:
koji = get_session()
packages = [pkg['package_name'] for pkg in koji.listPackages()]
log.debug("%d packages loaded from koji" % len(packages))
except Exception, e:
log.exception(e)
log.error("There was a problem loading the package list from koji")
else:
# Resort to looking on the filesystem for the package names
try:
packages = os.listdir(config.get('build_dir'))
except (OSError, TypeError):
log.warning("Warning: build_dir either invalid or not set in app.cfg")
class NewUpdateController(controllers.Controller):
def test_session(self):
from bodhi.buildsys import get_session
session = get_session()
assert session
from turbogears import expose, controllers, identity, config, flash
from bodhi.widgets import NewUpdateForm
from bodhi.buildsys import get_session
from bodhi.model import Release
from bodhi.util import url
log = logging.getLogger(__name__)
update_form = NewUpdateForm()
packages = []
# Load the list of packages from koji
if config.get('buildsystem') == 'koji':
try:
koji = get_session()
packages = [pkg['package_name'] for pkg in koji.listPackages()]
log.debug("%d packages loaded from koji" % len(packages))
except Exception, e:
log.exception(e)
log.error("There was a problem loading the package list from koji")
else:
# Resort to looking on the filesystem for the package names
try:
packages = os.listdir(config.get('build_dir'))
except (OSError, TypeError):
log.warning("Warning: build_dir either invalid or not set in app.cfg")
class NewUpdateController(controllers.Controller):
def test_extended_metadata_updating_with_old_stable_security(self):
testutil.capture_log(['bodhi.metadata'])
koji = get_session()
del (koji.__untag__[:])
assert not koji.__untag__, koji.__untag__
builds = koji.listTagged('dist-f7-updates', latest=True)
# Create all of the necessary database entries
release = Release(name='F17',
long_name='Fedora 17',
id_prefix='FEDORA',
dist_tag='dist-f17')
package = Package(name='TurboGears')
update = PackageUpdate(title='TurboGears-1.0.2.2-2.fc7',
release=release,
submitter=builds[0]['owner_name'],
status='stable',
notes='foobar',
type='security')
build = PackageBuild(nvr='TurboGears-1.0.2.2-2.fc7', package=package)
update.addPackageBuild(build)
update.assign_id()
assert update.updateid
## Initialize our temporary repo
temprepo = join(tempfile.mkdtemp('bodhi'), 'f7-updates')
print "Inserting updateinfo into temprepo: %s" % temprepo
mkmetadatadir(join(temprepo, 'i386'))
repodata = join(temprepo, 'i386', 'repodata')
assert exists(join(repodata, 'repomd.xml'))
## Generate the XML
md = ExtendedMetadata(temprepo)
## Insert the updateinfo.xml into the repository
md.insert_updateinfo()
updateinfo = self.__verify_updateinfo(repodata)
# Create a new non-security update for the same package
newbuild = 'TurboGears-1.0.2.2-3.fc7'
update = PackageUpdate(title=newbuild,
release=release,
submitter=builds[0]['owner_name'],
status='stable',
notes='foobar',
type='enhancement')
build = PackageBuild(nvr=newbuild, package=package)
update.addPackageBuild(build)
update.assign_id()
koji.__untag__.append('TurboGears-1.0.2.2-2.fc7')
## Test out updateinfo.xml updating via our ExtendedMetadata
md = ExtendedMetadata(temprepo, updateinfo)
md.insert_updateinfo()
updateinfo = self.__verify_updateinfo(repodata)
## Read an verify the updateinfo.xml.gz
uinfo = UpdateMetadata()
uinfo.add(updateinfo)
print(testutil.get_log())
assert len(uinfo.get_notices()) == 2, len(uinfo.get_notices())
assert uinfo.get_notice(get_nvr('TurboGears-1.0.2.2-2.fc7'))
notice = uinfo.get_notice(get_nvr(update.title))
assert notice, 'Cannot find update for %r' % get_nvr(update.title)
assert notice['status'] == update.status
assert notice['updated'] == update.date_modified
assert notice['from'] == str(config.get('bodhi_email'))
assert notice['description'] == update.notes
assert notice['issued'] is not None
assert notice['update_id'] == update.updateid
## Clean up
shutil.rmtree(temprepo)
del (koji.__untag__[:])
def work(self):
self.koji = buildsys.get_session()
self.release = self.db.query(Release)\
.filter_by(name=self.release).one()
self.id = getattr(self.release, '%s_tag' % self.request.value)
# Set our thread's "name" so it shows up nicely in the logs.
# https://docs.python.org/2/library/threading.html#thread-objects
self.name = self.id
# For 'pending' branched releases, we only want to perform repo-related
# tasks for testing updates. For stable updates, we should just add the
# dist_tag and do everything else other than mashing/updateinfo, since
# the nightly build-branched cron job mashes for us.
self.skip_mash = False
if (self.release.state is ReleaseState.pending and
self.request is UpdateRequest.stable):
self.skip_mash = True
self.log.info('Running MasherThread(%s)' % self.id)
self.init_state()
if not self.resume:
self.init_path()
notifications.publish(
topic="mashtask.mashing",
msg=dict(repo=self.id, updates=self.state['updates']),
force=True,
)
try:
if self.resume:
self.load_state()
else:
self.save_state()
self.load_updates()
self.verify_updates()
if self.request is UpdateRequest.stable:
self.perform_gating()
self.determine_and_perform_tag_actions()
self.update_security_bugs()
self.expire_buildroot_overrides()
self.remove_pending_tags()
self.update_comps()
if self.resume and self.path in self.state['completed_repos']:
self.log.info('Skipping completed repo: %s', self.path)
self.complete_requests()
# We still need to generate the testing digest, since it's stored in memory
self.generate_testing_digest()
else:
if not self.skip_mash:
mash_thread = self.mash()
# Things we can do while we're mashing
self.complete_requests()
self.generate_testing_digest()
if not self.skip_mash:
uinfo = self.generate_updateinfo()
self.wait_for_mash(mash_thread)
uinfo.insert_updateinfo()
uinfo.insert_pkgtags()
uinfo.cache_repodata()
# Compose OSTrees from our freshly mashed repos
if config.get('compose_atomic_trees'):
self.compose_atomic_trees()
if not self.skip_mash:
self.sanity_check_repo()
self.stage_repo()
# Wait for the repo to hit the master mirror
self.wait_for_sync()
# Send fedmsg notifications
self.send_notifications()
# Update bugzillas
self.modify_bugs()
# Add comments to updates
self.status_comments()
# Announce stable updates to the mailing list
self.send_stable_announcements()
# Email updates-testing digest
self.send_testing_digest()
self.success = True
self.remove_state()
self.unlock_updates()
self.check_all_karma_thresholds()
except:
self.log.exception('Exception in MasherThread(%s)' % self.id)
self.save_state()
raise
finally:
self.finish(self.success)
def tearDown(self):
self.db.close()
get_session().clear()
shutil.rmtree(self.tempdir)
def __init__(self, repo, cacheduinfo=None):
self.tag = get_repo_tag(repo)
self.repo = repo
self.doc = None
self.updates = set()
self.builds = {}
self._from = config.get('bodhi_email')
self.koji = get_session()
self._create_document()
self._fetch_updates()
missing_ids = []
if cacheduinfo and exists(cacheduinfo):
log.debug("Loading cached updateinfo.xml.gz")
umd = UpdateMetadata()
umd.add(cacheduinfo)
# Drop the old cached updateinfo.xml.gz, it's unneeded now
os.unlink(cacheduinfo)
existing_ids = set([up['update_id'] for up in umd.get_notices()])
seen_ids = set()
from_cache = set()
# Generate metadata for any new builds
for update in self.updates:
if update.updateid:
seen_ids.add(update.updateid)
if update.updateid in existing_ids:
notice = umd.get_notice(update.title)
if not notice:
log.warn(
'%s ID in cache but notice cannot be found' %
(update.title))
self.add_update(update)
continue
if notice['updated']:
if datetime.strptime(notice['updated'],
'%Y-%m-%d %H:%M:%S'
) < update.date_modified:
log.debug(
'Update modified, generating new notice: %s'
% update.title)
self.add_update(update)
else:
log.debug('Loading updated %s from cache' %
update.title)
from_cache.add(update.updateid)
elif update.date_modified:
log.debug(
'Update modified, generating new notice: %s' %
update.title)
self.add_update(update)
else:
log.debug('Loading %s from cache' % update.title)
from_cache.add(update.updateid)
else:
log.debug('Adding new update notice: %s' %
update.title)
self.add_update(update)
else:
missing_ids.append(update.title)
# Add all relevant notices from the cache to this document
for notice in umd.get_notices():
if notice['update_id'] in from_cache:
log.debug("Keeping existing notice: %s" % notice['title'])
self._add_notice(notice)
else:
# Keep all security notices in the stable repo
if 'testing' not in self.repo:
if notice['type'] == 'security':
if notice['update_id'] not in seen_ids:
log.debug(
"Keeping existing security notice: %s" %
notice['title'])
self._add_notice(notice)
else:
log.debug('%s already added?' %
notice['title'])
else:
log.debug('Purging cached stable notice %s' %
notice['title'])
else:
log.debug('Purging cached testing update %s' %
notice['title'])
# Clean metadata generation
else:
log.debug("Generating new updateinfo.xml")
for update in self.updates:
if update.updateid:
self.add_update(update)
else:
missing_ids.append(update.title)
# Add *all* security updates
# TODO: only the most recent
#for update in PackageUpdate.select(PackageUpdate.q.type=='security'):
# self.add_update(update)
if missing_ids:
log.error("%d updates with missing ID!" % len(missing_ids))
log.debug(missing_ids)
def main():
load_config()
__connection__ = hub = PackageHub("bodhi")
koji = get_session()
tasks = []
broke = set()
# Clean up any stray pending tags
for release in Release.select():
print "Finding all pending-testing builds..."
if release.name.startswith('EL'):
continue
tag = release.pending_testing_tag
tagged = [build['nvr'] for build in koji.listTagged(tag)]
for nvr in tagged:
try:
build = PackageBuild.byNvr(nvr)
for update in build.updates:
if update.status in ('testing', 'stable', 'obsolete'):
print "%s %s" % (nvr, update.status)
if '--fix' in sys.argv:
print "Untagging %s" % nvr
koji.untagBuild(tag, nvr, force=True)
except SQLObjectNotFound:
print "Can't find build for %s" % nvr
if '--fix' in sys.argv:
print "Untagging %s" % nvr
koji.untagBuild(tag, nvr, force=True)
tag = release.pending_stable_tag
tagged = [build['nvr'] for build in koji.listTagged(tag)]
for nvr in tagged:
try:
build = PackageBuild.byNvr(nvr)
for update in build.updates:
if update.status in ('pending', 'obsolete'):
print "%s %s" % (nvr, update.status)
if '--fix' in sys.argv:
print "Untagging %s" % nvr
koji.untagBuild(tag, nvr, force=True)
except SQLObjectNotFound:
print "Can't find build for %s" % nvr
if '--fix' in sys.argv:
print "Untagging %s" % nvr
koji.untagBuild(tag, nvr, force=True)
# Check for testing updates that aren't tagged properly
for update in PackageUpdate.select(PackageUpdate.q.status=='testing'):
dest_tag = update.release.testing_tag
for build in update.builds:
tags = [tag['name'] for tag in koji.listTags(build=build.nvr)]
if dest_tag not in tags:
print "%s marked as testing, but tagged with %s" % (build.nvr,
tags)
if '--fix' in sys.argv:
broke.add((tags[0], dest_tag, build.nvr))
# Check all candidate updates to see if they are in a different bodhi state
for release in Release.select():
tag = release.candidate_tag
tagged = [build['nvr'] for build in koji.listTagged(tag, latest=True)]
for nvr in tagged:
try:
build = PackageBuild.byNvr(nvr)
for update in build.updates:
if update.status in ('testing', 'stable'):
print "%s %s but tagged as %s" % (nvr,
update.status,
tag)
if '--fix' in sys.argv:
dest = release.testing_tag
if update.status == 'stable':
dest = release.stable_tag
elif update.status == 'obsolete':
dest = release.candidate_tag
broke.add((tag, dest, nvr))
except SQLObjectNotFound:
pass
# Make sure that all builds in koji tagged as an update exist
# in bodhi, and are in the expect state.
for release in Release.select():
for tag in (release.testing_tag, release.stable_tag):
tagged = [build['nvr'] for build in koji.listTagged(tag, latest=True)]
for nvr in tagged:
try:
build = PackageBuild.byNvr(nvr)
except SQLObjectNotFound:
print "PackageUpdate(%s) not found!" % nvr
continue
if not len(build.updates):
print "PackageBuild(%s) has no updates" % (build.nvr)
status = 'testing' in tag and 'testing' or 'stable'
for update in build.updates:
if update.status != status:
print "%s is %s in bodhi but tagged as %s in koji" % (
update.title, update.status, tag)
if '--fix' in sys.argv:
dest = release.testing_tag
if update.status == 'stable':
dest = release.stable_tag
elif update.status == 'obsolete':
dest = release.candidate_tag
for b in update.builds:
broke.add((tag, dest, b.nvr))
if broke:
print " ** Fixing broken tags! **"
koji.multicall = True
for tag, dest, build in broke:
print "Moving %s from %s to %s" % (build, tag, dest)
koji.moveBuild(tag, dest, build, force=True)
print "Running koji.multiCall()"
results = koji.multiCall()
success = False
print "Waiting for tasks"
bad_tasks = wait_for_tasks([task[0] for task in results])
if bad_tasks == 0:
success = True
if success:
print "Tags successfully moved!"
else:
print "Error moving tags!"
print "bad_tasks = %r" % bad_tasks
def list_tagged(tag):
koji = get_session()
builds = koji.listTagged(tag)
nvrs = [build['nvr'] for build in builds]
print "Fetched %d builds tagged with %s" % (len(builds), tag)
return nvrs
def __init__(self, repo, cacheduinfo=None):
self.tag = get_repo_tag(repo)
self.repo = repo
self.doc = None
self.updates = set()
self.builds = {}
self._from = config.get('bodhi_email')
self.koji = get_session()
self._create_document()
self._fetch_updates()
missing_ids = []
if cacheduinfo and exists(cacheduinfo):
log.debug("Loading cached updateinfo.xml.gz")
umd = UpdateMetadata()
umd.add(cacheduinfo)
existing_ids = set([up['update_id'] for up in umd.get_notices()])
seen_ids = set()
from_cache = set()
# Generate metadata for any new builds
for update in self.updates:
if update.updateid:
seen_ids.add(update.updateid)
if update.updateid in existing_ids:
notice = umd.get_notice(update.title)
if not notice:
log.warn('%s ID in cache but notice cannot be found' % (update.title))
self.add_update(update)
continue
if notice['updated']:
if datetime.strptime(notice['updated'], '%Y-%m-%d %H:%M:%S') < update.date_modified:
log.debug('Update modified, generating new notice: %s' % update.title)
self.add_update(update)
else:
log.debug('Loading updated %s from cache' % update.title)
from_cache.append(update.updateid)
elif update.date_modified:
log.debug('Update modified, generating new notice: %s' % update.title)
self.add_update(update)
else:
log.debug('Loading %s from cache' % update.title)
from_cache.add(update.updateid)
else:
log.debug('Adding new update notice: %s' % update.title)
self.add_update(update)
else:
missing_ids.append(update.title)
# Add all relevant notices from the cache to this document
for notice in umd.get_notices():
if notice['update_id'] in from_cache:
log.debug("Keeping existing notice: %s" % notice['title'])
self._add_notice(notice)
else:
# Keep all security notices in the stable repo
if 'testing' not in self.repo:
if notice['type'] == 'security':
if notice['update_id'] not in seen_ids:
log.debug("Keeping existing security notice: %s" %
notice['title'])
self._add_notice(notice)
else:
log.debug('%s already added?' % notice['title'])
else:
log.debug('Purging cached stable notice %s' % notice['title'])
else:
log.debug('Purging cached testing update %s' % notice['title'])
# Clean metadata generation
else:
log.debug("Generating new updateinfo.xml")
for update in self.updates:
if update.updateid:
self.add_update(update)
else:
missing_ids.append(update.title)
# Add *all* security updates
# TODO: only the most recent
#for update in PackageUpdate.select(PackageUpdate.q.type=='security'):
# self.add_update(update)
if missing_ids:
log.error("%d updates with missing ID!" % len(missing_ids))
log.debug(missing_ids)
def test_extended_metadata(self):
# grab the name of a build in updates-testing, and create it in our db
koji = get_session()
builds = koji.listTagged('dist-f13-updates-testing', latest=True)
# Create all of the necessary database entries
release = Release(name='F13', long_name='Fedora 13',
id_prefix='FEDORA', dist_tag='dist-f13')
package = Package(name=builds[0]['package_name'])
update = PackageUpdate(title=builds[0]['nvr'],
release=release,
submitter=builds[0]['owner_name'],
status='testing',
notes='foobar',
type='bugfix')
build = PackageBuild(nvr=builds[0]['nvr'], package=package)
update.addPackageBuild(build)
bug = Bugzilla(bz_id=1)
update.addBugzilla(bug)
cve = CVE(cve_id="CVE-2007-0000")
update.addCVE(cve)
update.assign_id()
print update
## Initialize our temporary repo
temprepo = join(tempfile.mkdtemp('bodhi'), 'f13-updates-testing')
print "Inserting updateinfo into temprepo: %s" % temprepo
mkmetadatadir(join(temprepo, 'i386'))
repodata = join(temprepo, 'i386', 'repodata')
assert exists(join(repodata, 'repomd.xml'))
## Generate the XML
md = ExtendedMetadata(temprepo)
## Insert the updateinfo.xml into the repository
md.insert_updateinfo()
updateinfo = self.__verify_updateinfo(repodata)
## Read an verify the updateinfo.xml.gz
uinfo = UpdateMetadata()
uinfo.add(updateinfo)
notice = uinfo.get_notice(('mutt', '1.5.14', '1.fc13'))
assert not notice
notice = uinfo.get_notice(get_nvr(update.title))
assert notice
assert notice['status'] == update.status
assert notice['updated'] == update.date_modified
assert notice['from'] == str(config.get('bodhi_email'))
assert notice['description'] == update.notes
assert notice['issued'] is not None
assert notice['update_id'] == update.updateid
assert notice['epoch'] is None
cve = notice['references'][0]
assert cve['type'] == 'cve'
assert cve['href'] == update.cves[0].get_url()
assert cve['id'] == update.cves[0].cve_id
bug = notice['references'][1]
assert bug['href'] == update.bugs[0].get_url()
assert bug['id'] == '1'
assert bug['type'] == 'bugzilla'
# FC6's yum update metadata parser doesn't know about some stuff
from yum import __version__
if __version__ >= '3.0.6':
assert notice['title'] == update.title
assert notice['release'] == update.release.long_name
assert cve['title'] is None
## Clean up
shutil.rmtree(temprepo)
def work(self):
self.koji = buildsys.get_session()
self.release = self.db.query(Release)\
.filter_by(name=self.release).one()
self.id = getattr(self.release, '%s_tag' % self.request.value)
# Set our thread's "name" so it shows up nicely in the logs.
# https://docs.python.org/2/library/threading.html#thread-objects
self.name = self.id
# For 'pending' branched releases, we only want to perform repo-related
# tasks for testing updates. For stable updates, we should just add the
# dist_tag and do everything else other than mashing/updateinfo, since
# the nightly build-branched cron job mashes for us.
self.skip_mash = False
if (self.release.state is ReleaseState.pending
and self.request is UpdateRequest.stable):
self.skip_mash = True
self.log.info('Running MasherThread(%s)' % self.id)
self.init_state()
if not self.resume:
self.init_path()
notifications.publish(
topic="mashtask.mashing",
msg=dict(repo=self.id, updates=self.state['updates']),
force=True,
)
try:
if self.resume:
self.load_state()
else:
self.save_state()
self.load_updates()
self.verify_updates()
if self.request is UpdateRequest.stable:
self.perform_gating()
self.determine_and_perform_tag_actions()
self.update_security_bugs()
self.expire_buildroot_overrides()
self.remove_pending_tags()
self.update_comps()
if self.resume and self.path in self.state['completed_repos']:
self.log.info('Skipping completed repo: %s', self.path)
self.complete_requests()
# We still need to generate the testing digest, since it's stored in memory
self.generate_testing_digest()
else:
if not self.skip_mash:
mash_thread = self.mash()
# Things we can do while we're mashing
self.complete_requests()
self.generate_testing_digest()
if not self.skip_mash:
uinfo = self.generate_updateinfo()
self.wait_for_mash(mash_thread)
uinfo.insert_updateinfo()
uinfo.insert_pkgtags()
uinfo.cache_repodata()
# Compose OSTrees from our freshly mashed repos
if config.get('compose_atomic_trees'):
self.compose_atomic_trees()
if not self.skip_mash:
self.sanity_check_repo()
self.stage_repo()
# Wait for the repo to hit the master mirror
self.wait_for_sync()
# Send fedmsg notifications
self.send_notifications()
# Update bugzillas
self.modify_bugs()
# Add comments to updates
self.status_comments()
# Announce stable updates to the mailing list
self.send_stable_announcements()
# Email updates-testing digest
self.send_testing_digest()
self.success = True
self.remove_state()
self.unlock_updates()
self.check_all_karma_thresholds()
except:
self.log.exception('Exception in MasherThread(%s)' % self.id)
self.save_state()
raise
finally:
self.finish(self.success)
def test_extended_metadata_updating_with_old_stable_security(self):
testutil.capture_log(['bodhi.metadata'])
koji = get_session()
del(koji.__untag__[:])
assert not koji.__untag__, koji.__untag__
builds = koji.listTagged('dist-f7-updates', latest=True)
# Create all of the necessary database entries
release = Release(name='F17', long_name='Fedora 17', id_prefix='FEDORA',
dist_tag='dist-f17')
package = Package(name='TurboGears')
update = PackageUpdate(title='TurboGears-1.0.2.2-2.fc7',
release=release,
submitter=builds[0]['owner_name'],
status='stable',
notes='foobar',
type='security')
build = PackageBuild(nvr='TurboGears-1.0.2.2-2.fc7', package=package)
update.addPackageBuild(build)
update.assign_id()
assert update.updateid
## Initialize our temporary repo
temprepo = join(tempfile.mkdtemp('bodhi'), 'f7-updates')
print "Inserting updateinfo into temprepo: %s" % temprepo
mkmetadatadir(join(temprepo, 'i386'))
repodata = join(temprepo, 'i386', 'repodata')
assert exists(join(repodata, 'repomd.xml'))
## Generate the XML
md = ExtendedMetadata(temprepo)
## Insert the updateinfo.xml into the repository
md.insert_updateinfo()
updateinfo = self.__verify_updateinfo(repodata)
# Create a new non-security update for the same package
newbuild = 'TurboGears-1.0.2.2-3.fc7'
update = PackageUpdate(title=newbuild,
release=release,
submitter=builds[0]['owner_name'],
status='stable',
notes='foobar',
type='enhancement')
build = PackageBuild(nvr=newbuild, package=package)
update.addPackageBuild(build)
update.assign_id()
koji.__untag__.append('TurboGears-1.0.2.2-2.fc7')
## Test out updateinfo.xml updating via our ExtendedMetadata
md = ExtendedMetadata(temprepo, updateinfo)
md.insert_updateinfo()
updateinfo = self.__verify_updateinfo(repodata)
## Read an verify the updateinfo.xml.gz
uinfo = UpdateMetadata()
uinfo.add(updateinfo)
print(testutil.get_log())
assert len(uinfo.get_notices()) == 2, len(uinfo.get_notices())
assert uinfo.get_notice(get_nvr('TurboGears-1.0.2.2-2.fc7'))
notice = uinfo.get_notice(get_nvr(update.title))
assert notice, 'Cannot find update for %r' % get_nvr(update.title)
assert notice['status'] == update.status
assert notice['updated'] == update.date_modified
assert notice['from'] == str(config.get('bodhi_email'))
assert notice['description'] == update.notes
assert notice['issued'] is not None
assert notice['update_id'] == update.updateid
## Clean up
shutil.rmtree(temprepo)
del(koji.__untag__[:])
def tearDown(self):
DBSession.remove()
get_session().clear()
shutil.rmtree(self.tempdir)
def get_all_packages():
""" Get a list of all packages in Koji """
log.debug('Fetching list of all packages...')
koji = buildsys.get_session()
return [pkg['package_name'] for pkg in koji.listPackages()]
def test_extended_metadata_updating(self):
# grab the name of a build in updates-testing, and create it in our db
koji = get_session()
builds = koji.listTagged('dist-f13-updates-testing', latest=True)
# Create all of the necessary database entries
release = Release(name='F13',
long_name='Fedora 13',
id_prefix='FEDORA',
dist_tag='dist-f13')
package = Package(name=builds[0]['package_name'])
update = PackageUpdate(title=builds[0]['nvr'],
release=release,
submitter=builds[0]['owner_name'],
status='testing',
notes='foobar',
type='bugfix')
build = PackageBuild(nvr=builds[0]['nvr'], package=package)
update.addPackageBuild(build)
bug = Bugzilla(bz_id=1)
bug.title = u'test bug'
update.addBugzilla(bug)
cve = CVE(cve_id="CVE-2007-0000")
update.addCVE(cve)
update.assign_id()
## Initialize our temporary repo
temprepo = join(tempfile.mkdtemp('bodhi'), 'f7-updates-testing')
print "Inserting updateinfo into temprepo: %s" % temprepo
mkmetadatadir(join(temprepo, 'i386'))
repodata = join(temprepo, 'i386', 'repodata')
assert exists(join(repodata, 'repomd.xml'))
## Generate the XML
md = ExtendedMetadata(temprepo)
## Insert the updateinfo.xml into the repository
md.insert_updateinfo()
updateinfo = join(repodata, 'updateinfo.xml.gz')
assert exists(updateinfo)
## Read an verify the updateinfo.xml.gz
uinfo = UpdateMetadata()
uinfo.add(updateinfo)
notice = uinfo.get_notice(('mutt', '1.5.14', '1.fc13'))
assert not notice
notice = uinfo.get_notice(get_nvr(update.title))
assert notice
assert notice['status'] == update.status
assert notice['updated'] == update.date_modified
assert notice['from'] == str(config.get('bodhi_email'))
assert notice['description'] == update.notes
assert notice['issued'] != None
assert notice['update_id'] == update.updateid
cve = notice['references'][0]
assert cve['type'] == 'cve'
assert cve['href'] == update.cves[0].get_url()
assert cve['id'] == update.cves[0].cve_id
bug = notice['references'][1]
assert bug['href'] == update.bugs[0].get_url()
assert bug['id'] == '1'
assert bug['type'] == 'bugzilla'
assert bug['title'] == 'test bug'
# FC6's yum update metadata parser doesn't know about some stuff
from yum import __version__
if __version__ >= '3.0.6':
assert notice['title'] == update.title
assert notice['release'] == update.release.long_name
assert cve['title'] == None
## Test out updateinfo.xml updating via our ExtendedMetadata
md = ExtendedMetadata(temprepo, updateinfo)
md.insert_updateinfo()
updateinfo = join(repodata, 'updateinfo.xml.gz')
assert exists(updateinfo)
## Read an verify the updateinfo.xml.gz
uinfo = UpdateMetadata()
uinfo.add(updateinfo)
notice = uinfo.get_notice(('mutt', '1.5.14', '1.fc13'))
assert not notice
notice = uinfo.get_notice(get_nvr(update.title))
assert notice
assert notice['status'] == update.status
assert notice['updated'] == update.date_modified
assert notice['from'] == str(config.get('bodhi_email'))
assert notice['description'] == update.notes
assert notice['issued'] != None
assert notice['update_id'] == update.updateid
cve = notice['references'][0]
assert cve['type'] == 'cve'
assert cve['href'] == update.cves[0].get_url()
assert cve['id'] == update.cves[0].cve_id
bug = notice['references'][1]
assert bug['href'] == update.bugs[0].get_url()
assert bug['id'] == '1'
assert bug['type'] == 'bugzilla'
assert bug['title'] == 'test bug', bug
# FC6's yum update metadata parser doesn't know about some stuff
from yum import __version__
if __version__ >= '3.0.6':
assert notice['title'] == update.title
assert notice['release'] == update.release.long_name
assert cve['title'] == None
## Clean up
shutil.rmtree(temprepo)
def main():
load_config()
__connection__ = hub = PackageHub("bodhi")
koji = get_session()
tasks = []
broke = set()
# Clean up any stray pending tags
for release in Release.select():
print "Finding all pending-testing builds..."
if release.name.startswith('EL'):
continue
tag = release.pending_testing_tag
tagged = [build['nvr'] for build in koji.listTagged(tag)]
for nvr in tagged:
try:
build = PackageBuild.byNvr(nvr)
for update in build.updates:
if update.status in ('testing', 'stable', 'obsolete'):
print "%s %s" % (nvr, update.status)
if '--fix' in sys.argv:
print "Untagging %s" % nvr
koji.untagBuild(tag, nvr, force=True)
except SQLObjectNotFound:
print "Can't find build for %s" % nvr
if '--fix' in sys.argv:
print "Untagging %s" % nvr
koji.untagBuild(tag, nvr, force=True)
tag = release.pending_stable_tag
tagged = [build['nvr'] for build in koji.listTagged(tag)]
for nvr in tagged:
try:
build = PackageBuild.byNvr(nvr)
for update in build.updates:
if update.status in ('pending', 'obsolete', 'stable'):
print "%s %s" % (nvr, update.status)
if '--fix' in sys.argv:
print "Untagging %s" % nvr
koji.untagBuild(tag, nvr, force=True)
except SQLObjectNotFound:
print "Can't find build for %s" % nvr
if '--fix' in sys.argv:
print "Untagging %s" % nvr
koji.untagBuild(tag, nvr, force=True)
# Check for testing updates that aren't tagged properly
for update in PackageUpdate.select(PackageUpdate.q.status == 'testing'):
dest_tag = update.release.testing_tag
for build in update.builds:
tags = [tag['name'] for tag in koji.listTags(build=build.nvr)]
if dest_tag not in tags:
print "%s marked as testing, but tagged with %s" % (build.nvr,
tags)
if '--fix' in sys.argv:
broke.add((tags[0], dest_tag, build.nvr))
# Check all candidate updates to see if they are in a different bodhi state
for release in Release.select():
tag = release.candidate_tag
tagged = [build['nvr'] for build in koji.listTagged(tag, latest=True)]
for nvr in tagged:
try:
build = PackageBuild.byNvr(nvr)
for update in build.updates:
if update.status in ('testing', 'stable'):
print "%s %s but tagged as %s" % (nvr, update.status,
tag)
if '--fix' in sys.argv:
dest = release.testing_tag
if update.status == 'stable':
dest = release.stable_tag
elif update.status == 'obsolete':
dest = release.candidate_tag
broke.add((tag, dest, nvr))
except SQLObjectNotFound:
pass
# Make sure that all builds in koji tagged as an update exist
# in bodhi, and are in the expect state.
for release in Release.select():
for tag in (release.testing_tag, release.stable_tag):
tagged = [
build['nvr'] for build in koji.listTagged(tag, latest=True)
]
for nvr in tagged:
try:
build = PackageBuild.byNvr(nvr)
except SQLObjectNotFound:
print "PackageUpdate(%s) not found!" % nvr
continue
if not len(build.updates):
print "PackageBuild(%s) has no updates" % (build.nvr)
status = 'testing' in tag and 'testing' or 'stable'
for update in build.updates:
if update.status != status:
print "%s is %s in bodhi but tagged as %s in koji" % (
update.title, update.status, tag)
if '--fix' in sys.argv:
dest = release.testing_tag
if update.status == 'stable':
dest = release.stable_tag
elif update.status == 'obsolete':
dest = release.candidate_tag
for b in update.builds:
broke.add((tag, dest, b.nvr))
if broke:
print " ** Fixing broken tags! **"
koji.multicall = True
for tag, dest, build in broke:
print "Moving %s from %s to %s" % (build, tag, dest)
koji.moveBuild(tag, dest, build, force=True)
print "Running koji.multiCall()"
results = koji.multiCall()
success = False
print "Waiting for tasks"
bad_tasks = wait_for_tasks([task[0] for task in results])
if bad_tasks == 0:
success = True
if success:
print "Tags successfully moved!"
else:
print "Error moving tags!"
print "bad_tasks = %r" % bad_tasks
def test_extended_metadata_updating_with_edited_updates(self):
testutil.capture_log(['bodhi.metadata'])
# grab the name of a build in updates-testing, and create it in our db
koji = get_session()
builds = koji.listTagged('dist-f13-updates-testing', latest=True)
# Create all of the necessary database entries
release = Release(name='F13',
long_name='Fedora 13',
id_prefix='FEDORA',
dist_tag='dist-f13')
package = Package(name=builds[0]['package_name'])
update = PackageUpdate(title=builds[0]['nvr'],
release=release,
submitter=builds[0]['owner_name'],
status='testing',
notes='foobar',
type='bugfix')
build = PackageBuild(nvr=builds[0]['nvr'], package=package)
update.addPackageBuild(build)
update.assign_id()
## Initialize our temporary repo
temprepo = join(tempfile.mkdtemp('bodhi'), 'f13-updates-testing')
print "Inserting updateinfo into temprepo: %s" % temprepo
mkmetadatadir(join(temprepo, 'i386'))
repodata = join(temprepo, 'i386', 'repodata')
assert exists(join(repodata, 'repomd.xml'))
## Generate the XML
md = ExtendedMetadata(temprepo)
## Insert the updateinfo.xml into the repository
md.insert_updateinfo()
updateinfo = self.__verify_updateinfo(repodata)
## Read an verify the updateinfo.xml.gz
uinfo = UpdateMetadata()
uinfo.add(updateinfo)
notice = uinfo.get_notice(('mutt', '1.5.14', '1.fc13'))
assert not notice
notice = uinfo.get_notice(get_nvr(update.title))
assert notice
assert notice['status'] == update.status
assert notice['updated'] == update.date_modified
assert notice['from'] == str(config.get('bodhi_email'))
assert notice['description'] == update.notes
assert notice['issued'] is not None
assert notice['update_id'] == update.updateid
assert notice['title'] == update.title
assert notice['release'] == update.release.long_name
## Edit the update and bump the build revision
nvr = 'TurboGears-1.0.2.2-3.fc7'
newbuild = PackageBuild(nvr=nvr, package=package)
update.removePackageBuild(build)
update.addPackageBuild(newbuild)
update.title = nvr
update.date_modified = datetime.utcnow()
# Pretend -2 was unpushed
assert len(koji.__untag__) == 0
koji.__untag__.append('TurboGears-1.0.2.2-2.fc7')
## Test out updateinfo.xml updating via our ExtendedMetadata
md = ExtendedMetadata(temprepo, updateinfo)
md.insert_updateinfo()
updateinfo = self.__verify_updateinfo(repodata)
## Read an verify the updateinfo.xml.gz
uinfo = UpdateMetadata()
uinfo.add(updateinfo)
print(testutil.get_log())
notice = uinfo.get_notice(('TurboGears', '1.0.2.2', '2.fc7'))
assert not notice, "Old TG notice did not get pruned: %s" % notice
notice = uinfo.get_notice(('TurboGears', '1.0.2.2', '3.fc7'))
assert notice, uinfo
assert notice['title'] == update.title
num_notices = len(uinfo.get_notices())
assert num_notices == 1, num_notices
## Clean up
shutil.rmtree(temprepo)
del (koji.__untag__[:])
