def _verify_updateinfo(self, update, repo):
""" Verify that the updateinfo.xml.gz for a given repo matches the
data for a given update
"""
print "_verify_updateinfo(%s, %s)" % (update.nvr, repo)
uinfo = UpdateMetadata()
uinfo.add(repo)
notice = uinfo.get_notice(update.nvr)
assert notice
assert notice['from'] == '*****@*****.**'
assert notice['title'] == update.nvr
assert notice['release'] == update.release.long_name
assert notice['type'] == update.type
assert notice['status'] == update.status
assert notice['update_id'] == update.updateid
assert notice['issued'] == str(update.date_pushed)
assert notice['description'] == update.notes
for collection in notice['pkglist']:
numfiles = 0
for archfiles in update.filelist.values():
for file in archfiles:
numfiles += 1
assert len(collection['packages']) == numfiles
for pkg in collection['packages']:
assert pkg['arch'] in update.filelist.keys()
found = False
for file in update.filelist[pkg['arch']]:
if pkg['filename'] in file:
found = True
break
assert found
def postresolve_hook(conduit):
_setup_changelog_from_cmdline(conduit)
if not (changelog or updateinfo) or conduit.resultcode == 1:
return
# Find currently installed versions of packages we're about to update
ts = conduit.getTsInfo()
rpmdb = conduit.getRpmDB()
if updateinfo:
repos = set()
for tsmem in ts.getMembers():
if tsmem.po.repoid == 'installed':
continue
repos.add(tsmem.po.repo)
mdi = UpdateMetadata(repos=list(repos))
for tsmem in ts.getMembers():
for po in rpmdb.searchNevra(name=tsmem.po.name, arch=tsmem.po.arch):
times = po['changelogtime']
try:
n,v,r,e,a = splitFilename(po.sourcerpm)
except TypeError:
n = po.name
if len(times) == 0:
# deal with packages without changelog
origpkgs[n] = 0
else:
origpkgs[n] = times[0]
if updateinfo:
for (pkgtup, notice) in mdi.get_applicable_notices(po.pkgtup):
orignots.add(notice)
if conduit.confString('main', 'when', default='post') == 'pre':
show_changes(conduit, 'Changes in packages about to be updated:')
def _verify_updateinfo(self, update, repo):
""" Verify that the updateinfo.xml.gz for a given repo matches the
data for a given update
"""
print "_verify_updateinfo(%s, %s)" % (update.nvr, repo)
uinfo = UpdateMetadata()
uinfo.add(repo)
notice = uinfo.get_notice(update.nvr)
assert notice
assert notice['from'] == '*****@*****.**'
assert notice['title'] == update.nvr
assert notice['release'] == update.release.long_name
assert notice['type'] == update.type
assert notice['status'] == update.status
assert notice['update_id'] == update.updateid
assert notice['issued'] == str(update.date_pushed)
assert notice['description'] == update.notes
for collection in notice['pkglist']:
numfiles = 0
for archfiles in update.filelist.values():
for file in archfiles:
numfiles += 1
assert len(collection['packages']) == numfiles
for pkg in collection['packages']:
assert pkg['arch'] in update.filelist.keys()
found = False
for file in update.filelist[pkg['arch']]:
if pkg['filename'] in file:
found = True
break
assert found
def postresolve_hook(conduit):
_setup_changelog_from_cmdline(conduit)
if not (changelog or updateinfo) or conduit.resultcode == 1:
return
# Find currently installed versions of packages we're about to update
ts = conduit.getTsInfo()
rpmdb = conduit.getRpmDB()
if updateinfo:
repos = set()
for tsmem in ts.getMembers():
if tsmem.po.repoid == 'installed':
continue
repos.add(tsmem.po.repo)
mdi = UpdateMetadata(repos=list(repos))
for tsmem in ts.getMembers():
for po in rpmdb.searchNevra(name=tsmem.po.name, arch=tsmem.po.arch):
times = po['changelogtime']
try:
n, v, r, e, a = splitFilename(po.sourcerpm)
except TypeError:
n = po.name
if len(times) == 0:
# deal with packages without changelog
origpkgs[n] = 0
else:
origpkgs[n] = times[0]
if updateinfo:
for (pkgtup, notice) in mdi.get_applicable_notices(po.pkgtup):
orignots.add(notice)
if conduit.confString('main', 'when', default='post') == 'pre':
show_changes(conduit, 'Changes in packages about to be updated:')
def __init__(self, repo, cacheduinfo=None):
self.tag = get_repo_tag(repo)
self.repo = repo
self.doc = None
self.updates = set()
self.builds = {}
self._from = config.get('bodhi_email')
self.koji = get_session()
self._create_document()
self._fetch_updates()
missing_ids = []
if cacheduinfo and exists(cacheduinfo):
log.debug("Loading cached updateinfo.xml.gz")
umd = UpdateMetadata()
umd.add(cacheduinfo)
existing_ids = set([up['update_id'] for up in umd.get_notices()])
# Generate metadata for any new builds
for update in self.updates:
if update.updateid:
self.add_update(update)
log.debug('Adding new update notice: %s' % update.title)
if update.updateid in existing_ids:
existing_ids.remove(update.updateid)
else:
missing_ids.append(update.title)
# Add all relevant notices from the metadata to this document
for notice in umd.get_notices():
if notice['update_id'] in existing_ids:
log.debug("Adding existing notice: %s" % notice['title'])
self._add_notice(notice)
existing_ids.remove(notice['update_id'])
else:
if notice['type'] == 'security':
log.debug("Adding existing security notice: %s" %
notice['title'])
self._add_notice(notice)
else:
log.debug("Removing %s from updateinfo" % notice['title'])
else:
log.debug("Generating new updateinfo.xml")
for update in self.updates:
if update.updateid:
self.add_update(update)
else:
missing_ids.append(update.title)
# Add *all* security updates
# TODO: only the most recent
#for update in PackageUpdate.select(PackageUpdate.q.type=='security'):
# self.add_update(update)
if missing_ids:
log.error("%d updates with missing ID!" % len(missing_ids))
log.debug(missing_ids)
def ysp_gen_metadata(repos):
""" Generate the info. from the updateinfo.xml files. """
md_info = UpdateMetadata()
for repo in repos:
if not repo.enabled:
continue
try: # attempt to grab the updateinfo.xml.gz from the repodata
md_info.add(repo)
except yum.Errors.RepoMDError:
continue # No metadata found for this repo
return md_info
def get_update_notices(path_to_updateinfo):
"""
path_to_updateinfo: path to updateinfo.xml
Returns a list of dictionaries
Dictionary is based on keys from yum.update_md.UpdateNotice
"""
um = UpdateMetadata()
um.add(path_to_updateinfo)
notices = []
for info in um.get_notices():
notices.append(info.get_metadata())
return notices
def get_update_notices(path_to_updateinfo):
"""
path_to_updateinfo: path to updateinfo.xml
Returns a list of dictionaries
Dictionary is based on keys from yum.update_md.UpdateNotice
"""
um = UpdateMetadata()
um.add(path_to_updateinfo)
notices = []
for info in um.get_notices():
notices.append(info.get_metadata())
return notices
def populateUpdateMetadata(self):
"""Populate the metadata for the packages in the update."""
self.updateMetadata = UpdateMetadata()
repos = []
for (new, old) in self.up.getUpdatesTuples():
pkg = self.getPackageObject(new)
if pkg.repoid not in repos:
repo = self.repos.getRepo(pkg.repoid)
repos.append(repo.id)
try: # grab the updateinfo.xml.gz from the repodata
md = repo.retrieveMD('updateinfo')
except Exception: # can't find any; silently move on
continue
md = gzip.open(md)
self.updateMetadata.add(md)
md.close()
def updateinfo(errata_units, save_location):
um = UpdateMetadata()
for e in errata_units:
encode_epoch(e)
un = UpdateNotice()
_md = {
'from' : e.metadata['from'],
'type' : e.metadata['type'],
'title' : e.metadata['title'],
'release' : e.metadata.get('release', ''),
'status' : e.metadata['status'],
'version' : e.metadata['version'],
'pushcount' : e.metadata.get('pushcount', ''),
'update_id' : e.unit_key['id'],
'issued' : e.metadata['issued'],
'updated' : e.metadata.get('updated', ''),
'description' : e.metadata['description'],
'references' : e.metadata['references'],
'pkglist' : e.metadata['pkglist'],
'reboot_suggested' : e.metadata.get('reboot_suggested', False),
'severity' : e.metadata.get('severity', ''),
'rights' : e.metadata.get('rights', ''),
'summary' : e.metadata.get('summary', ''),
'solution' : e.metadata.get('solution', ''),
}
un._md = _md
um.add_notice(un)
if not um._notices:
# nothing to do return
return
updateinfo_path = None
try:
updateinfo_path = "%s/%s" % (save_location, "updateinfo.xml")
f = open(updateinfo_path, 'wt')
try:
updateinfo_xml = um.xml(fileobj=f)
log.info("updateinfo.xml generated and written to file %s" % updateinfo_path)
finally:
f.close()
except Exception, e:
log.error("Error writing updateinfo.xml to path %s: %s" % (updateinfo_path, e))
def updateinfo(errata_units, save_location):
um = UpdateMetadata()
for e in errata_units:
un = UpdateNotice()
_md = {
'from' : e.metadata['from_str'],
'type' : e.metadata['type'],
'title' : e.metadata['title'],
'release' : e.metadata['release'],
'status' : e.metadata['status'],
'version' : e.metadata['version'],
'pushcount' : e.metadata['pushcount'],
'update_id' : e.unit_key['id'],
'issued' : e.metadata['issued'],
'updated' : e.metadata['updated'],
'description' : e.metadata['description'],
'references' : e.metadata['references'],
'pkglist' : e.metadata['pkglist'],
'reboot_suggested' : e.metadata['reboot_suggested'],
'severity' : e.metadata['severity'],
'rights' : e.metadata['rights'],
'summary' : e.metadata['summary'],
'solution' : e.metadata['solution'],
}
un._md = _md
um.add_notice(un)
if not um._notices:
# nothing to do return
return
updateinfo_path = None
try:
updateinfo_path = "%s/%s" % (save_location, "updateinfo.xml")
updateinfo_xml = um.xml(fileobj=open(updateinfo_path, 'wt'))
log.info("updateinfo.xml generated and written to file %s" % updateinfo_path)
except:
log.error("Error writing updateinfo.xml to path %s" % updateinfo_path)
return updateinfo_path
def __init__(self, repo, cacheduinfo=None):
self.tag = get_repo_tag(repo)
self.repo = repo
self.doc = None
self.updates = set()
self.builds = {}
self._from = config.get('bodhi_email')
self.koji = get_session()
self._create_document()
self._fetch_updates()
missing_ids = []
if cacheduinfo and exists(cacheduinfo):
log.debug("Loading cached updateinfo.xml.gz")
umd = UpdateMetadata()
umd.add(cacheduinfo)
# Generate metadata for any new builds
for update in self.updates:
for build in update.builds:
if not umd.get_notice(build.nvr):
if update.updateid:
self.add_update(update)
break
else:
missing_ids.append(update.title)
# Add all relevant notices from the metadata to this document
ids = [
update.updateid for update in self.updates if update.updateid
]
for notice in umd.get_notices():
if notice['update_id'] in ids or notice['type'] == 'security':
self._add_notice(notice)
else:
log.debug("Removing %s from updateinfo" % notice['title'])
else:
log.debug("Generating new updateinfo.xml")
for update in self.updates:
if update.updateid:
self.add_update(update)
else:
missing_ids.append(update.title)
# Add *all* security updates
# TODO: only the most recent
#for update in PackageUpdate.select(PackageUpdate.q.type=='security'):
# self.add_update(update)
if missing_ids:
log.error("%d updates with missing ID!" % len(missing_ids))
log.debug(missing_ids)
def updateinfo(errata_units, save_location):
um = UpdateMetadata()
for e in errata_units:
encode_epoch(e)
un = UpdateNotice()
_md = {
'from': e.metadata['from'],
'type': e.metadata['type'],
'title': e.metadata['title'],
'release': e.metadata.get('release', ''),
'status': e.metadata['status'],
'version': e.metadata['version'],
'pushcount': e.metadata.get('pushcount', ''),
'update_id': e.unit_key['id'],
'issued': e.metadata['issued'],
'updated': e.metadata.get('updated', ''),
'description': e.metadata['description'],
'references': e.metadata['references'],
'pkglist': e.metadata['pkglist'],
'reboot_suggested': e.metadata.get('reboot_suggested', False),
'severity': e.metadata.get('severity', ''),
'rights': e.metadata.get('rights', ''),
'summary': e.metadata.get('summary', ''),
'solution': e.metadata.get('solution', ''),
}
un._md = _md
um.add_notice(un)
if not um._notices:
# nothing to do return
return
updateinfo_path = None
try:
updateinfo_path = "%s/%s" % (save_location, "updateinfo.xml")
f = open(updateinfo_path, 'wt')
try:
um.xml(fileobj=f)
log.info("updateinfo.xml generated and written to file %s" %
updateinfo_path)
finally:
f.close()
except Exception, e:
log.error("Error writing updateinfo.xml to path %s: %s" %
(updateinfo_path, e))
def prepare(self):
"""Prepare for the user's request
This implies getting the various repository metadata.
"""
self.setCacheDir()
if not self.conf.cache:
# If we force the cache usage, the next operation will not imply a
# download, so don't log in that case
self.logger.log_progress({
"current":
0,
"total":
1,
"hint":
"Downloading the package "
"metadata..."
})
self._getSacks()
self.updatemd = UpdateMetadata(self.repos.listEnabled())
def test_extended_metadata(self):
# grab the name of a build in updates-testing, and create it in our db
koji = get_session()
builds = koji.listTagged('dist-f13-updates-testing', latest=True)
# Create all of the necessary database entries
release = Release(name='F13', long_name='Fedora 13',
id_prefix='FEDORA', dist_tag='dist-f13')
package = Package(name=builds[0]['package_name'])
update = PackageUpdate(title=builds[0]['nvr'],
release=release,
submitter=builds[0]['owner_name'],
status='testing',
notes='foobar',
type='bugfix')
build = PackageBuild(nvr=builds[0]['nvr'], package=package)
update.addPackageBuild(build)
bug = Bugzilla(bz_id=1)
update.addBugzilla(bug)
cve = CVE(cve_id="CVE-2007-0000")
update.addCVE(cve)
update.assign_id()
print update
## Initialize our temporary repo
temprepo = join(tempfile.mkdtemp('bodhi'), 'f13-updates-testing')
print "Inserting updateinfo into temprepo: %s" % temprepo
mkmetadatadir(join(temprepo, 'i386'))
repodata = join(temprepo, 'i386', 'repodata')
assert exists(join(repodata, 'repomd.xml'))
## Generate the XML
md = ExtendedMetadata(temprepo)
## Insert the updateinfo.xml into the repository
md.insert_updateinfo()
updateinfo = self.__verify_updateinfo(repodata)
## Read an verify the updateinfo.xml.gz
uinfo = UpdateMetadata()
uinfo.add(updateinfo)
notice = uinfo.get_notice(('mutt', '1.5.14', '1.fc13'))
assert not notice
notice = uinfo.get_notice(get_nvr(update.title))
assert notice
assert notice['status'] == update.status
assert notice['updated'] == update.date_modified
assert notice['from'] == str(config.get('bodhi_email'))
assert notice['description'] == update.notes
assert notice['issued'] is not None
assert notice['update_id'] == update.updateid
assert notice['epoch'] is None
cve = notice['references'][0]
assert cve['type'] == 'cve'
assert cve['href'] == update.cves[0].get_url()
assert cve['id'] == update.cves[0].cve_id
bug = notice['references'][1]
assert bug['href'] == update.bugs[0].get_url()
assert bug['id'] == '1'
assert bug['type'] == 'bugzilla'
# FC6's yum update metadata parser doesn't know about some stuff
from yum import __version__
if __version__ >= '3.0.6':
assert notice['title'] == update.title
assert notice['release'] == update.release.long_name
assert cve['title'] is None
## Clean up
shutil.rmtree(temprepo)
def test_extended_metadata_updating_with_edited_updates(self):
testutil.capture_log(['bodhi.metadata'])
# grab the name of a build in updates-testing, and create it in our db
koji = get_session()
builds = koji.listTagged('dist-f13-updates-testing', latest=True)
# Create all of the necessary database entries
release = Release(name='F13', long_name='Fedora 13', id_prefix='FEDORA',
dist_tag='dist-f13')
package = Package(name=builds[0]['package_name'])
update = PackageUpdate(title=builds[0]['nvr'],
release=release,
submitter=builds[0]['owner_name'],
status='testing',
notes='foobar',
type='bugfix')
build = PackageBuild(nvr=builds[0]['nvr'], package=package)
update.addPackageBuild(build)
update.assign_id()
## Initialize our temporary repo
temprepo = join(tempfile.mkdtemp('bodhi'), 'f13-updates-testing')
print "Inserting updateinfo into temprepo: %s" % temprepo
mkmetadatadir(join(temprepo, 'i386'))
repodata = join(temprepo, 'i386', 'repodata')
assert exists(join(repodata, 'repomd.xml'))
## Generate the XML
md = ExtendedMetadata(temprepo)
## Insert the updateinfo.xml into the repository
md.insert_updateinfo()
updateinfo = self.__verify_updateinfo(repodata)
## Read an verify the updateinfo.xml.gz
uinfo = UpdateMetadata()
uinfo.add(updateinfo)
notice = uinfo.get_notice(('mutt', '1.5.14', '1.fc13'))
assert not notice
notice = uinfo.get_notice(get_nvr(update.title))
assert notice
assert notice['status'] == update.status
assert notice['updated'] == update.date_modified
assert notice['from'] == str(config.get('bodhi_email'))
assert notice['description'] == update.notes
assert notice['issued'] is not None
assert notice['update_id'] == update.updateid
assert notice['title'] == update.title
assert notice['release'] == update.release.long_name
## Edit the update and bump the build revision
nvr = 'TurboGears-1.0.2.2-3.fc7'
newbuild = PackageBuild(nvr=nvr, package=package)
update.removePackageBuild(build)
update.addPackageBuild(newbuild)
update.title = nvr
update.date_modified = datetime.utcnow()
# Pretend -2 was unpushed
assert len(koji.__untag__) == 0
koji.__untag__.append('TurboGears-1.0.2.2-2.fc7')
## Test out updateinfo.xml updating via our ExtendedMetadata
md = ExtendedMetadata(temprepo, updateinfo)
md.insert_updateinfo()
updateinfo = self.__verify_updateinfo(repodata)
## Read an verify the updateinfo.xml.gz
uinfo = UpdateMetadata()
uinfo.add(updateinfo)
print(testutil.get_log())
notice = uinfo.get_notice(('TurboGears', '1.0.2.2', '2.fc7'))
assert not notice, "Old TG notice did not get pruned: %s" % notice
notice = uinfo.get_notice(('TurboGears', '1.0.2.2', '3.fc7'))
assert notice, uinfo
assert notice['title'] == update.title
num_notices = len(uinfo.get_notices())
assert num_notices == 1, num_notices
## Clean up
shutil.rmtree(temprepo)
del(koji.__untag__[:])
def test_extended_metadata_updating_with_old_stable_security(self):
testutil.capture_log(['bodhi.metadata'])
koji = get_session()
del(koji.__untag__[:])
assert not koji.__untag__, koji.__untag__
builds = koji.listTagged('dist-f7-updates', latest=True)
# Create all of the necessary database entries
release = Release(name='F17', long_name='Fedora 17', id_prefix='FEDORA',
dist_tag='dist-f17')
package = Package(name='TurboGears')
update = PackageUpdate(title='TurboGears-1.0.2.2-2.fc7',
release=release,
submitter=builds[0]['owner_name'],
status='stable',
notes='foobar',
type='security')
build = PackageBuild(nvr='TurboGears-1.0.2.2-2.fc7', package=package)
update.addPackageBuild(build)
update.assign_id()
assert update.updateid
## Initialize our temporary repo
temprepo = join(tempfile.mkdtemp('bodhi'), 'f7-updates')
print "Inserting updateinfo into temprepo: %s" % temprepo
mkmetadatadir(join(temprepo, 'i386'))
repodata = join(temprepo, 'i386', 'repodata')
assert exists(join(repodata, 'repomd.xml'))
## Generate the XML
md = ExtendedMetadata(temprepo)
## Insert the updateinfo.xml into the repository
md.insert_updateinfo()
updateinfo = self.__verify_updateinfo(repodata)
# Create a new non-security update for the same package
newbuild = 'TurboGears-1.0.2.2-3.fc7'
update = PackageUpdate(title=newbuild,
release=release,
submitter=builds[0]['owner_name'],
status='stable',
notes='foobar',
type='enhancement')
build = PackageBuild(nvr=newbuild, package=package)
update.addPackageBuild(build)
update.assign_id()
koji.__untag__.append('TurboGears-1.0.2.2-2.fc7')
## Test out updateinfo.xml updating via our ExtendedMetadata
md = ExtendedMetadata(temprepo, updateinfo)
md.insert_updateinfo()
updateinfo = self.__verify_updateinfo(repodata)
## Read an verify the updateinfo.xml.gz
uinfo = UpdateMetadata()
uinfo.add(updateinfo)
print(testutil.get_log())
assert len(uinfo.get_notices()) == 2, len(uinfo.get_notices())
assert uinfo.get_notice(get_nvr('TurboGears-1.0.2.2-2.fc7'))
notice = uinfo.get_notice(get_nvr(update.title))
assert notice, 'Cannot find update for %r' % get_nvr(update.title)
assert notice['status'] == update.status
assert notice['updated'] == update.date_modified
assert notice['from'] == str(config.get('bodhi_email'))
assert notice['description'] == update.notes
assert notice['issued'] is not None
assert notice['update_id'] == update.updateid
## Clean up
shutil.rmtree(temprepo)
del(koji.__untag__[:])
def test_extended_metadata_updating_with_old_stable_security(self):
testutil.capture_log(['bodhi.metadata'])
koji = get_session()
del (koji.__untag__[:])
assert not koji.__untag__, koji.__untag__
builds = koji.listTagged('dist-f7-updates', latest=True)
# Create all of the necessary database entries
release = Release(name='F17',
long_name='Fedora 17',
id_prefix='FEDORA',
dist_tag='dist-f17')
package = Package(name='TurboGears')
update = PackageUpdate(title='TurboGears-1.0.2.2-2.fc7',
release=release,
submitter=builds[0]['owner_name'],
status='stable',
notes='foobar',
type='security')
build = PackageBuild(nvr='TurboGears-1.0.2.2-2.fc7', package=package)
update.addPackageBuild(build)
update.assign_id()
assert update.updateid
## Initialize our temporary repo
temprepo = join(tempfile.mkdtemp('bodhi'), 'f7-updates')
print "Inserting updateinfo into temprepo: %s" % temprepo
mkmetadatadir(join(temprepo, 'i386'))
repodata = join(temprepo, 'i386', 'repodata')
assert exists(join(repodata, 'repomd.xml'))
## Generate the XML
md = ExtendedMetadata(temprepo)
## Insert the updateinfo.xml into the repository
md.insert_updateinfo()
updateinfo = self.__verify_updateinfo(repodata)
# Create a new non-security update for the same package
newbuild = 'TurboGears-1.0.2.2-3.fc7'
update = PackageUpdate(title=newbuild,
release=release,
submitter=builds[0]['owner_name'],
status='stable',
notes='foobar',
type='enhancement')
build = PackageBuild(nvr=newbuild, package=package)
update.addPackageBuild(build)
update.assign_id()
koji.__untag__.append('TurboGears-1.0.2.2-2.fc7')
## Test out updateinfo.xml updating via our ExtendedMetadata
md = ExtendedMetadata(temprepo, updateinfo)
md.insert_updateinfo()
updateinfo = self.__verify_updateinfo(repodata)
## Read an verify the updateinfo.xml.gz
uinfo = UpdateMetadata()
uinfo.add(updateinfo)
print(testutil.get_log())
assert len(uinfo.get_notices()) == 2, len(uinfo.get_notices())
assert uinfo.get_notice(get_nvr('TurboGears-1.0.2.2-2.fc7'))
notice = uinfo.get_notice(get_nvr(update.title))
assert notice, 'Cannot find update for %r' % get_nvr(update.title)
assert notice['status'] == update.status
assert notice['updated'] == update.date_modified
assert notice['from'] == str(config.get('bodhi_email'))
assert notice['description'] == update.notes
assert notice['issued'] is not None
assert notice['update_id'] == update.updateid
## Clean up
shutil.rmtree(temprepo)
del (koji.__untag__[:])
def __init__(self, repo, cacheduinfo=None):
self.tag = get_repo_tag(repo)
self.repo = repo
self.doc = None
self.updates = set()
self.builds = {}
self._from = config.get('bodhi_email')
self.koji = get_session()
self._create_document()
self._fetch_updates()
missing_ids = []
if cacheduinfo and exists(cacheduinfo):
log.debug("Loading cached updateinfo.xml.gz")
umd = UpdateMetadata()
umd.add(cacheduinfo)
# Drop the old cached updateinfo.xml.gz, it's unneeded now
os.unlink(cacheduinfo)
existing_ids = set([up['update_id'] for up in umd.get_notices()])
seen_ids = set()
from_cache = set()
# Generate metadata for any new builds
for update in self.updates:
if update.updateid:
seen_ids.add(update.updateid)
if update.updateid in existing_ids:
notice = umd.get_notice(update.title)
if not notice:
log.warn(
'%s ID in cache but notice cannot be found' %
(update.title))
self.add_update(update)
continue
if notice['updated']:
if datetime.strptime(notice['updated'],
'%Y-%m-%d %H:%M:%S'
) < update.date_modified:
log.debug(
'Update modified, generating new notice: %s'
% update.title)
self.add_update(update)
else:
log.debug('Loading updated %s from cache' %
update.title)
from_cache.add(update.updateid)
elif update.date_modified:
log.debug(
'Update modified, generating new notice: %s' %
update.title)
self.add_update(update)
else:
log.debug('Loading %s from cache' % update.title)
from_cache.add(update.updateid)
else:
log.debug('Adding new update notice: %s' %
update.title)
self.add_update(update)
else:
missing_ids.append(update.title)
# Add all relevant notices from the cache to this document
for notice in umd.get_notices():
if notice['update_id'] in from_cache:
log.debug("Keeping existing notice: %s" % notice['title'])
self._add_notice(notice)
else:
# Keep all security notices in the stable repo
if 'testing' not in self.repo:
if notice['type'] == 'security':
if notice['update_id'] not in seen_ids:
log.debug(
"Keeping existing security notice: %s" %
notice['title'])
self._add_notice(notice)
else:
log.debug('%s already added?' %
notice['title'])
else:
log.debug('Purging cached stable notice %s' %
notice['title'])
else:
log.debug('Purging cached testing update %s' %
notice['title'])
# Clean metadata generation
else:
log.debug("Generating new updateinfo.xml")
for update in self.updates:
if update.updateid:
self.add_update(update)
else:
missing_ids.append(update.title)
# Add *all* security updates
# TODO: only the most recent
#for update in PackageUpdate.select(PackageUpdate.q.type=='security'):
# self.add_update(update)
if missing_ids:
log.error("%d updates with missing ID!" % len(missing_ids))
log.debug(missing_ids)
class UpdatesDaemon(yum.YumBase):
"""Class to implement the update checking daemon."""
def __init__(self, opts):
yum.YumBase.__init__(self)
self.opts = opts
self.didSetup = False
self.emitters = []
if 'dbus' in self.opts.emit_via:
self.emitters.append(DbusUpdateEmitter())
if 'email' in self.opts.emit_via:
self.emitters.append(
EmailUpdateEmitter(self.opts.email_from, self.opts.email_to))
if 'syslog' in self.opts.emit_via:
self.emitters.append(
SyslogUpdateEmitter(self.opts.syslog_facility,
self.opts.syslog_ident,
self.opts.syslog_level))
self.updateInfo = []
self.updateInfoTime = None
def doSetup(self):
"""Perform set up, including setting up directories and
parsing options.
"""
# if we are not root do the special subdir thing
if os.geteuid() != 0:
if not os.path.exists(self.opts.nonroot_workdir):
os.makedirs(self.opts.nonroot_workdir)
self.repos.setCacheDir(self.opts.nonroot_workdir)
self.doConfigSetup(fn=self.opts.yum_config)
def refreshUpdates(self):
"""Retrieve information about what updates are available."""
self.doLock()
try:
self.doRepoSetup()
self.doSackSetup()
self.updateCheckSetup()
except Exception as e:
syslog.syslog(syslog.LOG_WARNING,
"error getting update info: %s" % (e, ))
self.emitCheckFailed("%s" % (e, ))
self.doUnlock()
return False
return True
def populateUpdateMetadata(self):
"""Populate the metadata for the packages in the update."""
self.updateMetadata = UpdateMetadata()
repos = []
for (new, old) in self.up.getUpdatesTuples():
pkg = self.getPackageObject(new)
if pkg.repoid not in repos:
repo = self.repos.getRepo(pkg.repoid)
repos.append(repo.id)
try: # grab the updateinfo.xml.gz from the repodata
md = repo.retrieveMD('updateinfo')
except Exception: # can't find any; silently move on
continue
md = gzip.open(md)
self.updateMetadata.add(md)
md.close()
def populateUpdates(self):
"""Retrieve and set up information about the updates available
for installed packages.
"""
def getDbusPackageDict(pkg):
"""Returns a dictionary corresponding to the package object
in the form that we can send over the wire for dbus."""
pkgDict = {
"name": pkg.name,
"version": pkg.version,
"release": pkg.release,
"epoch": pkg.epoch,
"arch": pkg.arch,
"sourcerpm": pkg.sourcerpm,
"summary": pkg.summary or "",
}
# check if any updateinfo is available
md = self.updateMetadata.get_notice((pkg.name, pkg.ver, pkg.rel))
if md:
# right now we only want to know if it is a security update
pkgDict['type'] = md['type']
return pkgDict
if self.up is None:
# we're _only_ called after updates are setup
return
self.populateUpdateMetadata()
self.updateInfo = []
for (new, old) in self.up.getUpdatesTuples():
n = getDbusPackageDict(self.getPackageObject(new))
o = getDbusPackageDict(self.rpmdb.searchPkgTuple(old)[0])
self.updateInfo.append((n, o))
if self.conf.obsoletes:
for (obs, inst) in self.up.getObsoletesTuples():
n = getDbusPackageDict(self.getPackageObject(obs))
o = getDbusPackageDict(self.rpmdb.searchPkgTuple(inst)[0])
self.updateInfo.append((n, o))
self.updateInfoTime = time.time()
def populateTsInfo(self):
"""Set up information about the update in the tsInfo object."""
# figure out the updates
for (new, old) in self.up.getUpdatesTuples():
updating = self.getPackageObject(new)
updated = self.rpmdb.searchPkgTuple(old)[0]
self.tsInfo.addUpdate(updating, updated)
# and the obsoletes
if self.conf.obsoletes:
for (obs, inst) in self.up.getObsoletesTuples():
obsoleting = self.getPackageObject(obs)
installed = self.rpmdb.searchPkgTuple(inst)[0]
self.tsInfo.addObsoleting(obsoleting, installed)
self.tsInfo.addObsoleted(installed, obsoleting)
def updatesCheck(self):
"""Check to see whether updates are available for any
installed packages. If updates are available, install them,
download them, or just emit a message, depending on what
options are selected in the configuration file.
:return: whether the daemon should continue looping
"""
if not self.didSetup:
try:
self.doSetup()
except Exception as e:
syslog.syslog(syslog.LOG_WARNING, "error initializing: %s" % e)
if isinstance(e, yum.plugins.PluginYumExit):
self.emitSetupFailed(e.value, e.translation_domain)
else:
# if we don't know where the string is from, then assume
# it's not marked for translation (versus sending
# gettext.textdomain() and assuming it's from the default
# domain for this app)
self.emitSetupFailed(str(e))
# Setup failed, let's restart and try again after the update
# interval
restart()
else:
self.didSetup = True
try:
if not self.refreshUpdates():
return
except yum.Errors.LockError:
return True # just pass for now
try:
self.populateTsInfo()
self.populateUpdates()
if self.opts.do_update:
uit = UpdateInstallThread(self)
uit.start()
elif self.opts.do_download:
self.emitDownloading()
dl = UpdateDownloadThread(self)
dl.start()
else:
# just notify about things being available
self.emitAvailable()
self.releaseLocks()
except Exception as e:
self.emitCheckFailed("%s" % (e, ))
self.doUnlock()
return True
def getUpdateInfo(self):
"""Return information about the update. This may be
previously cached information if the configured time interval
between update retrievals has not yet elapsed, or there is an
error in trying to retrieve the update information.
:return: a list of tuples of dictionaries. Each
dictionary contains information about a package, and each
tuple specifies an available upgrade: the second dictionary
in the tuple has information about a package that is
currently installed, and the first dictionary has
information what the package can be upgraded to
"""
# if we have a cached copy, use it
if self.updateInfoTime and (time.time() - self.updateInfoTime <
self.opts.updaterefresh):
return self.updateInfo
# try to get the lock so we can update the info. fall back to
# cached if available or try a few times.
for i in range(10):
try:
self.doLock()
break
except yum.Errors.LockError:
# if we can't get the lock, return what we have if we can
if self.updateInfo:
return self.updateInfo
time.sleep(1)
else:
return []
try:
self.updateCheckSetup()
self.populateUpdates()
self.releaseLocks()
except:
self.doUnlock()
return self.updateInfo
def updateCheckSetup(self):
"""Set up the transaction set, rpm database, and prepare to
get updates.
"""
self.doTsSetup()
self.doRpmDBSetup()
self.doUpdateSetup()
def releaseLocks(self):
"""Close the rpm database, and release the yum lock."""
self.closeRpmDB()
self.doUnlock()
def emitAvailable(self):
"""Emit a notice stating whether updates are available."""
list(map(lambda x: x.updatesAvailable(self.updateInfo), self.emitters))
def emitDownloading(self):
"""Emit a notice stating that updates are downloading."""
list(
map(lambda x: x.updatesDownloading(self.updateInfo),
self.emitters))
def emitUpdateApplied(self):
"""Emit a notice stating that automatic updates have been applied."""
list(map(lambda x: x.updatesApplied(self.updateInfo), self.emitters))
def emitUpdateFailed(self, errmsgs):
"""Emit a notice stating that automatic updates failed."""
list(map(lambda x: x.updatesFailed(errmsgs), self.emitters))
def emitCheckFailed(self, error):
"""Emit a notice stating that checking for updates failed."""
list(map(lambda x: x.checkFailed(error), self.emitters))
def emitSetupFailed(self, error, translation_domain=""):
"""Emit a notice stating that checking for updates failed."""
list(
map(lambda x: x.setupFailed(error, translation_domain),
self.emitters))
def test_extended_metadata_updating(self):
# grab the name of a build in updates-testing, and create it in our db
koji = get_session()
builds = koji.listTagged('dist-f13-updates-testing', latest=True)
# Create all of the necessary database entries
release = Release(name='F13',
long_name='Fedora 13',
id_prefix='FEDORA',
dist_tag='dist-f13')
package = Package(name=builds[0]['package_name'])
update = PackageUpdate(title=builds[0]['nvr'],
release=release,
submitter=builds[0]['owner_name'],
status='testing',
notes='foobar',
type='bugfix')
build = PackageBuild(nvr=builds[0]['nvr'], package=package)
update.addPackageBuild(build)
bug = Bugzilla(bz_id=1)
bug.title = u'test bug'
update.addBugzilla(bug)
cve = CVE(cve_id="CVE-2007-0000")
update.addCVE(cve)
update.assign_id()
## Initialize our temporary repo
temprepo = join(tempfile.mkdtemp('bodhi'), 'f7-updates-testing')
print "Inserting updateinfo into temprepo: %s" % temprepo
mkmetadatadir(join(temprepo, 'i386'))
repodata = join(temprepo, 'i386', 'repodata')
assert exists(join(repodata, 'repomd.xml'))
## Generate the XML
md = ExtendedMetadata(temprepo)
## Insert the updateinfo.xml into the repository
md.insert_updateinfo()
updateinfo = join(repodata, 'updateinfo.xml.gz')
assert exists(updateinfo)
## Read an verify the updateinfo.xml.gz
uinfo = UpdateMetadata()
uinfo.add(updateinfo)
notice = uinfo.get_notice(('mutt', '1.5.14', '1.fc13'))
assert not notice
notice = uinfo.get_notice(get_nvr(update.title))
assert notice
assert notice['status'] == update.status
assert notice['updated'] == update.date_modified
assert notice['from'] == str(config.get('bodhi_email'))
assert notice['description'] == update.notes
assert notice['issued'] != None
assert notice['update_id'] == update.updateid
cve = notice['references'][0]
assert cve['type'] == 'cve'
assert cve['href'] == update.cves[0].get_url()
assert cve['id'] == update.cves[0].cve_id
bug = notice['references'][1]
assert bug['href'] == update.bugs[0].get_url()
assert bug['id'] == '1'
assert bug['type'] == 'bugzilla'
assert bug['title'] == 'test bug'
# FC6's yum update metadata parser doesn't know about some stuff
from yum import __version__
if __version__ >= '3.0.6':
assert notice['title'] == update.title
assert notice['release'] == update.release.long_name
assert cve['title'] == None
## Test out updateinfo.xml updating via our ExtendedMetadata
md = ExtendedMetadata(temprepo, updateinfo)
md.insert_updateinfo()
updateinfo = join(repodata, 'updateinfo.xml.gz')
assert exists(updateinfo)
## Read an verify the updateinfo.xml.gz
uinfo = UpdateMetadata()
uinfo.add(updateinfo)
notice = uinfo.get_notice(('mutt', '1.5.14', '1.fc13'))
assert not notice
notice = uinfo.get_notice(get_nvr(update.title))
assert notice
assert notice['status'] == update.status
assert notice['updated'] == update.date_modified
assert notice['from'] == str(config.get('bodhi_email'))
assert notice['description'] == update.notes
assert notice['issued'] != None
assert notice['update_id'] == update.updateid
cve = notice['references'][0]
assert cve['type'] == 'cve'
assert cve['href'] == update.cves[0].get_url()
assert cve['id'] == update.cves[0].cve_id
bug = notice['references'][1]
assert bug['href'] == update.bugs[0].get_url()
assert bug['id'] == '1'
assert bug['type'] == 'bugzilla'
assert bug['title'] == 'test bug', bug
# FC6's yum update metadata parser doesn't know about some stuff
from yum import __version__
if __version__ >= '3.0.6':
assert notice['title'] == update.title
assert notice['release'] == update.release.long_name
assert cve['title'] == None
## Clean up
shutil.rmtree(temprepo)
def __init__(self, repo, cacheduinfo=None):
self.tag = get_repo_tag(repo)
self.repo = repo
self.doc = None
self.updates = set()
self.builds = {}
self._from = config.get('bodhi_email')
self.koji = get_session()
self._create_document()
self._fetch_updates()
missing_ids = []
if cacheduinfo and exists(cacheduinfo):
log.debug("Loading cached updateinfo.xml.gz")
umd = UpdateMetadata()
umd.add(cacheduinfo)
existing_ids = set([up['update_id'] for up in umd.get_notices()])
seen_ids = set()
from_cache = set()
# Generate metadata for any new builds
for update in self.updates:
if update.updateid:
seen_ids.add(update.updateid)
if update.updateid in existing_ids:
notice = umd.get_notice(update.title)
if not notice:
log.warn('%s ID in cache but notice cannot be found' % (update.title))
self.add_update(update)
continue
if notice['updated']:
if datetime.strptime(notice['updated'], '%Y-%m-%d %H:%M:%S') < update.date_modified:
log.debug('Update modified, generating new notice: %s' % update.title)
self.add_update(update)
else:
log.debug('Loading updated %s from cache' % update.title)
from_cache.append(update.updateid)
elif update.date_modified:
log.debug('Update modified, generating new notice: %s' % update.title)
self.add_update(update)
else:
log.debug('Loading %s from cache' % update.title)
from_cache.add(update.updateid)
else:
log.debug('Adding new update notice: %s' % update.title)
self.add_update(update)
else:
missing_ids.append(update.title)
# Add all relevant notices from the cache to this document
for notice in umd.get_notices():
if notice['update_id'] in from_cache:
log.debug("Keeping existing notice: %s" % notice['title'])
self._add_notice(notice)
else:
# Keep all security notices in the stable repo
if 'testing' not in self.repo:
if notice['type'] == 'security':
if notice['update_id'] not in seen_ids:
log.debug("Keeping existing security notice: %s" %
notice['title'])
self._add_notice(notice)
else:
log.debug('%s already added?' % notice['title'])
else:
log.debug('Purging cached stable notice %s' % notice['title'])
else:
log.debug('Purging cached testing update %s' % notice['title'])
# Clean metadata generation
else:
log.debug("Generating new updateinfo.xml")
for update in self.updates:
if update.updateid:
self.add_update(update)
else:
missing_ids.append(update.title)
# Add *all* security updates
# TODO: only the most recent
#for update in PackageUpdate.select(PackageUpdate.q.type=='security'):
# self.add_update(update)
if missing_ids:
log.error("%d updates with missing ID!" % len(missing_ids))
log.debug(missing_ids)
def test_extended_metadata_updating_with_edited_updates(self):
testutil.capture_log(['bodhi.metadata'])
# grab the name of a build in updates-testing, and create it in our db
koji = get_session()
builds = koji.listTagged('dist-f13-updates-testing', latest=True)
# Create all of the necessary database entries
release = Release(name='F13',
long_name='Fedora 13',
id_prefix='FEDORA',
dist_tag='dist-f13')
package = Package(name=builds[0]['package_name'])
update = PackageUpdate(title=builds[0]['nvr'],
release=release,
submitter=builds[0]['owner_name'],
status='testing',
notes='foobar',
type='bugfix')
build = PackageBuild(nvr=builds[0]['nvr'], package=package)
update.addPackageBuild(build)
update.assign_id()
## Initialize our temporary repo
temprepo = join(tempfile.mkdtemp('bodhi'), 'f13-updates-testing')
print "Inserting updateinfo into temprepo: %s" % temprepo
mkmetadatadir(join(temprepo, 'i386'))
repodata = join(temprepo, 'i386', 'repodata')
assert exists(join(repodata, 'repomd.xml'))
## Generate the XML
md = ExtendedMetadata(temprepo)
## Insert the updateinfo.xml into the repository
md.insert_updateinfo()
updateinfo = self.__verify_updateinfo(repodata)
## Read an verify the updateinfo.xml.gz
uinfo = UpdateMetadata()
uinfo.add(updateinfo)
notice = uinfo.get_notice(('mutt', '1.5.14', '1.fc13'))
assert not notice
notice = uinfo.get_notice(get_nvr(update.title))
assert notice
assert notice['status'] == update.status
assert notice['updated'] == update.date_modified
assert notice['from'] == str(config.get('bodhi_email'))
assert notice['description'] == update.notes
assert notice['issued'] is not None
assert notice['update_id'] == update.updateid
assert notice['title'] == update.title
assert notice['release'] == update.release.long_name
## Edit the update and bump the build revision
nvr = 'TurboGears-1.0.2.2-3.fc7'
newbuild = PackageBuild(nvr=nvr, package=package)
update.removePackageBuild(build)
update.addPackageBuild(newbuild)
update.title = nvr
update.date_modified = datetime.utcnow()
# Pretend -2 was unpushed
assert len(koji.__untag__) == 0
koji.__untag__.append('TurboGears-1.0.2.2-2.fc7')
## Test out updateinfo.xml updating via our ExtendedMetadata
md = ExtendedMetadata(temprepo, updateinfo)
md.insert_updateinfo()
updateinfo = self.__verify_updateinfo(repodata)
## Read an verify the updateinfo.xml.gz
uinfo = UpdateMetadata()
uinfo.add(updateinfo)
print(testutil.get_log())
notice = uinfo.get_notice(('TurboGears', '1.0.2.2', '2.fc7'))
assert not notice, "Old TG notice did not get pruned: %s" % notice
notice = uinfo.get_notice(('TurboGears', '1.0.2.2', '3.fc7'))
assert notice, uinfo
assert notice['title'] == update.title
num_notices = len(uinfo.get_notices())
assert num_notices == 1, num_notices
## Clean up
shutil.rmtree(temprepo)
del (koji.__untag__[:])