def retrieve_group_comps(repo):
""" Retrieve group comps XML data from a remote repository.
This data can be used while running createrepo to provide package groups
data that clients can use while installing software.
"""
if repo.enablegroups:
try:
yb = util.get_yum()
yb.repos.add(repo)
comps = yb._getGroups().xml()
log.info('Group data retrieved for repository %s' % repo.id)
return comps
except yum.Errors.GroupsError:
log.debug('No group data available for repository %s' % repo.id)
return None
def factory(name, baseurls=None, mirrorlist=None):
""" Generate a pakrat.yumbase.YumBase object on-the-fly.
This makes it possible to mirror YUM repositories without having any stored
configuration anywhere. Simply pass in the name of the repository, and
either one or more baseurl's or a mirrorlist URL, and you will get an
object in return that you can pass to a mirroring function.
"""
yb = util.get_yum()
if baseurls is not None:
util.validate_baseurls(baseurls)
repo = yb.add_enable_repo(name, baseurls=baseurls)
elif mirrorlist is not None:
util.validate_mirrorlist(mirrorlist)
repo = yb.add_enable_repo(name, mirrorlist=mirrorlist)
else:
raise Exception('One or more baseurls or mirrorlist required')
return repo
def from_file(path):
""" Read repository configuration from a YUM config file.
Using the YUM client library, read in a configuration file and return
a list of repository objects.
"""
if not os.path.exists(path):
raise Exception('No such file or directory: %s' % path)
yb = util.get_yum()
yb.getReposFromConfigFile(path)
for repo in yb.repos.findRepos('*'):
yb.doSackSetup(thisrepo=repo.getAttribute('name'))
repos = []
for repo in yb.repos.findRepos('*'):
if repo.isEnabled():
log.info('Added repo %s from file %s' % (repo.id, path))
repos.append(repo)
else:
log.debug('Not adding repo %s because it is disabled' % repo.id)
return repos
def sync(repo,
dest,
version,
delete=False,
combined=False,
yumcallback=None,
repocallback=None,
includelist=None):
""" Sync repository contents from a remote source.
Accepts a repository, destination path, and an optional version, and uses
the YUM client library to download all available packages from the mirror.
If the delete flag is passed, any packages found on the local filesystem
which are not present in the remote repository will be deleted.
"""
util.make_dir(util.get_packages_dir(dest)) # Make package storage dir
@contextmanager
def suppress():
""" Suppress stdout within a context.
This is necessary in this use case because, unfortunately, the YUM
library will do direct printing to stdout in many error conditions.
Since we are maintaining a real-time, in-place updating presentation
of progress, we must suppress this, as we receive exceptions for our
reporting purposes anyways.
"""
stdout = sys.stdout
sys.stdout = open(os.devnull, 'w')
yield
sys.stdout = stdout
if version:
dest_dir = util.get_versioned_dir(dest, version)
util.make_dir(dest_dir)
packages_dir = util.get_packages_dir(dest_dir)
util.symlink(packages_dir, util.get_relative_packages_dir())
else:
dest_dir = dest
packages_dir = util.get_packages_dir(dest_dir)
try:
yb = util.get_yum()
repo = set_path(repo, packages_dir)
if yumcallback:
repo.setCallback(yumcallback)
yb.repos.add(repo)
yb.repos.enableRepo(repo.id)
with suppress():
# showdups allows us to get multiple versions of the same package.
ygh = yb.doPackageLists(showdups=True)
# reinstall_available = Available packages which are installed.
packages = ygh.available + ygh.reinstall_available
try:
packages = [p for p in packages if p.name in includelist]
except TypeError:
pass
# Inform about number of packages total in the repo.
callback(repocallback, repo, 'repo_init', len(packages))
# Check if the packages are already downloaded. This is probably a bit
# expensive, but the alternative is simply not knowing, which is
# horrible for progress indication.
for po in packages:
local = po.localPkg()
if os.path.exists(local):
if yb.verifyPkg(local, po, False):
callback(repocallback, repo, 'local_pkg_exists',
util.get_package_filename(po))
with suppress():
yb.downloadPkgs(packages)
except (KeyboardInterrupt, SystemExit):
pass
except Exception, e:
callback(repocallback, repo, 'repo_error', str(e))
log.error(str(e))
return False
def sync(repo, dest, version, delete=False, combined=False, yumcallback=None,
repocallback=None):
""" Sync repository contents from a remote source.
Accepts a repository, destination path, and an optional version, and uses
the YUM client library to download all available packages from the mirror.
If the delete flag is passed, any packages found on the local filesystem
which are not present in the remote repository will be deleted.
"""
util.make_dir(util.get_packages_dir(dest)) # Make package storage dir
@contextmanager
def suppress():
""" Suppress stdout within a context.
This is necessary in this use case because, unfortunately, the YUM
library will do direct printing to stdout in many error conditions.
Since we are maintaining a real-time, in-place updating presentation
of progress, we must suppress this, as we receive exceptions for our
reporting purposes anyways.
"""
stdout = sys.stdout
sys.stdout = open(os.devnull, 'w')
yield
sys.stdout = stdout
if version:
dest_dir = util.get_versioned_dir(dest, version)
util.make_dir(dest_dir)
packages_dir = util.get_packages_dir(dest_dir)
util.symlink(packages_dir, util.get_relative_packages_dir())
else:
dest_dir = dest
packages_dir = util.get_packages_dir(dest_dir)
try:
yb = util.get_yum()
repo = set_path(repo, packages_dir)
if yumcallback:
repo.setCallback(yumcallback)
yb.repos.add(repo)
yb.repos.enableRepo(repo.id)
with suppress():
# showdups allows us to get multiple versions of the same package.
ygh = yb.doPackageLists(showdups=True)
# reinstall_available = Available packages which are installed.
packages = ygh.available + ygh.reinstall_available
# Inform about number of packages total in the repo.
callback(repocallback, repo, 'repo_init', len(packages))
# Check if the packages are already downloaded. This is probably a bit
# expensive, but the alternative is simply not knowing, which is
# horrible for progress indication.
for po in packages:
local = po.localPkg()
if os.path.exists(local):
if yb.verifyPkg(local, po, False):
callback(repocallback, repo, 'local_pkg_exists',
util.get_package_filename(po))
with suppress():
yb.downloadPkgs(packages)
except (KeyboardInterrupt, SystemExit):
pass
except Exception, e:
callback(repocallback, repo, 'repo_error', str(e))
log.error(str(e))
return False
def sync(repo, dest, version, delete=False, combined=False, yumcallback=None,
repocallback=None):
""" Sync repository contents from a remote source.
Accepts a repository, destination path, and an optional version, and uses
the YUM client library to download all available packages from the mirror.
If the delete flag is passed, any packages found on the local filesystem
which are not present in the remote repository will be deleted.
"""
util.make_dir(util.get_packages_dir(dest)) # Make package storage dir
@contextmanager
def suppress():
""" Suppress stdout within a context.
This is necessary in this use case because, unfortunately, the YUM
library will do direct printing to stdout in many error conditions.
Since we are maintaining a real-time, in-place updating presentation
of progress, we must suppress this, as we receive exceptions for our
reporting purposes anyways.
"""
stdout = sys.stdout
sys.stdout = open(os.devnull, 'w')
yield
sys.stdout = stdout
if version:
dest_dir = util.get_versioned_dir(dest, version)
util.make_dir(dest_dir)
packages_dir = util.get_packages_dir(dest_dir)
util.symlink(packages_dir, util.get_relative_packages_dir())
else:
dest_dir = dest
packages_dir = util.get_packages_dir(dest_dir)
try:
yb = util.get_yum()
repo = set_path(repo, packages_dir)
if yumcallback:
repo.setCallback(yumcallback)
yb.repos.add(repo)
yb.repos.enableRepo(repo.id)
with suppress():
# showdups allows us to get multiple versions of the same package.
ygh = yb.doPackageLists(showdups=True)
# reinstall_available = Available packages which are installed.
packages = ygh.available + ygh.reinstall_available
# Inform about number of packages total in the repo.
callback(repocallback, repo, 'repo_init', len(packages))
# Check if the packages are already downloaded. This is probably a bit
# expensive, but the alternative is simply not knowing, which is
# horrible for progress indication.
for po in packages:
local = po.localPkg()
if os.path.exists(local):
if yb.verifyPkg(local, po, False):
callback(repocallback, repo, 'local_pkg_exists',
util.get_package_filename(po))
with suppress():
yb.downloadPkgs(packages)
except (KeyboardInterrupt, SystemExit):
pass
except Exception as e:
callback(repocallback, repo, 'repo_error', str(e))
log.error(str(e))
return False
callback(repocallback, repo, 'repo_complete')
if delete:
package_names = []
for package in packages:
package_names.append(util.get_package_filename(package))
for _file in os.listdir(util.get_packages_dir(dest)):
if not _file in package_names:
package_path = util.get_package_path(dest, _file)
log.debug('Deleting file %s' % package_path)
os.remove(package_path)
log.info('Finished downloading packages from repository %s' % repo.id)
log.info('Creating metadata for repository %s' % repo.id)
callback(repocallback, repo, 'repo_metadata', 'working')
comps = retrieve_group_comps(repo) # try group data
pkglist = []
for pkg in packages:
pkglist.append(
util.get_package_relativedir(util.get_package_filename(pkg))
)
create_metadata(repo, pkglist, comps)
if combined and version:
create_combined_metadata(repo, dest, comps)
elif os.path.exists(util.get_metadata_dir(dest)):
# At this point the combined metadata is stale, so remove it.
log.debug('Removing combined metadata for repository %s' % repo.id)
shutil.rmtree(util.get_metadata_dir(dest))
callback(repocallback, repo, 'repo_metadata', 'complete')
log.info('Finished creating metadata for repository %s' % repo.id)
if version:
latest_symlink = util.get_latest_symlink_path(dest)
util.symlink(latest_symlink, version)