def create_combined_metadata(repo, dest, comps=None):
""" Creates YUM metadata for the entire Packages directory.
When used with versioning, this creates a combined repository of all
packages ever synced for the repository.
"""
combined_repo = set_path(repo, util.get_packages_dir(dest))
create_metadata(combined_repo, None, comps)
def create_combined_metadata(repo, dest, comps=None):
""" Creates YUM metadata for the entire Packages directory.
When used with versioning, this creates a combined repository of all
packages ever synced for the repository.
"""
combined_repo = set_path(repo, util.get_packages_dir(dest))
create_metadata(combined_repo, None, comps)
def sync(repo,
dest,
version,
delete=False,
combined=False,
yumcallback=None,
repocallback=None,
includelist=None):
""" Sync repository contents from a remote source.
Accepts a repository, destination path, and an optional version, and uses
the YUM client library to download all available packages from the mirror.
If the delete flag is passed, any packages found on the local filesystem
which are not present in the remote repository will be deleted.
"""
util.make_dir(util.get_packages_dir(dest)) # Make package storage dir
@contextmanager
def suppress():
""" Suppress stdout within a context.
This is necessary in this use case because, unfortunately, the YUM
library will do direct printing to stdout in many error conditions.
Since we are maintaining a real-time, in-place updating presentation
of progress, we must suppress this, as we receive exceptions for our
reporting purposes anyways.
"""
stdout = sys.stdout
sys.stdout = open(os.devnull, 'w')
yield
sys.stdout = stdout
if version:
dest_dir = util.get_versioned_dir(dest, version)
util.make_dir(dest_dir)
packages_dir = util.get_packages_dir(dest_dir)
util.symlink(packages_dir, util.get_relative_packages_dir())
else:
dest_dir = dest
packages_dir = util.get_packages_dir(dest_dir)
try:
yb = util.get_yum()
repo = set_path(repo, packages_dir)
if yumcallback:
repo.setCallback(yumcallback)
yb.repos.add(repo)
yb.repos.enableRepo(repo.id)
with suppress():
# showdups allows us to get multiple versions of the same package.
ygh = yb.doPackageLists(showdups=True)
# reinstall_available = Available packages which are installed.
packages = ygh.available + ygh.reinstall_available
try:
packages = [p for p in packages if p.name in includelist]
except TypeError:
pass
# Inform about number of packages total in the repo.
callback(repocallback, repo, 'repo_init', len(packages))
# Check if the packages are already downloaded. This is probably a bit
# expensive, but the alternative is simply not knowing, which is
# horrible for progress indication.
for po in packages:
local = po.localPkg()
if os.path.exists(local):
if yb.verifyPkg(local, po, False):
callback(repocallback, repo, 'local_pkg_exists',
util.get_package_filename(po))
with suppress():
yb.downloadPkgs(packages)
except (KeyboardInterrupt, SystemExit):
pass
except Exception, e:
callback(repocallback, repo, 'repo_error', str(e))
log.error(str(e))
return False
with suppress():
yb.downloadPkgs(packages)
except (KeyboardInterrupt, SystemExit):
pass
except Exception, e:
callback(repocallback, repo, 'repo_error', str(e))
log.error(str(e))
return False
callback(repocallback, repo, 'repo_complete')
if delete:
package_names = []
for package in packages:
package_names.append(util.get_package_filename(package))
for _file in os.listdir(util.get_packages_dir(dest)):
if not _file in package_names:
package_path = util.get_package_path(dest, _file)
log.debug('Deleting file %s' % package_path)
os.remove(package_path)
log.info('Finished downloading packages from repository %s' % repo.id)
log.info('Creating metadata for repository %s' % repo.id)
callback(repocallback, repo, 'repo_metadata', 'working')
comps = retrieve_group_comps(repo) # try group data
pkglist = []
for pkg in packages:
pkglist.append(
util.get_package_relativedir(util.get_package_filename(pkg)))
create_metadata(repo, pkglist, comps)
def sync(repo, dest, version, delete=False, combined=False, yumcallback=None,
repocallback=None):
""" Sync repository contents from a remote source.
Accepts a repository, destination path, and an optional version, and uses
the YUM client library to download all available packages from the mirror.
If the delete flag is passed, any packages found on the local filesystem
which are not present in the remote repository will be deleted.
"""
util.make_dir(util.get_packages_dir(dest)) # Make package storage dir
@contextmanager
def suppress():
""" Suppress stdout within a context.
This is necessary in this use case because, unfortunately, the YUM
library will do direct printing to stdout in many error conditions.
Since we are maintaining a real-time, in-place updating presentation
of progress, we must suppress this, as we receive exceptions for our
reporting purposes anyways.
"""
stdout = sys.stdout
sys.stdout = open(os.devnull, 'w')
yield
sys.stdout = stdout
if version:
dest_dir = util.get_versioned_dir(dest, version)
util.make_dir(dest_dir)
packages_dir = util.get_packages_dir(dest_dir)
util.symlink(packages_dir, util.get_relative_packages_dir())
else:
dest_dir = dest
packages_dir = util.get_packages_dir(dest_dir)
try:
yb = util.get_yum()
repo = set_path(repo, packages_dir)
if yumcallback:
repo.setCallback(yumcallback)
yb.repos.add(repo)
yb.repos.enableRepo(repo.id)
with suppress():
# showdups allows us to get multiple versions of the same package.
ygh = yb.doPackageLists(showdups=True)
# reinstall_available = Available packages which are installed.
packages = ygh.available + ygh.reinstall_available
# Inform about number of packages total in the repo.
callback(repocallback, repo, 'repo_init', len(packages))
# Check if the packages are already downloaded. This is probably a bit
# expensive, but the alternative is simply not knowing, which is
# horrible for progress indication.
for po in packages:
local = po.localPkg()
if os.path.exists(local):
if yb.verifyPkg(local, po, False):
callback(repocallback, repo, 'local_pkg_exists',
util.get_package_filename(po))
with suppress():
yb.downloadPkgs(packages)
except (KeyboardInterrupt, SystemExit):
pass
except Exception, e:
callback(repocallback, repo, 'repo_error', str(e))
log.error(str(e))
return False
with suppress():
yb.downloadPkgs(packages)
except (KeyboardInterrupt, SystemExit):
pass
except Exception, e:
callback(repocallback, repo, 'repo_error', str(e))
log.error(str(e))
return False
callback(repocallback, repo, 'repo_complete')
if delete:
package_names = []
for package in packages:
package_names.append(util.get_package_filename(package))
for _file in os.listdir(util.get_packages_dir(dest)):
if not _file in package_names:
package_path = util.get_package_path(dest, _file)
log.debug('Deleting file %s' % package_path)
os.remove(package_path)
log.info('Finished downloading packages from repository %s' % repo.id)
log.info('Creating metadata for repository %s' % repo.id)
callback(repocallback, repo, 'repo_metadata', 'working')
comps = retrieve_group_comps(repo) # try group data
pkglist = []
for pkg in packages:
pkglist.append(
util.get_package_relativedir(util.get_package_filename(pkg))
)
def sync(repo, dest, version, delete=False, combined=False, yumcallback=None,
repocallback=None):
""" Sync repository contents from a remote source.
Accepts a repository, destination path, and an optional version, and uses
the YUM client library to download all available packages from the mirror.
If the delete flag is passed, any packages found on the local filesystem
which are not present in the remote repository will be deleted.
"""
util.make_dir(util.get_packages_dir(dest)) # Make package storage dir
@contextmanager
def suppress():
""" Suppress stdout within a context.
This is necessary in this use case because, unfortunately, the YUM
library will do direct printing to stdout in many error conditions.
Since we are maintaining a real-time, in-place updating presentation
of progress, we must suppress this, as we receive exceptions for our
reporting purposes anyways.
"""
stdout = sys.stdout
sys.stdout = open(os.devnull, 'w')
yield
sys.stdout = stdout
if version:
dest_dir = util.get_versioned_dir(dest, version)
util.make_dir(dest_dir)
packages_dir = util.get_packages_dir(dest_dir)
util.symlink(packages_dir, util.get_relative_packages_dir())
else:
dest_dir = dest
packages_dir = util.get_packages_dir(dest_dir)
try:
yb = util.get_yum()
repo = set_path(repo, packages_dir)
if yumcallback:
repo.setCallback(yumcallback)
yb.repos.add(repo)
yb.repos.enableRepo(repo.id)
with suppress():
# showdups allows us to get multiple versions of the same package.
ygh = yb.doPackageLists(showdups=True)
# reinstall_available = Available packages which are installed.
packages = ygh.available + ygh.reinstall_available
# Inform about number of packages total in the repo.
callback(repocallback, repo, 'repo_init', len(packages))
# Check if the packages are already downloaded. This is probably a bit
# expensive, but the alternative is simply not knowing, which is
# horrible for progress indication.
for po in packages:
local = po.localPkg()
if os.path.exists(local):
if yb.verifyPkg(local, po, False):
callback(repocallback, repo, 'local_pkg_exists',
util.get_package_filename(po))
with suppress():
yb.downloadPkgs(packages)
except (KeyboardInterrupt, SystemExit):
pass
except Exception as e:
callback(repocallback, repo, 'repo_error', str(e))
log.error(str(e))
return False
callback(repocallback, repo, 'repo_complete')
if delete:
package_names = []
for package in packages:
package_names.append(util.get_package_filename(package))
for _file in os.listdir(util.get_packages_dir(dest)):
if not _file in package_names:
package_path = util.get_package_path(dest, _file)
log.debug('Deleting file %s' % package_path)
os.remove(package_path)
log.info('Finished downloading packages from repository %s' % repo.id)
log.info('Creating metadata for repository %s' % repo.id)
callback(repocallback, repo, 'repo_metadata', 'working')
comps = retrieve_group_comps(repo) # try group data
pkglist = []
for pkg in packages:
pkglist.append(
util.get_package_relativedir(util.get_package_filename(pkg))
)
create_metadata(repo, pkglist, comps)
if combined and version:
create_combined_metadata(repo, dest, comps)
elif os.path.exists(util.get_metadata_dir(dest)):
# At this point the combined metadata is stale, so remove it.
log.debug('Removing combined metadata for repository %s' % repo.id)
shutil.rmtree(util.get_metadata_dir(dest))
callback(repocallback, repo, 'repo_metadata', 'complete')
log.info('Finished creating metadata for repository %s' % repo.id)
if version:
latest_symlink = util.get_latest_symlink_path(dest)
util.symlink(latest_symlink, version)