def commit(self):
os.environ["DEBIAN_FRONTEND"] = "noninteractive"
os.environ["DEBONF_NONINTERACTIVE_SEEN"] = "true"
print("Commiting changes ...")
self.cache.commit(ElbeAcquireProgress(),
ElbeInstallProgress(fileno=sys.stdout.fileno()))
self.cache.open(progress=ElbeOpProgress())
def download_binary(self, pkgname, path, version=None):
p = self.cache[pkgname]
if version is None:
pkgver = p.installed
else:
pkgver = p.versions[version]
rel_filename = fetch_binary(pkgver, path, ElbeAcquireProgress())
return self.rfs.fname(rel_filename)
def download_source( self, pkgname, path, version=None ):
p = self.cache[pkgname]
if version is None:
pkgver = p.installed
else:
pkgver = p.versions[version]
rel_filename = pkgver.fetch_source(path,
ElbeAcquireProgress(cb=self.co_cb), unpack=False)
return self.rfs.fname( rel_filename )
def mk_source_cdrom(rfs,
arch,
codename,
init_codename,
target,
log,
cdrom_size=CDROM_SIZE):
hostfs.mkdir_p('/var/cache/elbe/sources')
rfs.mkdir_p('/var/cache/elbe/sources')
repo = CdromSrcRepo(codename, init_codename,
os.path.join(target, "srcrepo"), log, cdrom_size)
cache = get_rpcaptcache(rfs, "aptcache.log", arch)
pkglist = cache.get_installed_pkgs()
for pkg in pkglist:
try:
dsc = cache.download_source(pkg.name, '/var/cache/elbe/sources')
repo.includedsc(dsc)
except ValueError as ve:
log.printo("No sources for Package " + pkg.name + "-" +
pkg.installed_version)
except FetchError as fe:
log.printo("Source for Package " + pkg.name + "-" +
pkg.installed_version + " could not be downloaded")
pkglist = get_initvm_pkglist()
cache = Cache()
cache.open()
for pkg in pkglist:
try:
p = cache[pkg.name]
if pkg.name == 'elbe-bootstrap':
pkgver = p.versions[0]
else:
pkgver = p.installed
dsc = pkgver.fetch_source('/var/cache/elbe/sources',
ElbeAcquireProgress(cb=None),
unpack=False)
repo.includedsc(dsc)
except ValueError as ve:
log.printo("No sources for Package " + pkg.name + "-" +
str(pkg.installed_version))
except FetchError as fe:
log.printo("Source for Package " + pkg.name + "-" +
pkgver.version + " could not be downloaded")
return repo.buildiso(os.path.join(target, "src-cdrom.iso"))
def download_binary(self, pkgname, path, version=None):
p = self.cache[pkgname]
if version is None:
pkgver = p.installed
else:
pkgver = p.versions[version]
# avoid DeprecationWarning:
# "MD5Hash is deprecated, use Hashes instead"
# triggerd by python-apt
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=DeprecationWarning)
rel_filename = pkgver.fetch_binary(path, ElbeAcquireProgress())
return self.rfs.fname(rel_filename)
def update(self):
self.cache.update(fetch_progress=ElbeAcquireProgress())
self.cache.open(progress=ElbeOpProgress())
def run_command(argv):
# TODO - Set threshold and remove pylint directives
#
# We might want to make the threshold higher for certain
# files/directories or just globaly.
# pylint: disable=too-many-locals
# pylint: disable=too-many-branches
# pylint: disable=too-many-statements
oparser = OptionParser(
usage="usage: %prog fetch_initvm_pkgs [options] <xmlfile>")
oparser.add_option("-b",
"--binrepo",
dest="binrepo",
default="/var/cache/elbe/initvm-bin-repo",
help="directory where the bin repo should reside")
oparser.add_option("-s",
"--srcrepo",
dest="srcrepo",
default="/var/cache/elbe/initvm-src-repo",
help="directory where the src repo should reside")
oparser.add_option("--skip-validation",
action="store_true",
dest="skip_validation",
default=False,
help="Skip xml schema validation")
oparser.add_option("--cdrom-mount-path",
dest="cdrom_path",
help="path where cdrom is mounted")
oparser.add_option("--cdrom-device",
dest="cdrom_device",
help="cdrom device, in case it has to be mounted")
oparser.add_option("--apt-archive",
dest="archive",
default="/var/cache/elbe/binaries/main",
help="path where binary packages are downloaded to.")
oparser.add_option("--src-archive",
dest="srcarchive",
default="/var/cache/elbe/sources",
help="path where src packages are downloaded to.")
oparser.add_option("--skip-build-sources",
action="store_false",
dest="build_sources",
default=True,
help="Skip downloading Source Packages")
oparser.add_option("--skip-build-bin",
action="store_false",
dest="build_bin",
default=True,
help="Skip downloading binary packages")
(opt, args) = oparser.parse_args(argv)
if len(args) != 1:
print("wrong number of arguments")
oparser.print_help()
sys.exit(20)
try:
xml = ElbeXML(args[0], skip_validate=opt.skip_validation)
except ValidationError as e:
print(str(e))
print("xml validation failed. Bailing out")
sys.exit(20)
with elbe_logging({"streams": sys.stdout}):
if opt.cdrom_path:
if opt.cdrom_device:
do('mount "%s" "%s"' % (opt.cdrom_device, opt.cdrom_path))
# a cdrom build is identified by the cdrom option
# the xml file that is copied into the initvm
# by the initrd does not have the cdrom tags setup.
mirror = "file://%s" % opt.cdrom_path
else:
mirror = xml.get_initvm_primary_mirror(opt.cdrom_path)
init_codename = xml.get_initvm_codename()
# Binary Repo
#
repo = CdromInitRepo(init_codename, opt.binrepo, mirror)
hostfs.mkdir_p(opt.archive)
if opt.build_bin:
pkglist = get_initvm_pkglist()
cache = Cache()
cache.open()
for pkg in pkglist:
pkg_id = "%s-%s" % (pkg.name, pkg.installed_version)
try:
p = cache[pkg.name]
pkgver = p.installed
deb = fetch_binary(pkgver, opt.archive,
ElbeAcquireProgress(cb=None))
repo.includedeb(deb, 'main')
except ValueError:
logging.exception('No package "%s"', pkg_id)
except FetchError:
logging.exception(
'Package "%s-%s" could not be downloaded', pkg.name,
pkgver.version)
except TypeError:
logging.exception('Package "%s" missing name or version',
pkg_id)
repo.finalize()
# Source Repo
#
repo = CdromSrcRepo(init_codename, init_codename, opt.srcrepo, 0,
mirror)
hostfs.mkdir_p(opt.srcarchive)
# a cdrom build does not have sources
# skip adding packages to the source repo
#
# FIXME: we need a way to add source cdroms later on
if opt.cdrom_path:
opt.build_sources = False
if opt.build_sources:
for pkg in pkglist:
pkg_id = "%s-%s" % (pkg.name, pkg.installed_version)
try:
p = cache[pkg.name]
pkgver = p.installed
dsc = pkgver.fetch_source(opt.srcarchive,
ElbeAcquireProgress(cb=None),
unpack=False)
repo.include_init_dsc(dsc, 'initvm')
except ValueError:
logging.exception('No package "%s"', pkg_id)
except FetchError:
logging.exception(
'Package "%s-%s" could not be downloaded', pkg.name,
pkgver.version)
except TypeError:
logging.exception('Package "%s" missing name or version',
pkg_id)
repo.finalize()
if opt.cdrom_device:
do('umount "%s"' % opt.cdrom_device)
def _apply_update(fname, status):
try:
xml = etree(fname)
except:
return "read %s failed" % fname
fpl = xml.node("fullpkgs")
sources = apt_pkg.SourceList()
sources.read_main_list()
status.log("initialize apt")
apt_pkg.init()
cache = apt_pkg.Cache(progress=ElbeOpProgress(cb=status.log))
status.set_progress(1)
status.log("updating package cache")
cache.update(ElbeAcquireProgress(cb=status.log), sources)
# quote from python-apt api doc: "A call to this method does not affect the
# current Cache object, instead a new one should be created in order to use
# the changed index files."
cache = apt_pkg.Cache(progress=ElbeOpProgress(cb=status.log))
depcache = apt_pkg.DepCache(cache)
hl_cache = apt.cache.Cache(progress=ElbeOpProgress(cb=status.log))
hl_cache.update(fetch_progress=ElbeAcquireProgress(cb=status.log))
# go through package cache, if a package is in the fullpkg list of the XML
# mark the package for installation (with the specified version)
# if it is not mentioned in the fullpkg list purge the package out of the
# system.
status.set_progress(2)
status.log("calculating packages to install/remove")
count = len(hl_cache)
step = count / 10
i = 0
percent = 0
for p in hl_cache:
i = i + 1
if not (i % step):
percent = percent + 10
status.log(str(percent) + "% - " + str(i) + "/" + str(count))
status.set_progress(2, str(percent) + "%")
pkg = cache[p.name]
marked = False
for fpi in fpl:
if pkg.name == fpi.et.text:
mark_install(depcache, pkg, fpi.et.get('version'),
fpi.et.get('auto'), status)
marked = True
if not marked:
depcache.mark_delete(pkg, True)
status.set_progress(3)
status.log("applying snapshot")
depcache.commit(ElbeAcquireProgress(cb=status.log),
ElbeInstallProgress(cb=status.log))
del depcache
del hl_cache
del cache
del sources
version_file = open("/etc/updated_version", "w")
version_file.write(xml.text("/project/version"))
version_file.close()
def download_source(self, src_name, src_version, dest_dir):
allow_untrusted = config.find_b("APT::Get::AllowUnauthenticated",
False)
rec = SourceRecords()
acq = Acquire(ElbeAcquireProgress())
# poorman's iterator
while True:
next_p = rec.lookup(src_name)
# End of the list?
if not next_p:
raise ("No source found for %s_%s" % (src_name, src_version))
if src_version == rec.version:
break
# We don't allow untrusted package and the package is not
# marks as trusted
if not (allow_untrusted or rec.index.is_trusted):
raise FetchError(
"Can't fetch source %s_%s; Source %r is not trusted" %
(src_name, src_version, rec.index.describe))
# Copy from src to dst all files of the source package
dsc = None
files = []
for _file in rec.files:
src = os.path.basename(_file.path)
dst = os.path.join(dest_dir, src)
if 'dsc' == _file.type:
dsc = dst
if not (allow_untrusted or _file.hashes.usable):
raise FetchError(
"Can't fetch file %s. No trusted hash found." % dst)
# acq is accumlating the AcquireFile, the files list only
# exists to prevent Python from GC the object .. I guess.
# Anyway, if we don't keep the list, We will get an empty
# directory
files.append(
AcquireFile(acq,
rec.index.archive_uri(_file.path),
_file.hashes,
_file.size,
src,
destfile=dst))
acq.run()
if dsc is None:
raise ValueError("No source found for %s_%s" %
(src_name, src_version))
for item in acq.items:
if item.STAT_DONE != item.status:
raise FetchError("Can't fetch item %s: %s" %
(item.destfile, item.error_text))
return self.rfs.fname(os.path.abspath(dsc))
def commit(self):
os.environ["DEBIAN_FRONTEND"] = "noninteractive"
os.environ["DEBONF_NONINTERACTIVE_SEEN"] = "true"
self.cache.commit(ElbeAcquireProgress(),
ElbeInstallProgress(fileno=self.logfile.fileno()))
self.cache.open(progress=ElbeOpProgress())
def mk_binary_cdrom(rfs,
arch,
codename,
init_codename,
xml,
target,
log,
cdrom_size=CDROM_SIZE):
rfs.mkdir_p('/var/cache/elbe/binaries/added')
rfs.mkdir_p('/var/cache/elbe/binaries/main')
hostfs.mkdir_p('/var/cache/elbe/binaries/main')
if not xml is None:
mirror = xml.get_primary_mirror(rfs.fname("cdrom"))
else:
mirror = 'http://ftp.debian.org/debian'
repo_path = os.path.join(target, "binrepo")
repo = CdromBinRepo(arch, codename, init_codename, repo_path, log,
cdrom_size, mirror)
if not xml is None:
pkglist = get_initvm_pkglist()
cache = Cache()
cache.open()
for pkg in pkglist:
try:
p = cache[pkg.name]
if pkg.name == 'elbe-bootstrap':
pkgver = p.versions[0]
else:
pkgver = p.installed
deb = pkgver.fetch_binary('/var/cache/elbe/binaries/main',
ElbeAcquireProgress(cb=None))
repo.include_init_deb(deb, 'main')
except ValueError as ve:
log.printo("No Package " + pkg.name + "-" +
str(pkg.installed_version))
except FetchError as fe:
log.printo("Package " + pkg.name + "-" + pkgver.version +
" could not be downloaded")
except TypeError as te:
log.printo("Package " + pkg.name + "-" +
str(pkg.installed_version) +
" missing name or version")
cache = get_rpcaptcache(rfs, "aptcache.log", arch)
for p in xml.node("debootstrappkgs"):
pkg = XMLPackage(p, arch)
try:
deb = cache.download_binary(pkg.name,
'/var/cache/elbe/binaries/main',
pkg.installed_version)
repo.includedeb(deb, 'main')
except ValueError as ve:
log.printo("No Package " + pkg.name + "-" +
pkg.installed_version)
except FetchError as fe:
log.printo("Package " + pkg.name + "-" +
pkg.installed_version + " could not be downloaded")
except TypeError as te:
log.printo("Package " + pkg.name + "-" +
pkg.installed_version + " missing name or version")
cache = get_rpcaptcache(rfs, "aptcache.log", arch)
pkglist = cache.get_installed_pkgs()
for pkg in pkglist:
try:
deb = cache.download_binary(pkg.name,
'/var/cache/elbe/binaries/added',
pkg.installed_version)
repo.includedeb(deb, 'added')
except KeyError as ke:
log.printo(str(ke))
except ValueError as ve:
log.printo("No Package " + pkg.name + "-" + pkg.installed_version)
except FetchError as fe:
log.printo("Package " + pkg.name + "-" +
str(pkg.installed_version) + " could not be downloaded")
except TypeError as te:
log.printo("Package " + pkg.name + "-" + pkg.installed_version +
" missing name or version")
# Mark the binary repo with the necessary Files
# to make the installer accept this as a CDRom
repo_fs = Filesystem(repo_path)
repo_fs.mkdir_p(".disk")
repo_fs.write_file(".disk/base_installable", 0644, "main\n")
repo_fs.write_file(".disk/base_components", 0644, "main\n")
repo_fs.write_file(".disk/cd_type", 0644, "not_complete\n")
repo_fs.write_file(".disk/info", 0644, "elbe inst cdrom - full cd\n")
repo_fs.symlink(".", "debian", allow_exists=True)
repo_fs.write_file("md5sum.txt", 0644, "")
# write source xml onto cdrom
xml.xml.write(repo_fs.fname('source.xml'))
return repo.buildiso(os.path.join(target, "bin-cdrom.iso"))
def mk_source_cdrom(rfs,
arch,
codename,
init_codename,
target,
log,
cdrom_size=CDROM_SIZE,
xml=None):
hostfs.mkdir_p('/var/cache/elbe/sources')
rfs.mkdir_p('/var/cache/elbe/sources')
repo = CdromSrcRepo(codename, init_codename,
os.path.join(target, "srcrepo"), log, cdrom_size)
cache = get_rpcaptcache(rfs, "aptcache.log", arch)
pkglist = cache.get_installed_pkgs()
forbiddenPackages = []
if xml != None and xml.has('target/pkg-list'):
for i in xml.node('target/pkg-list'):
try:
if i.tag == 'pkg' and i.et.attrib['on_src_cd'] == 'False':
forbiddenPackages.append(i.text('.').strip())
except KeyError:
pass
for pkg in pkglist:
# Do not include forbidden packages in src cdrom
if pkg.name in forbiddenPackages:
continue
try:
dsc = cache.download_source(pkg.name, '/var/cache/elbe/sources')
repo.includedsc(dsc, force=True)
except ValueError as ve:
log.printo("No sources for Package " + pkg.name + "-" +
pkg.installed_version)
except FetchError as fe:
log.printo("Source for Package " + pkg.name + "-" +
pkg.installed_version + " could not be downloaded")
repo.finalize()
pkglist = get_initvm_pkglist()
cache = Cache()
cache.open()
for pkg in pkglist:
# Do not include forbidden packages in src cdrom
if pkg.name in forbiddenPackages:
continue
try:
p = cache[pkg.name]
if pkg.name == 'elbe-bootstrap':
pkgver = p.versions[0]
else:
pkgver = p.installed
dsc = pkgver.fetch_source('/var/cache/elbe/sources',
ElbeAcquireProgress(cb=None),
unpack=False)
repo.includedsc(dsc)
except ValueError as ve:
log.printo("No sources for Package " + pkg.name + "-" +
str(pkg.installed_version))
except FetchError as fe:
log.printo("Source for Package " + pkg.name + "-" +
pkgver.version + " could not be downloaded")
repo.finalize()
return repo.buildiso(os.path.join(target, "src-cdrom.iso"))
def commit(self):
os.environ["DEBIAN_FRONTEND"]="noninteractive"
os.environ["DEBONF_NONINTERACTIVE_SEEN"]="true"
self.cache.commit( ElbeAcquireProgress(cb=self.co_cb),
ElbeInstallProgress(cb=self.co_cb) )
self.cache.open()