def mk_source_cdrom(rfs, arch, codename, init_codename, target,
cdrom_size=CDROM_SIZE, xml=None):
# pylint: disable=too-many-arguments
# pylint: disable=too-many-locals
# pylint: disable=too-many-branches
hostfs.mkdir_p('/var/cache/elbe/sources')
rfs.mkdir_p('/var/cache/elbe/sources')
if xml is not None:
mirror = xml.get_primary_mirror(rfs.fname("cdrom"))
else:
mirror = 'http://ftp.de.debian.org/debian'
repo = CdromSrcRepo(codename, init_codename,
os.path.join(target, "srcrepo"),
cdrom_size,
mirror)
cache = get_rpcaptcache(rfs, arch)
cache.update()
pkglist = cache.get_installed_pkgs()
forbiddenPackages = []
if xml is not None and xml.has('target/pkg-list'):
for i in xml.node('target/pkg-list'):
try:
if i.tag == 'pkg' and i.et.attrib['on_src_cd'] == 'False':
forbiddenPackages.append(i.text('.').strip())
except KeyError:
pass
for pkg in pkglist:
# Do not include forbidden packages in src cdrom
if pkg.name in forbiddenPackages:
continue
pkg_id = "%s-%s" % (pkg.name, pkg.installed_version)
try:
dsc = cache.download_source(pkg.name, '/var/cache/elbe/sources')
repo.includedsc(dsc, force=True)
except ValueError:
logging.error("No sources for package '%s'", pkg_id)
except FetchError:
logging.error("Source for package '%s' could not be downloaded", pkg_id)
# elbe fetch_initvm_pkgs has downloaded all sources to
# /var/cache/elbe/sources
# use walk_files to scan it, and add all dsc files.
#
# we can not just copy the source repo, like we do
# with the bin repo, because the src cdrom can be split
# into multiple cdroms
initvm_repo = Filesystem('/var/cache/elbe/sources')
for _ , dsc_real in initvm_repo.walk_files():
if not dsc_real.endswith('.dsc'):
continue
repo.include_init_dsc(dsc_real, 'initvm')
repo.finalize()
if xml is not None:
options = get_iso_options(xml)
for arch_vol in xml.all('src-cdrom/archive'):
volume_attr = arch_vol.et.get('volume')
if volume_attr == 'all':
volume_list = repo.volume_indexes
else:
volume_list = [int(v) for v in volume_attr.split(",")]
for volume_number in volume_list:
with archive_tmpfile(arch_vol.text(".")) as fp:
if volume_number in repo.volume_indexes:
do('tar xvfj "%s" -h -C "%s"' % (fp.name,
repo.get_volume_fs(volume_number).path))
else:
logging.warning("The src-cdrom archive's volume value "
"is not contained in the actual volumes")
else:
options = ""
return repo.buildiso(os.path.join(target, "src-cdrom.iso"), options=options)
def elbe_report(xml, buildenv, cache, targetfs):
# pylint: disable=too-many-arguments
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
rfs = buildenv.rfs
report.info(
"ELBE Report for Project %s\n\n"
"Report timestamp: %s\n"
"elbe: %s", xml.text("project/name"),
datetime.now().strftime("%Y%m%d-%H%M%S"), str(elbe_version))
slist = rfs.read_file('etc/apt/sources.list')
report.info("")
report.info("Apt Sources dump")
report.info("----------------")
report.info("")
report.info("%s", slist)
report.info("")
try:
prefs = rfs.read_file("etc/apt/preferences")
except IOError:
prefs = ""
report.info("")
report.info("Apt Preferences dump")
report.info("--------------------")
report.info("")
report.info("%s", prefs)
report.info("")
report.info("Installed Packages List")
report.info("-----------------------")
report.info("")
instpkgs = cache.get_installed_pkgs()
for p in instpkgs:
report.info("|%s|%s|%s", p.name, p.installed_version, p.origin)
index = cache.get_fileindex()
mt_index = targetfs.mtime_snap()
if xml.has("archive") and not xml.text("archive") is None:
with archive_tmpfile(xml.text("archive")) as fp:
do('tar xvfj "%s" -h -C "%s"' % (fp.name, targetfs.path))
mt_index_postarch = targetfs.mtime_snap()
else:
mt_index_postarch = mt_index
if xml.has("target/finetuning"):
do_finetuning(xml, buildenv, targetfs)
mt_index_post_fine = targetfs.mtime_snap()
else:
mt_index_post_fine = mt_index_postarch
report.info("")
report.info("File List")
report.info("---------")
report.info("")
tgt_pkg_list = set()
for fpath, _ in targetfs.walk_files():
if fpath in index:
pkg = index[fpath]
tgt_pkg_list.add(pkg)
else:
pkg = "postinst generated"
if fpath in mt_index_post_fine:
if fpath in mt_index_postarch:
if mt_index_post_fine[fpath] != mt_index_postarch[fpath]:
pkg = "modified finetuning"
elif fpath in mt_index:
if mt_index_postarch[fpath] != mt_index[fpath]:
pkg = "from archive"
# else leave pkg as is
else:
pkg = "added in archive"
else:
pkg = "added in finetuning"
# else leave pkg as is
report.info("|+%s+|%s", fpath, pkg)
report.info("")
report.info("Deleted Files")
report.info("-------------")
report.info("")
for fpath in list(mt_index.keys()):
if fpath not in mt_index_post_fine:
if fpath in index:
pkg = index[fpath]
else:
pkg = "postinst generated"
report.info("|+%s+|%s", fpath, pkg)
report.info("")
report.info("Target Package List")
report.info("-------------------")
report.info("")
instpkgs = cache.get_installed_pkgs()
pkgindex = {}
for p in instpkgs:
pkgindex[p.name] = p
if xml.has("target/pkgversionlist"):
targetfs.remove('etc/elbe_pkglist')
f = targetfs.open('etc/elbe_pkglist', 'w')
for pkg in tgt_pkg_list:
p = pkgindex[pkg]
report.info("|%s|%s|%s|%s", p.name, p.installed_version,
p.is_auto_installed, p.installed_md5)
if xml.has("target/pkgversionlist"):
f.write("%s %s %s\n" %
(p.name, p.installed_version, p.installed_md5))
if xml.has("target/pkgversionlist"):
f.close()
if not xml.has("archive") or xml.text("archive") is None:
return
validation.info("")
validation.info("Archive validation")
validation.info("------------------")
validation.info("")
errors = 0
for fpath in list(mt_index_postarch.keys()):
if fpath not in mt_index or \
mt_index_postarch[fpath] != mt_index[fpath]:
if fpath not in mt_index_post_fine:
validation.error("Archive file %s deleted in finetuning",
fpath)
errors += 1
elif mt_index_post_fine[fpath] != mt_index_postarch[fpath]:
validation.error("Archive file %s modified in finetuning",
fpath)
errors += 1
if errors == 0:
validation.info("No Errors found")
def mk_source_cdrom(components, codename,
init_codename, target,
cdrom_size=CDROM_SIZE, xml=None,
mirror='http://ftp.de.debian.org/debian'):
# pylint: disable=too-many-arguments
# pylint: disable=too-many-locals
# pylint: disable=too-many-branches
hostfs.mkdir_p('/var/cache/elbe/sources')
forbiddenPackages = []
if xml is not None and xml.has('target/pkg-list'):
for i in xml.node('target/pkg-list'):
try:
if i.tag == 'pkg' and i.et.attrib['on_src_cd'] == 'False':
forbiddenPackages.append(i.text('.').strip())
except KeyError:
pass
repos = {}
for component in components.keys():
rfs, cache, pkg_lst = components[component]
logging.info("Adding %s component", component)
rfs.mkdir_p("/var/cache/elbe/sources")
repo = CdromSrcRepo(codename, init_codename,
os.path.join(target, "srcrepo-%s" % component),
cdrom_size, mirror)
repos[component] = repo
for pkg, version in pkg_lst:
add_source_pkg(repo, component,
cache, pkg, version,
forbiddenPackages)
# elbe fetch_initvm_pkgs has downloaded all sources to
# /var/cache/elbe/sources
# use walk_files to scan it, and add all dsc files.
#
# we can not just copy the source repo, like we do
# with the bin repo, because the src cdrom can be split
# into multiple cdroms
initvm_repo = Filesystem('/var/cache/elbe/sources')
for _ , dsc_real in initvm_repo.walk_files():
if not dsc_real.endswith('.dsc'):
continue
repos["main"].include_init_dsc(dsc_real, "initvm")
for repo in repos.values():
repo.finalize()
if xml is not None:
options = get_iso_options(xml)
for arch_vol in xml.all('src-cdrom/archive'):
volume_attr = arch_vol.et.get('volume')
for repo in repos.values():
if volume_attr == 'all':
volume_list = repo.volume_indexes
else:
volume_list = [int(v) for v in volume_attr.split(",")]
for volume_number in volume_list:
with archive_tmpfile(arch_vol.text(".")) as fp:
if volume_number in repo.volume_indexes:
do('tar xvfj "%s" -h -C "%s"' % (fp.name,
repo.get_volume_fs(volume_number).path))
else:
logging.warning("The src-cdrom archive's volume value "
"is not contained in the actual volumes")
else:
options = ""
return [(repo.buildiso(os.path.join(target, "src-cdrom-%s.iso" % component),
options=options)) for component, repo in repos.items()]