def mk_source_cdrom(rfs,
arch,
codename,
init_codename,
target,
log,
cdrom_size=CDROM_SIZE):
hostfs.mkdir_p('/var/cache/elbe/sources')
rfs.mkdir_p('/var/cache/elbe/sources')
repo = CdromSrcRepo(codename, init_codename,
os.path.join(target, "srcrepo"), log, cdrom_size)
cache = get_rpcaptcache(rfs, "aptcache.log", arch)
pkglist = cache.get_installed_pkgs()
for pkg in pkglist:
try:
dsc = cache.download_source(pkg.name, '/var/cache/elbe/sources')
repo.includedsc(dsc)
except ValueError as ve:
log.printo("No sources for Package " + pkg.name + "-" +
pkg.installed_version)
except FetchError as fe:
log.printo("Source for Package " + pkg.name + "-" +
pkg.installed_version + " could not be downloaded")
pkglist = get_initvm_pkglist()
cache = Cache()
cache.open()
for pkg in pkglist:
try:
p = cache[pkg.name]
if pkg.name == 'elbe-bootstrap':
pkgver = p.versions[0]
else:
pkgver = p.installed
dsc = pkgver.fetch_source('/var/cache/elbe/sources',
ElbeAcquireProgress(cb=None),
unpack=False)
repo.includedsc(dsc)
except ValueError as ve:
log.printo("No sources for Package " + pkg.name + "-" +
str(pkg.installed_version))
except FetchError as fe:
log.printo("Source for Package " + pkg.name + "-" +
pkgver.version + " could not be downloaded")
return repo.buildiso(os.path.join(target, "src-cdrom.iso"))
def run_command(argv):
# TODO - Set threshold and remove pylint directives
#
# We might want to make the threshold higher for certain
# files/directories or just globaly.
# pylint: disable=too-many-locals
# pylint: disable=too-many-branches
# pylint: disable=too-many-statements
oparser = OptionParser(
usage="usage: %prog fetch_initvm_pkgs [options] <xmlfile>")
oparser.add_option("-b",
"--binrepo",
dest="binrepo",
default="/var/cache/elbe/initvm-bin-repo",
help="directory where the bin repo should reside")
oparser.add_option("-s",
"--srcrepo",
dest="srcrepo",
default="/var/cache/elbe/initvm-src-repo",
help="directory where the src repo should reside")
oparser.add_option("--skip-validation",
action="store_true",
dest="skip_validation",
default=False,
help="Skip xml schema validation")
oparser.add_option("--cdrom-mount-path",
dest="cdrom_path",
help="path where cdrom is mounted")
oparser.add_option("--cdrom-device",
dest="cdrom_device",
help="cdrom device, in case it has to be mounted")
oparser.add_option("--apt-archive",
dest="archive",
default="/var/cache/elbe/binaries/main",
help="path where binary packages are downloaded to.")
oparser.add_option("--src-archive",
dest="srcarchive",
default="/var/cache/elbe/sources",
help="path where src packages are downloaded to.")
oparser.add_option("--skip-build-sources",
action="store_false",
dest="build_sources",
default=True,
help="Skip downloading Source Packages")
oparser.add_option("--skip-build-bin",
action="store_false",
dest="build_bin",
default=True,
help="Skip downloading binary packages")
(opt, args) = oparser.parse_args(argv)
if len(args) != 1:
print("wrong number of arguments")
oparser.print_help()
sys.exit(20)
try:
xml = ElbeXML(args[0], skip_validate=opt.skip_validation)
except ValidationError as e:
print(str(e))
print("xml validation failed. Bailing out")
sys.exit(20)
with elbe_logging({"streams": sys.stdout}):
if opt.cdrom_path:
if opt.cdrom_device:
do('mount "%s" "%s"' % (opt.cdrom_device, opt.cdrom_path))
# a cdrom build is identified by the cdrom option
# the xml file that is copied into the initvm
# by the initrd does not have the cdrom tags setup.
mirror = "file://%s" % opt.cdrom_path
else:
mirror = xml.get_initvm_primary_mirror(opt.cdrom_path)
init_codename = xml.get_initvm_codename()
# Binary Repo
#
repo = CdromInitRepo(init_codename, opt.binrepo, mirror)
hostfs.mkdir_p(opt.archive)
if opt.build_bin:
pkglist = get_initvm_pkglist()
cache = Cache()
cache.open()
for pkg in pkglist:
pkg_id = "%s-%s" % (pkg.name, pkg.installed_version)
try:
p = cache[pkg.name]
pkgver = p.installed
deb = fetch_binary(pkgver, opt.archive,
ElbeAcquireProgress(cb=None))
repo.includedeb(deb, 'main')
except ValueError:
logging.exception('No package "%s"', pkg_id)
except FetchError:
logging.exception(
'Package "%s-%s" could not be downloaded', pkg.name,
pkgver.version)
except TypeError:
logging.exception('Package "%s" missing name or version',
pkg_id)
repo.finalize()
# Source Repo
#
repo = CdromSrcRepo(init_codename, init_codename, opt.srcrepo, 0,
mirror)
hostfs.mkdir_p(opt.srcarchive)
# a cdrom build does not have sources
# skip adding packages to the source repo
#
# FIXME: we need a way to add source cdroms later on
if opt.cdrom_path:
opt.build_sources = False
if opt.build_sources:
for pkg in pkglist:
pkg_id = "%s-%s" % (pkg.name, pkg.installed_version)
try:
p = cache[pkg.name]
pkgver = p.installed
dsc = pkgver.fetch_source(opt.srcarchive,
ElbeAcquireProgress(cb=None),
unpack=False)
repo.include_init_dsc(dsc, 'initvm')
except ValueError:
logging.exception('No package "%s"', pkg_id)
except FetchError:
logging.exception(
'Package "%s-%s" could not be downloaded', pkg.name,
pkgver.version)
except TypeError:
logging.exception('Package "%s" missing name or version',
pkg_id)
repo.finalize()
if opt.cdrom_device:
do('umount "%s"' % opt.cdrom_device)
def mk_source_cdrom(rfs, arch, codename, init_codename, target,
cdrom_size=CDROM_SIZE, xml=None):
# pylint: disable=too-many-arguments
# pylint: disable=too-many-locals
# pylint: disable=too-many-branches
hostfs.mkdir_p('/var/cache/elbe/sources')
rfs.mkdir_p('/var/cache/elbe/sources')
if xml is not None:
mirror = xml.get_primary_mirror(rfs.fname("cdrom"))
else:
mirror = 'http://ftp.de.debian.org/debian'
repo = CdromSrcRepo(codename, init_codename,
os.path.join(target, "srcrepo"),
cdrom_size,
mirror)
cache = get_rpcaptcache(rfs, arch)
cache.update()
pkglist = cache.get_installed_pkgs()
forbiddenPackages = []
if xml is not None and xml.has('target/pkg-list'):
for i in xml.node('target/pkg-list'):
try:
if i.tag == 'pkg' and i.et.attrib['on_src_cd'] == 'False':
forbiddenPackages.append(i.text('.').strip())
except KeyError:
pass
for pkg in pkglist:
# Do not include forbidden packages in src cdrom
if pkg.name in forbiddenPackages:
continue
pkg_id = "%s-%s" % (pkg.name, pkg.installed_version)
try:
dsc = cache.download_source(pkg.name, '/var/cache/elbe/sources')
repo.includedsc(dsc, force=True)
except ValueError:
logging.error("No sources for package '%s'", pkg_id)
except FetchError:
logging.error("Source for package '%s' could not be downloaded", pkg_id)
# elbe fetch_initvm_pkgs has downloaded all sources to
# /var/cache/elbe/sources
# use walk_files to scan it, and add all dsc files.
#
# we can not just copy the source repo, like we do
# with the bin repo, because the src cdrom can be split
# into multiple cdroms
initvm_repo = Filesystem('/var/cache/elbe/sources')
for _ , dsc_real in initvm_repo.walk_files():
if not dsc_real.endswith('.dsc'):
continue
repo.include_init_dsc(dsc_real, 'initvm')
repo.finalize()
if xml is not None:
options = get_iso_options(xml)
for arch_vol in xml.all('src-cdrom/archive'):
volume_attr = arch_vol.et.get('volume')
if volume_attr == 'all':
volume_list = repo.volume_indexes
else:
volume_list = [int(v) for v in volume_attr.split(",")]
for volume_number in volume_list:
with archive_tmpfile(arch_vol.text(".")) as fp:
if volume_number in repo.volume_indexes:
do('tar xvfj "%s" -h -C "%s"' % (fp.name,
repo.get_volume_fs(volume_number).path))
else:
logging.warning("The src-cdrom archive's volume value "
"is not contained in the actual volumes")
else:
options = ""
return repo.buildiso(os.path.join(target, "src-cdrom.iso"), options=options)
def mk_source_cdrom(components, codename,
init_codename, target,
cdrom_size=CDROM_SIZE, xml=None,
mirror='http://ftp.de.debian.org/debian'):
# pylint: disable=too-many-arguments
# pylint: disable=too-many-locals
# pylint: disable=too-many-branches
hostfs.mkdir_p('/var/cache/elbe/sources')
forbiddenPackages = []
if xml is not None and xml.has('target/pkg-list'):
for i in xml.node('target/pkg-list'):
try:
if i.tag == 'pkg' and i.et.attrib['on_src_cd'] == 'False':
forbiddenPackages.append(i.text('.').strip())
except KeyError:
pass
repos = {}
for component in components.keys():
rfs, cache, pkg_lst = components[component]
logging.info("Adding %s component", component)
rfs.mkdir_p("/var/cache/elbe/sources")
repo = CdromSrcRepo(codename, init_codename,
os.path.join(target, "srcrepo-%s" % component),
cdrom_size, mirror)
repos[component] = repo
for pkg, version in pkg_lst:
add_source_pkg(repo, component,
cache, pkg, version,
forbiddenPackages)
# elbe fetch_initvm_pkgs has downloaded all sources to
# /var/cache/elbe/sources
# use walk_files to scan it, and add all dsc files.
#
# we can not just copy the source repo, like we do
# with the bin repo, because the src cdrom can be split
# into multiple cdroms
initvm_repo = Filesystem('/var/cache/elbe/sources')
for _ , dsc_real in initvm_repo.walk_files():
if not dsc_real.endswith('.dsc'):
continue
repos["main"].include_init_dsc(dsc_real, "initvm")
for repo in repos.values():
repo.finalize()
if xml is not None:
options = get_iso_options(xml)
for arch_vol in xml.all('src-cdrom/archive'):
volume_attr = arch_vol.et.get('volume')
for repo in repos.values():
if volume_attr == 'all':
volume_list = repo.volume_indexes
else:
volume_list = [int(v) for v in volume_attr.split(",")]
for volume_number in volume_list:
with archive_tmpfile(arch_vol.text(".")) as fp:
if volume_number in repo.volume_indexes:
do('tar xvfj "%s" -h -C "%s"' % (fp.name,
repo.get_volume_fs(volume_number).path))
else:
logging.warning("The src-cdrom archive's volume value "
"is not contained in the actual volumes")
else:
options = ""
return [(repo.buildiso(os.path.join(target, "src-cdrom-%s.iso" % component),
options=options)) for component, repo in repos.items()]
def mk_source_cdrom(
rfs,
arch,
codename,
init_codename,
target,
log,
cdrom_size=CDROM_SIZE,
xml=None):
# pylint: disable=too-many-arguments
# pylint: disable=too-many-locals
# pylint: disable=too-many-branches
hostfs.mkdir_p('/var/cache/elbe/sources')
rfs.mkdir_p('/var/cache/elbe/sources')
if xml is not None:
mirror = xml.get_primary_mirror(rfs.fname("cdrom"))
else:
mirror = 'http://ftp.de.debian.org/debian'
repo = CdromSrcRepo(codename, init_codename,
os.path.join(target, "srcrepo"),
log,
cdrom_size,
mirror)
cache = get_rpcaptcache(rfs, "aptcache.log", arch)
cache.update()
pkglist = cache.get_installed_pkgs()
forbiddenPackages = []
if xml is not None and xml.has('target/pkg-list'):
for i in xml.node('target/pkg-list'):
try:
if i.tag == 'pkg' and i.et.attrib['on_src_cd'] == 'False':
forbiddenPackages.append(i.text('.').strip())
except KeyError:
pass
for pkg in pkglist:
# Do not include forbidden packages in src cdrom
if pkg.name in forbiddenPackages:
continue
try:
dsc = cache.download_source(pkg.name, '/var/cache/elbe/sources')
repo.includedsc(dsc, force=True)
except ValueError:
log.printo(
"No sources for Package " +
pkg.name +
"-" +
pkg.installed_version)
except FetchError:
log.printo(
"Source for Package " +
pkg.name +
"-" +
pkg.installed_version +
" could not be downloaded")
# elbe fetch_initvm_pkgs has downloaded all sources to
# /var/cache/elbe/sources
# use walk_files to scan it, and add all dsc files.
#
# we can not just copy the source repo, like we do
# with the bin repo, because the src cdrom can be split
# into multiple cdroms
initvm_repo = Filesystem('/var/cache/elbe/sources')
for _ , dsc_real in initvm_repo.walk_files():
if not dsc_real.endswith('.dsc'):
continue
repo.include_init_dsc(dsc_real, 'initvm')
repo.finalize()
return repo.buildiso(os.path.join(target, "src-cdrom.iso"))
def mk_source_cdrom(rfs,
arch,
codename,
init_codename,
target,
log,
cdrom_size=CDROM_SIZE,
xml=None):
hostfs.mkdir_p('/var/cache/elbe/sources')
rfs.mkdir_p('/var/cache/elbe/sources')
repo = CdromSrcRepo(codename, init_codename,
os.path.join(target, "srcrepo"), log, cdrom_size)
cache = get_rpcaptcache(rfs, "aptcache.log", arch)
pkglist = cache.get_installed_pkgs()
forbiddenPackages = []
if xml != None and xml.has('target/pkg-list'):
for i in xml.node('target/pkg-list'):
try:
if i.tag == 'pkg' and i.et.attrib['on_src_cd'] == 'False':
forbiddenPackages.append(i.text('.').strip())
except KeyError:
pass
for pkg in pkglist:
# Do not include forbidden packages in src cdrom
if pkg.name in forbiddenPackages:
continue
try:
dsc = cache.download_source(pkg.name, '/var/cache/elbe/sources')
repo.includedsc(dsc, force=True)
except ValueError as ve:
log.printo("No sources for Package " + pkg.name + "-" +
pkg.installed_version)
except FetchError as fe:
log.printo("Source for Package " + pkg.name + "-" +
pkg.installed_version + " could not be downloaded")
repo.finalize()
pkglist = get_initvm_pkglist()
cache = Cache()
cache.open()
for pkg in pkglist:
# Do not include forbidden packages in src cdrom
if pkg.name in forbiddenPackages:
continue
try:
p = cache[pkg.name]
if pkg.name == 'elbe-bootstrap':
pkgver = p.versions[0]
else:
pkgver = p.installed
dsc = pkgver.fetch_source('/var/cache/elbe/sources',
ElbeAcquireProgress(cb=None),
unpack=False)
repo.includedsc(dsc)
except ValueError as ve:
log.printo("No sources for Package " + pkg.name + "-" +
str(pkg.installed_version))
except FetchError as fe:
log.printo("Source for Package " + pkg.name + "-" +
pkgver.version + " could not be downloaded")
repo.finalize()
return repo.buildiso(os.path.join(target, "src-cdrom.iso"))