def test_displaymode(self):
"""Test a kickstart with displaymode set"""
opts = DataHolder(no_virt=True,
make_fsimage=False,
make_pxe_live=False)
ks_version = makeVersion()
ks = KickstartParser(ks_version,
errorsAreFatal=False,
missingIncludeIsFatal=False)
ks.readKickstartFromString(
"url --url=http://dl.fedoraproject.com\n"
"network --bootproto=dhcp --activate\n"
"repo --name=other --baseurl=http://dl.fedoraproject.com\n"
"part / --size=4096\n"
"shutdown\n"
"graphical\n")
errors = check_kickstart(ks, opts)
self.assertTrue("must not set a display mode" in errors[0])
def disk_size_reqpart_boot_test(self):
"""Test calculating the disk size with reqpart --add-boot and a / partition"""
opts = DataHolder(no_virt=True,
make_fsimage=False,
make_iso=False,
make_pxe_live=False)
ks_version = makeVersion()
ks = KickstartParser(ks_version,
errorsAreFatal=False,
missingIncludeIsFatal=False)
ks.readKickstartFromString(
"url --url=http://dl.fedoraproject.com\n"
"network --bootproto=dhcp --activate\n"
"repo --name=other --baseurl=http://dl.fedoraproject.com\n"
"part / --size=4096\n"
"reqpart --add-boot\n"
"shutdown\n")
self.assertEqual(calculate_disk_size(opts, ks), 5622)
def load_or_default(system_ks_path):
"""load system ks or default ks"""
ksparser = KickstartParser(makeVersion())
try:
ksparser.readKickstart(system_ks_path)
except (KickstartError, IOError):
log_message(
"Can't read system kickstart at {0}".format(system_ks_path))
try:
ksparser.readKickstart(settings.KS_TEMPLATE)
except AttributeError:
log_message(
"There is no KS_TEMPLATE_POSTSCRIPT specified in settings.py"
)
except IOError:
log_message("Can't read kickstart template {0}".format(
settings.KS_TEMPLATE))
return None
return ksparser
def load_or_default(system_ks_path, ks_template):
""" load system ks or default ks """
ksparser = KickstartParser(makeVersion())
try:
ksparser.readKickstart(system_ks_path)
except (KickstartError, IOError):
log_message("Cannot read the system Kickstart at %s." %
system_ks_path)
try:
ksparser.readKickstart(ks_template)
except AttributeError:
log_message(
"There is no KS_POSTSCRIPT_TEMPLATE specified in settings.py.",
level=logging.DEBUG)
except IOError:
log_message("Cannot read the Kickstart template %s." %
ks_template)
return None
return ksparser
def runTest(self):
for version, command_map in control.commandMap.items():
handler = makeVersion(version)
parser = KickstartParser(handler)
for command_name, command_class in command_map.items():
if not issubclass(command_class, DeprecatedCommand):
continue
if issubclass(command_class, RemovedCommand):
continue
with warnings.catch_warnings(record=True):
# The deprecated commands should be ignored with
# a warning when they are parsed. Make sure that
# they will not cause any errors.
with self.assertWarns(KickstartDeprecationWarning) as cm:
parser.readKickstartFromString(command_name)
# Check the warning message.
expected = " {} command has been deprecated ".format(
command_name)
self.assertIn(expected, str(cm.warning))
def bootloader_append(line, kernel_append):
""" Insert the kernel_append string into the --append argument
:param line: The bootloader ... line
:type line: str
:param kernel_append: The arguments to append to the --append section
:type kernel_append: str
Using pykickstart to process the line is the best way to make sure it
is parsed correctly, and re-assembled for inclusion into the final kickstart
"""
ks_version = makeVersion()
ks = KickstartParser(ks_version,
errorsAreFatal=False,
missingIncludeIsFatal=False)
ks.readKickstartFromString(line)
if ks.handler.bootloader.appendLine:
ks.handler.bootloader.appendLine += " %s" % kernel_append
else:
ks.handler.bootloader.appendLine = kernel_append
# Converting back to a string includes a comment, return just the bootloader line
return str(ks.handler.bootloader).splitlines()[-1]
def start_build(cfg,
dnflock,
gitlock,
branch,
recipe_name,
compose_type,
test_mode=0):
""" Start the build
:param cfg: Configuration object
:type cfg: ComposerConfig
:param dnflock: Lock and YumBase for depsolving
:type dnflock: YumLock
:param recipe: The recipe to build
:type recipe: str
:param compose_type: The type of output to create from the recipe
:type compose_type: str
:returns: Unique ID for the build that can be used to track its status
:rtype: str
"""
share_dir = cfg.get("composer", "share_dir")
lib_dir = cfg.get("composer", "lib_dir")
# Make sure compose_type is valid
if compose_type not in compose_types(share_dir):
raise RuntimeError("Invalid compose type (%s), must be one of %s" %
(compose_type, compose_types(share_dir)))
# Some image types (live-iso) need extra packages for composer to execute the output template
with dnflock.lock:
extra_pkgs = get_extra_pkgs(dnflock.dbo, share_dir, compose_type)
log.debug("Extra packages needed for %s: %s", compose_type, extra_pkgs)
with gitlock.lock:
(commit_id, recipe) = read_recipe_and_id(gitlock.repo, branch,
recipe_name)
# Combine modules and packages and depsolve the list
module_nver = recipe.module_nver
package_nver = recipe.package_nver
package_nver.extend([(name, '*') for name in extra_pkgs])
projects = sorted(set(module_nver + package_nver),
key=lambda p: p[0].lower())
deps = []
log.info("depsolving %s", recipe["name"])
try:
# This can possibly update repodata and reset the YumBase object.
with dnflock.lock_check:
(installed_size,
deps) = projects_depsolve_with_size(dnflock.dbo,
projects,
recipe.group_names,
with_core=False)
except ProjectsError as e:
log.error("start_build depsolve: %s", str(e))
raise RuntimeError("Problem depsolving %s: %s" %
(recipe["name"], str(e)))
# Read the kickstart template for this type
ks_template_path = joinpaths(share_dir, "composer", compose_type) + ".ks"
ks_template = open(ks_template_path, "r").read()
# How much space will the packages in the default template take?
ks_version = makeVersion()
ks = KickstartParser(ks_version,
errorsAreFatal=False,
missingIncludeIsFatal=False)
ks.readKickstartFromString(ks_template + "\n%end\n")
pkgs = [(name, "*") for name in ks.handler.packages.packageList]
grps = [grp.name for grp in ks.handler.packages.groupList]
try:
with dnflock.lock:
(template_size, _) = projects_depsolve_with_size(
dnflock.dbo,
pkgs,
grps,
with_core=not ks.handler.packages.nocore)
except ProjectsError as e:
log.error("start_build depsolve: %s", str(e))
raise RuntimeError("Problem depsolving %s: %s" %
(recipe["name"], str(e)))
log.debug("installed_size = %d, template_size=%d", installed_size,
template_size)
# Minimum LMC disk size is 1GiB, and anaconda bumps the estimated size up by 10% (which doesn't always work).
installed_size = int((installed_size + template_size)) * 1.2
log.debug("/ partition size = %d", installed_size)
# Create the results directory
build_id = str(uuid4())
results_dir = joinpaths(lib_dir, "results", build_id)
os.makedirs(results_dir)
# Write the recipe commit hash
commit_path = joinpaths(results_dir, "COMMIT")
with open(commit_path, "w") as f:
f.write(commit_id)
# Write the original recipe
recipe_path = joinpaths(results_dir, "blueprint.toml")
with open(recipe_path, "w") as f:
f.write(recipe.toml())
# Write the frozen recipe
frozen_recipe = recipe.freeze(deps)
recipe_path = joinpaths(results_dir, "frozen.toml")
with open(recipe_path, "w") as f:
f.write(frozen_recipe.toml())
# Write out the dependencies to the results dir
deps_path = joinpaths(results_dir, "deps.toml")
with open(deps_path, "w") as f:
f.write(toml.dumps({"packages": deps}))
# Save a copy of the original kickstart
shutil.copy(ks_template_path, results_dir)
with dnflock.lock:
repos = list(dnflock.dbo.repos.iter_enabled())
if not repos:
raise RuntimeError("No enabled repos, canceling build.")
# Create the git rpms, if any, and return the path to the repo under results_dir
gitrpm_repo = create_gitrpm_repo(results_dir, recipe)
# Create the final kickstart with repos and package list
ks_path = joinpaths(results_dir, "final-kickstart.ks")
with open(ks_path, "w") as f:
ks_url = repo_to_ks(repos[0], "url")
log.debug("url = %s", ks_url)
f.write('url %s\n' % ks_url)
for idx, r in enumerate(repos[1:]):
ks_repo = repo_to_ks(r, "baseurl")
log.debug("repo composer-%s = %s", idx, ks_repo)
f.write('repo --name="composer-%s" %s\n' % (idx, ks_repo))
if gitrpm_repo:
log.debug("repo gitrpms = %s", gitrpm_repo)
f.write('repo --name="gitrpms" --baseurl="file://%s"\n' %
gitrpm_repo)
# Setup the disk for booting
# TODO Add GPT and UEFI boot support
f.write('clearpart --all --initlabel\n')
# Write the root partition and it's size in MB (rounded up)
f.write('part / --size=%d\n' % ceil(installed_size / 1024**2))
# Some customizations modify the template before writing it
f.write(customize_ks_template(ks_template, recipe))
for d in deps:
f.write(dep_nevra(d) + "\n")
# Include the rpms from the gitrpm repo directory
if gitrpm_repo:
for rpm in glob(os.path.join(gitrpm_repo, "*.rpm")):
f.write(os.path.basename(rpm)[:-4] + "\n")
f.write("%end\n")
# Other customizations can be appended to the kickstart
add_customizations(f, recipe)
# Setup the config to pass to novirt_install
log_dir = joinpaths(results_dir, "logs/")
cfg_args = compose_args(compose_type)
# Get the title, project, and release version from the host
if not os.path.exists("/etc/os-release"):
log.error(
"/etc/os-release is missing, cannot determine product or release version"
)
os_release = flatconfig("/etc/os-release")
log.debug("os_release = %s", dict(os_release.items()))
cfg_args["title"] = os_release.get("PRETTY_NAME", "")
cfg_args["project"] = os_release.get("NAME", "")
cfg_args["releasever"] = os_release.get("VERSION_ID", "")
cfg_args["volid"] = ""
cfg_args["extra_boot_args"] = get_kernel_append(recipe)
cfg_args.update({
"compression": "xz",
"compress_args": [],
"ks": [ks_path],
"logfile": log_dir,
"timeout": 60, # 60 minute timeout
})
with open(joinpaths(results_dir, "config.toml"), "w") as f:
f.write(toml.dumps(cfg_args))
# Set the initial status
open(joinpaths(results_dir, "STATUS"), "w").write("WAITING")
# Set the test mode, if requested
if test_mode > 0:
open(joinpaths(results_dir, "TEST"), "w").write("%s" % test_mode)
write_timestamp(results_dir, TS_CREATED)
log.info("Adding %s (%s %s) to compose queue", build_id, recipe["name"],
compose_type)
os.symlink(results_dir, joinpaths(lib_dir, "queue/new/", build_id))
return build_id
def get_parser(self):
"""Command sequence tests need a fresh parser
for each test"""
handler = makeVersion(self.version)
return KickstartParser(handler)
def main(argv):
op = argparse.ArgumentParser(usage="%(prog)s [options] ksfile", add_help=False)
op.add_argument("ksfile", nargs="?",
help=_("filename or URL to read from"))
op.add_argument("-e", "--firsterror", dest="firsterror", action="store_true",
default=False, help=_("halt after the first error or warning"))
op.add_argument("-i", "--followincludes", dest="followincludes",
action="store_true", default=False,
help=_("parse include files when %%include is seen"))
op.add_argument("-l", "--listversions", dest="listversions", action="store_true",
default=False,
help=_("list the available versions of kickstart syntax"))
op.add_argument("-v", "--version", dest="version", default=DEVEL,
help=_("version of kickstart syntax to validate against"))
op.add_argument("-h", "--help", dest="help", action="store_true", default=False,
help=_("show this help message and exit"))
opts = op.parse_args(argv)
# parse --help manually b/c we don't want to sys.exit before the
# tests have finished
if opts.help:
return (0, op.format_help().split("\n"))
if opts.listversions:
versions = []
for key in sorted(versionMap.keys()):
versions.append(key)
return (0, versions)
if not opts.ksfile:
return (1, op.format_usage().split("\n"))
destdir = tempfile.mkdtemp("", "ksvalidator-tmp-", "/tmp")
try:
f = load_to_file(opts.ksfile, "%s/ks.cfg" % destdir)
except KickstartError as e:
return (cleanup(destdir),
[_("Error reading %(filename)s:\n%(version)s") % {"filename": opts.ksfile, "version": e}])
try:
handler = makeVersion(opts.version)
except KickstartVersionError:
return (cleanup(destdir),
[_("The version %s is not supported by pykickstart") % opts.version])
ksparser = KickstartParser(handler, followIncludes=opts.followincludes,
errorsAreFatal=opts.firsterror)
# turn kickstart parse warnings into errors
warnings.filterwarnings(action="error", category=KickstartParseWarning)
processedFile = None
try:
processedFile = preprocessKickstart(f)
ksparser.readKickstart(processedFile)
return (cleanup(destdir, processedFile, exitval=ksparser.errorsCount), [])
except KickstartDeprecationWarning as err:
return (cleanup(destdir, processedFile),
[_("File uses a deprecated option or command.\n%s") % err])
except KickstartParseError as err:
return (cleanup(destdir, processedFile), [str(err)])
except KickstartError:
return (cleanup(destdir, processedFile),
[_("General kickstart error in input file")])
except Exception as e:
return (cleanup(destdir, processedFile),
[_("General error in input file: %s") % e])
def run_creator(opts, cancel_func=None):
"""Run the image creator process
:param opts: Commandline options to control the process
:type opts: Either a DataHolder or ArgumentParser
:param cancel_func: Function that returns True to cancel build
:type cancel_func: function
:returns: The result directory and the disk image path.
:rtype: Tuple of str
This function takes the opts arguments and creates the selected output image.
See the cmdline --help for livemedia-creator for the possible options
(Yes, this is not ideal, but we can fix that later)
"""
result_dir = None
# Parse the kickstart
if opts.ks:
ks_version = makeVersion()
ks = KickstartParser(ks_version,
errorsAreFatal=False,
missingIncludeIsFatal=False)
ks.readKickstart(opts.ks[0])
# live iso usually needs dracut-live so warn the user if it is missing
if opts.ks and opts.make_iso:
if "dracut-live" not in ks.handler.packages.packageList:
log.error("dracut-live package is missing from the kickstart.")
raise RuntimeError(
"dracut-live package is missing from the kickstart.")
# Make the disk or filesystem image
if not opts.disk_image and not opts.fs_image:
if not opts.ks:
raise RuntimeError("Image creation requires a kickstart file")
# Check the kickstart for problems
errors = check_kickstart(ks, opts)
if errors:
list(log.error(e) for e in errors)
raise RuntimeError("\n".join(errors))
# Make the image. Output of this is either a partitioned disk image or a fsimage
try:
disk_img = make_image(opts, ks, cancel_func=cancel_func)
except InstallError as e:
log.error("ERROR: Image creation failed: %s", e)
raise RuntimeError("Image creation failed: %s" % e)
if opts.image_only:
return (result_dir, disk_img)
if opts.make_iso:
work_dir = tempfile.mkdtemp(prefix="lmc-work-")
log.info("working dir is %s", work_dir)
if (opts.fs_image or opts.no_virt) and not opts.disk_image:
# Create iso from a filesystem image
disk_img = opts.fs_image or disk_img
if not make_squashfs(opts, disk_img, work_dir):
log.error("squashfs.img creation failed")
raise RuntimeError("squashfs.img creation failed")
if cancel_func and cancel_func():
raise RuntimeError("ISO creation canceled")
with Mount(disk_img, opts="loop") as mount_dir:
result_dir = make_livecd(opts, mount_dir, work_dir)
else:
# Create iso from a partitioned disk image
disk_img = opts.disk_image or disk_img
with PartitionMount(disk_img) as img_mount:
if img_mount and img_mount.mount_dir:
make_runtime(opts, img_mount.mount_dir, work_dir,
calculate_disk_size(opts, ks) / 1024.0)
result_dir = make_livecd(opts, img_mount.mount_dir,
work_dir)
# --iso-only removes the extra build artifacts, keeping only the boot.iso
if opts.iso_only and result_dir:
boot_iso = joinpaths(result_dir, "images/boot.iso")
if not os.path.exists(boot_iso):
log.error("%s is missing, skipping --iso-only.", boot_iso)
else:
iso_dir = tempfile.mkdtemp(prefix="lmc-result-")
dest_file = joinpaths(iso_dir, opts.iso_name or "boot.iso")
shutil.move(boot_iso, dest_file)
shutil.rmtree(result_dir)
result_dir = iso_dir
# cleanup the mess
# cleanup work_dir?
if disk_img and not (opts.keep_image or opts.disk_image
or opts.fs_image):
os.unlink(disk_img)
log.info("Disk image erased")
disk_img = None
elif opts.make_appliance:
if not opts.ks:
networks = []
else:
networks = ks.handler.network.network
make_appliance(opts.disk_image or disk_img, opts.app_name,
opts.app_template, opts.app_file, networks, opts.ram,
opts.vcpus or 1, opts.arch, opts.title, opts.project,
opts.releasever)
elif opts.make_pxe_live:
work_dir = tempfile.mkdtemp(prefix="lmc-work-")
log.info("working dir is %s", work_dir)
disk_img = opts.fs_image or opts.disk_image or disk_img
log.debug("disk image is %s", disk_img)
result_dir = make_live_images(opts, work_dir, disk_img)
if result_dir is None:
log.error("Creating PXE live image failed.")
raise RuntimeError("Creating PXE live image failed.")
if opts.result_dir != opts.tmp and result_dir:
copytree(result_dir, opts.result_dir, preserve=False)
shutil.rmtree(result_dir)
result_dir = None
return (result_dir, disk_img)
def run_creator(opts, callback_func=None):
"""Run the image creator process
:param opts: Commandline options to control the process
:type opts: Either a DataHolder or ArgumentParser
:returns: The result directory and the disk image path.
:rtype: Tuple of str
This function takes the opts arguments and creates the selected output image.
See the cmdline --help for livemedia-creator for the possible options
(Yes, this is not ideal, but we can fix that later)
"""
result_dir = None
# Parse the kickstart
if opts.ks:
ks_version = makeVersion()
ks = KickstartParser(ks_version, errorsAreFatal=False, missingIncludeIsFatal=False)
ks.readKickstart(opts.ks[0])
# live iso usually needs dracut-live so warn the user if it is missing
if opts.ks and opts.make_iso:
if "dracut-live" not in ks.handler.packages.packageList:
log.error("dracut-live package is missing from the kickstart.")
raise RuntimeError("dracut-live package is missing from the kickstart.")
# Make the disk or filesystem image
if not opts.disk_image and not opts.fs_image:
if not opts.ks:
raise RuntimeError("Image creation requires a kickstart file")
errors = []
if opts.no_virt and ks.handler.method.method not in ("url", "nfs") \
and not ks.handler.ostreesetup.seen:
errors.append("Only url, nfs and ostreesetup install methods are currently supported."
"Please fix your kickstart file." )
if ks.handler.method.method in ("url", "nfs") and not ks.handler.network.seen:
errors.append("The kickstart must activate networking if "
"the url or nfs install method is used.")
if ks.handler.displaymode.displayMode is not None:
errors.append("The kickstart must not set a display mode (text, cmdline, "
"graphical), this will interfere with livemedia-creator.")
if opts.make_fsimage or (opts.make_pxe_live and opts.no_virt):
# Make sure the kickstart isn't using autopart and only has a / mountpoint
part_ok = not any(p for p in ks.handler.partition.partitions
if p.mountpoint not in ["/", "swap"])
if not part_ok or ks.handler.autopart.seen:
errors.append("Filesystem images must use a single / part, not autopart or "
"multiple partitions. swap is allowed but not used.")
if not opts.no_virt and ks.handler.reboot.action != KS_SHUTDOWN:
errors.append("The kickstart must include shutdown when using virt installation.")
if errors:
list(log.error(e) for e in errors)
raise RuntimeError("\n".join(errors))
# Make the image. Output of this is either a partitioned disk image or a fsimage
try:
disk_img = make_image(opts, ks)
except InstallError as e:
log.error("ERROR: Image creation failed: %s", e)
raise RuntimeError("Image creation failed: %s" % e)
if opts.image_only:
return (result_dir, disk_img)
if opts.make_iso:
work_dir = tempfile.mkdtemp(prefix="lmc-work-")
log.info("working dir is %s", work_dir)
if (opts.fs_image or opts.no_virt) and not opts.disk_image:
# Create iso from a filesystem image
disk_img = opts.fs_image or disk_img
if not make_squashfs(opts, disk_img, work_dir):
log.error("squashfs.img creation failed")
raise RuntimeError("squashfs.img creation failed")
with Mount(disk_img, opts="loop") as mount_dir:
result_dir = make_livecd(opts, mount_dir, work_dir)
else:
# Create iso from a partitioned disk image
disk_img = opts.disk_image or disk_img
with PartitionMount(disk_img) as img_mount:
if img_mount and img_mount.mount_dir:
make_runtime(opts, img_mount.mount_dir, work_dir, calculate_disk_size(opts, ks)/1024.0)
result_dir = make_livecd(opts, img_mount.mount_dir, work_dir)
# --iso-only removes the extra build artifacts, keeping only the boot.iso
if opts.iso_only and result_dir:
boot_iso = joinpaths(result_dir, "images/boot.iso")
if not os.path.exists(boot_iso):
log.error("%s is missing, skipping --iso-only.", boot_iso)
else:
iso_dir = tempfile.mkdtemp(prefix="lmc-result-")
dest_file = joinpaths(iso_dir, opts.iso_name or "boot.iso")
shutil.move(boot_iso, dest_file)
shutil.rmtree(result_dir)
result_dir = iso_dir
# cleanup the mess
# cleanup work_dir?
if disk_img and not (opts.keep_image or opts.disk_image or opts.fs_image):
os.unlink(disk_img)
log.info("Disk image erased")
disk_img = None
elif opts.make_appliance:
if not opts.ks:
networks = []
else:
networks = ks.handler.network.network
make_appliance(opts.disk_image or disk_img, opts.app_name,
opts.app_template, opts.app_file, networks, opts.ram,
opts.vcpus or 1, opts.arch, opts.title, opts.project, opts.releasever)
elif opts.make_pxe_live:
work_dir = tempfile.mkdtemp(prefix="lmc-work-")
log.info("working dir is %s", work_dir)
disk_img = opts.fs_image or opts.disk_image or disk_img
log.debug("disk image is %s", disk_img)
result_dir = make_live_images(opts, work_dir, disk_img)
if result_dir is None:
log.error("Creating PXE live image failed.")
raise RuntimeError("Creating PXE live image failed.")
if opts.result_dir != opts.tmp and result_dir:
copytree(result_dir, opts.result_dir, preserve=False)
shutil.rmtree(result_dir)
result_dir = None
return (result_dir, disk_img)
def main(argv=None):
##
## OPTION PROCESSING
##
op = argparse.ArgumentParser()
op.add_argument("-i", "--input", dest="input",
help=_("a basis file to use for seeding the kickstart data (optional)"))
op.add_argument("-o", "--output", dest="output",
help=_("the location to write the finished kickstart file, or stdout if not given"))
op.add_argument("-v", "--version", dest="version", default=DEVEL,
help=_("version of kickstart syntax to validate against"))
opts = op.parse_args(argv)
##
## SETTING UP PYKICKSTART
##
try:
kshandler = makeVersion(opts.version)
except KickstartVersionError:
print(_("The version %s is not supported by pykickstart") % opts.version)
return 1
ksparser = KickstartParser(kshandler, followIncludes=True, errorsAreFatal=False)
if opts.input:
try:
processedFile = preprocessKickstart(opts.input)
ksparser.readKickstart(processedFile)
os.remove(processedFile)
except KickstartError as e:
# Errors should just dump you to the prompt anyway.
print(_("Warning: The following error occurred when processing the input file:\n%s\n") % e)
internalCommands = {".clear": ClearCommand(),
".show": ShowCommand(),
".quit": QuitCommand()}
##
## SETTING UP READLINE
##
readline.parse_and_bind("tab: complete")
readline.set_completer(KickstartCompleter(kshandler, internalCommands).complete)
# Since everything in kickstart looks like a command line arg, we need to
# remove '-' from the delimiter string.
delims = readline.get_completer_delims()
readline.set_completer_delims(delims.replace('-', ''))
##
## REPL
##
print("Press ^D to exit.")
while True:
try:
line = six.moves.input("ks> ") # pylint: disable=no-member
except EOFError:
# ^D was hit, time to quit.
break
except KeyboardInterrupt:
# ^C was hit, time to quit. Don't be like other programs.
break
# All internal commands start with a ., so if that's the beginning of the
# line, we need to dispatch ourselves.
if line.startswith("."):
words = line.split()
if words[0] in internalCommands:
try:
internalCommands[words[0]].execute(ksparser)
except EOFError:
# ".quit" was typed, time to quit.
break
else:
print(_("Internal command %s not recognized.") % words[0])
continue
# Now process the line of input as if it were a kickstart file - just an
# extremely short one.
try:
ksparser.readKickstartFromString(line)
except KickstartError as e:
print(e)
# And finally, print the output kickstart file.
if opts.output:
with open(opts.output, "w") as fd:
fd.write(str(ksparser.handler))
else:
print("\n" + str(ksparser.handler))
return 0
except KickstartError as e:
print(
_("Error reading %(filename)s:\n%(version)s") % {
"filename": opts.ksfile,
"version": e
})
cleanup(destdir)
try:
handler = makeVersion(opts.version)
except KickstartVersionError:
print(_("The version %s is not supported by pykickstart") % opts.version)
cleanup(destdir)
ksparser = KickstartParser(handler,
followIncludes=opts.followincludes,
errorsAreFatal=opts.firsterror)
# turn DeprecationWarnings into errors
warnings.filterwarnings("error")
processedFile = None
try:
processedFile = preprocessKickstart(f)
ksparser.readKickstart(processedFile)
cleanup(destdir, processedFile, exitval=0)
except DeprecationWarning as msg:
print(_("File uses a deprecated option or command.\n%s") % msg)
cleanup(destdir, processedFile)
except KickstartParseError as msg:
def start_build(cfg,
dnflock,
gitlock,
branch,
recipe_name,
compose_type,
test_mode=0):
""" Start the build
:param cfg: Configuration object
:type cfg: ComposerConfig
:param dnflock: Lock and YumBase for depsolving
:type dnflock: YumLock
:param recipe: The recipe to build
:type recipe: str
:param compose_type: The type of output to create from the recipe
:type compose_type: str
:returns: Unique ID for the build that can be used to track its status
:rtype: str
"""
share_dir = cfg.get("composer", "share_dir")
lib_dir = cfg.get("composer", "lib_dir")
# Make sure compose_type is valid
if compose_type not in compose_types(share_dir):
raise RuntimeError("Invalid compose type (%s), must be one of %s" %
(compose_type, compose_types(share_dir)))
with gitlock.lock:
(commit_id, recipe) = read_recipe_and_id(gitlock.repo, branch,
recipe_name)
# Combine modules and packages and depsolve the list
# TODO include the version/glob in the depsolving
module_nver = recipe.module_nver
package_nver = recipe.package_nver
projects = sorted(set(module_nver + package_nver),
key=lambda p: p[0].lower())
deps = []
try:
with dnflock.lock:
(installed_size,
deps) = projects_depsolve_with_size(dnflock.dbo,
projects,
with_core=False)
except ProjectsError as e:
log.error("start_build depsolve: %s", str(e))
raise RuntimeError("Problem depsolving %s: %s" %
(recipe["name"], str(e)))
# Read the kickstart template for this type
ks_template_path = joinpaths(share_dir, "composer", compose_type) + ".ks"
ks_template = open(ks_template_path, "r").read()
# How much space will the packages in the default template take?
ks_version = makeVersion()
ks = KickstartParser(ks_version,
errorsAreFatal=False,
missingIncludeIsFatal=False)
ks.readKickstartFromString(ks_template + "\n%end\n")
ks_projects = [(name, "*") for name in ks.handler.packages.packageList]
try:
with dnflock.lock:
(template_size, _) = projects_depsolve_with_size(
dnflock.dbo,
ks_projects,
with_core=not ks.handler.packages.nocore)
except ProjectsError as e:
log.error("start_build depsolve: %s", str(e))
raise RuntimeError("Problem depsolving %s: %s" %
(recipe["name"], str(e)))
log.debug("installed_size = %d, template_size=%d", installed_size,
template_size)
# Minimum LMC disk size is 1GiB, and anaconda bumps the estimated size up by 10% (which doesn't always work).
# XXX BUT Anaconda has a bug, it won't execute a kickstart on a disk smaller than 3000 MB
# XXX There is an upstream patch pending, but until then, use that as the minimum
installed_size = max(3e9, int((installed_size + template_size))) * 1.2
log.debug("/ partition size = %d", installed_size)
# Create the results directory
build_id = str(uuid4())
results_dir = joinpaths(lib_dir, "results", build_id)
os.makedirs(results_dir)
# Write the recipe commit hash
commit_path = joinpaths(results_dir, "COMMIT")
with open(commit_path, "w") as f:
f.write(commit_id)
# Write the original recipe
recipe_path = joinpaths(results_dir, "blueprint.toml")
with open(recipe_path, "w") as f:
f.write(recipe.toml())
# Write the frozen recipe
frozen_recipe = recipe.freeze(deps)
recipe_path = joinpaths(results_dir, "frozen.toml")
with open(recipe_path, "w") as f:
f.write(frozen_recipe.toml())
# Write out the dependencies to the results dir
deps_path = joinpaths(results_dir, "deps.toml")
with open(deps_path, "w") as f:
f.write(toml.dumps({"packages": deps}))
# Save a copy of the original kickstart
shutil.copy(ks_template_path, results_dir)
with dnflock.lock:
repos = list(dnflock.dbo.repos.iter_enabled())
if not repos:
raise RuntimeError("No enabled repos, canceling build.")
# Create the final kickstart with repos and package list
ks_path = joinpaths(results_dir, "final-kickstart.ks")
with open(ks_path, "w") as f:
ks_url = repo_to_ks(repos[0], "url")
log.debug("url = %s", ks_url)
f.write('url %s\n' % ks_url)
for idx, r in enumerate(repos[1:]):
ks_repo = repo_to_ks(r, "baseurl")
log.debug("repo composer-%s = %s", idx, ks_repo)
f.write('repo --name="composer-%s" %s\n' % (idx, ks_repo))
# Write the root partition and it's size in MB (rounded up)
f.write('part / --fstype="ext4" --size=%d\n' %
ceil(installed_size / 1024**2))
f.write(ks_template)
for d in deps:
f.write(dep_nevra(d) + "\n")
f.write("%end\n")
add_customizations(f, recipe)
# Setup the config to pass to novirt_install
log_dir = joinpaths(results_dir, "logs/")
cfg_args = compose_args(compose_type)
# Get the title, project, and release version from the host
if not os.path.exists("/etc/os-release"):
log.error(
"/etc/os-release is missing, cannot determine product or release version"
)
os_release = SimpleConfigFile("/etc/os-release")
os_release.read()
log.debug("os_release = %s", os_release)
cfg_args["title"] = os_release.get("PRETTY_NAME")
cfg_args["project"] = os_release.get("NAME")
cfg_args["releasever"] = os_release.get("VERSION_ID")
cfg_args["volid"] = ""
cfg_args.update({
"compression": "xz",
"compress_args": [],
"ks": [ks_path],
"logfile": log_dir,
"timeout": 60, # 60 minute timeout
})
with open(joinpaths(results_dir, "config.toml"), "w") as f:
f.write(toml.dumps(cfg_args))
# Set the initial status
open(joinpaths(results_dir, "STATUS"), "w").write("WAITING")
# Set the test mode, if requested
if test_mode > 0:
open(joinpaths(results_dir, "TEST"), "w").write("%s" % test_mode)
log.info("Adding %s (%s %s) to compose queue", build_id, recipe["name"],
compose_type)
os.symlink(results_dir, joinpaths(lib_dir, "queue/new/", build_id))
return build_id
help=_("version of kickstart syntax to validate against"))
opts = op.parse_args(sys.argv[1:])
##
## SETTING UP PYKICKSTART
##
try:
kshandler = makeVersion(opts.version)
except KickstartVersionError:
print(_("The version %s is not supported by pykickstart") % opts.version)
sys.exit(1)
ksparser = KickstartParser(kshandler,
followIncludes=True,
errorsAreFatal=False)
if opts.input:
try:
processedFile = preprocessKickstart(opts.input)
ksparser.readKickstart(processedFile)
os.remove(processedFile)
except KickstartError as e:
# Errors should just dump you to the prompt anyway.
print(
_("Warning: The following error occurred when processing the input file:\n%s\n"
) % e)
internalCommands = {
".clear": ClearCommand(),
def main(argv):
op = argparse.ArgumentParser(usage="%(prog)s [options] ksfile [ksfile...]",
add_help=False)
op.add_argument("ksfile",
nargs="*",
help=_("filename or URL to read from"))
op.add_argument("-e",
"--firsterror",
dest="firsterror",
action="store_true",
default=False,
help=_("halt after the first error or warning"))
op.add_argument("-i",
"--followincludes",
dest="followincludes",
action="store_true",
default=False,
help=_("parse include files when %%include is seen"))
op.add_argument("-l",
"--listversions",
dest="listversions",
action="store_true",
default=False,
help=_("list the available versions of kickstart syntax"))
op.add_argument("-v",
"--version",
dest="version",
default=DEVEL,
help=_("version of kickstart syntax to validate against"))
op.add_argument("-h",
"--help",
dest="help",
action="store_true",
default=False,
help=_("show this help message and exit"))
opts = op.parse_args(argv)
# parse --help manually b/c we don't want to sys.exit before the
# tests have finished
if opts.help:
return (0, op.format_help().split("\n"))
if opts.listversions:
versions = []
for key in sorted(versionMap.keys()):
versions.append(key)
return (0, versions)
if not opts.ksfile:
return (1, op.format_usage().split("\n"))
rc = 0
retmsg = []
# unpack any globs
ksfiles = []
for inksfile in opts.ksfile:
if is_url(inksfile):
ksfiles.append(inksfile)
else:
ksfiles.extend(glob.glob(inksfile))
# check if there are any files to check
if not ksfiles:
return (1, ["No files match the patterns."])
# iterate over files to check them
with tempfile.TemporaryDirectory(prefix="ksvalidator-tmp-") as destdir:
for ksfile in ksfiles:
print(
_("\nChecking kickstart file %(filename)s\n" %
{"filename": ksfile}))
try:
f = load_to_file(ksfile, os.path.join(destdir, "ks.cfg"))
except KickstartError as e:
rc += 1
retmsg.append(
_("Error reading %(filename)s:\n%(version)s") % {
"filename": ksfile,
"version": e
})
try:
handler = makeVersion(opts.version)
except KickstartVersionError:
# return immediately because bad version is fatal for all files
return (1, [
_("The version %s is not supported by pykickstart") %
opts.version
])
# turn kickstart parse warnings into errors
warnings.filterwarnings(action="error",
category=KickstartParseWarning)
ksparser = KickstartParser(handler,
followIncludes=opts.followincludes,
errorsAreFatal=opts.firsterror)
try:
processedFile = preprocessKickstart(f)
if processedFile is None:
raise RuntimeError("Empty file")
ksparser.readKickstart(processedFile)
rc += ksparser.errorsCount
except KickstartDeprecationWarning as err:
rc += 1
retmsg.append(
_("File uses a deprecated option or command.\n%s") % err)
except KickstartParseError as err:
rc += 1
retmsg.append(str(err))
except KickstartError:
rc += 1
retmsg.append(_("General kickstart error in input file"))
except Exception as e: # pylint: disable=broad-except
rc += 1
retmsg.append(_("General error in input file: %s") % e)
return rc, retmsg
