def monitor(cfg):
"""Monitor the queue for new compose requests
:param cfg: Configuration settings
:type cfg: DataHolder
:returns: Does not return
The queue has 2 subdirectories, new and run. When a compose is ready to be run
a symlink to the uniquely named results directory should be placed in ./queue/new/
When the it is ready to be run (it is checked every 30 seconds or after a previous
compose is finished) the symlink will be moved into ./queue/run/ and a STATUS file
will be created in the results directory.
STATUS can contain one of: WAITING, RUNNING, FINISHED, FAILED
If the system is restarted while a compose is running it will move any old symlinks
from ./queue/run/ to ./queue/new/ and rerun them.
"""
def queue_sort(uuid):
"""Sort the queue entries by their mtime, not their names"""
return os.stat(joinpaths(cfg.composer_dir, "queue/new", uuid)).st_mtime
check_queues(cfg)
while True:
uuids = sorted(os.listdir(joinpaths(cfg.composer_dir, "queue/new")),
key=queue_sort)
# Pick the oldest and move it into ./run/
if not uuids:
# No composes left to process, sleep for a bit
time.sleep(5)
else:
src = joinpaths(cfg.composer_dir, "queue/new", uuids[0])
dst = joinpaths(cfg.composer_dir, "queue/run", uuids[0])
try:
os.rename(src, dst)
except OSError:
# The symlink may vanish if uuid_cancel() has been called
continue
log.info("Starting new compose: %s", dst)
open(joinpaths(dst, "STATUS"), "w").write("RUNNING\n")
try:
make_compose(cfg, os.path.realpath(dst))
log.info("Finished building %s, results are in %s", dst,
os.path.realpath(dst))
open(joinpaths(dst, "STATUS"), "w").write("FINISHED\n")
write_timestamp(dst, TS_FINISHED)
except Exception:
import traceback
log.error("traceback: %s", traceback.format_exc())
# TODO - Write the error message to an ERROR-LOG file to include with the status
# log.error("Error running compose: %s", e)
open(joinpaths(dst, "STATUS"), "w").write("FAILED\n")
write_timestamp(dst, TS_FINISHED)
os.unlink(dst)
def test_timestamp(self):
"""Test writing and reading compose timestamps"""
write_timestamp(self.test_dir, TS_CREATED)
ts = timestamp_dict(self.test_dir)
self.assertTrue(TS_CREATED in ts)
self.assertTrue(TS_STARTED not in ts)
self.assertTrue(TS_FINISHED not in ts)
write_timestamp(self.test_dir, TS_STARTED)
ts = timestamp_dict(self.test_dir)
self.assertTrue(TS_CREATED in ts)
self.assertTrue(TS_STARTED in ts)
self.assertTrue(TS_FINISHED not in ts)
write_timestamp(self.test_dir, TS_FINISHED)
ts = timestamp_dict(self.test_dir)
self.assertTrue(TS_CREATED in ts)
self.assertTrue(TS_STARTED in ts)
self.assertTrue(TS_FINISHED in ts)
def start_build(cfg,
dnflock,
gitlock,
branch,
recipe_name,
compose_type,
test_mode=0):
""" Start the build
:param cfg: Configuration object
:type cfg: ComposerConfig
:param dnflock: Lock and YumBase for depsolving
:type dnflock: YumLock
:param recipe: The recipe to build
:type recipe: str
:param compose_type: The type of output to create from the recipe
:type compose_type: str
:returns: Unique ID for the build that can be used to track its status
:rtype: str
"""
share_dir = cfg.get("composer", "share_dir")
lib_dir = cfg.get("composer", "lib_dir")
# Make sure compose_type is valid
if compose_type not in compose_types(share_dir):
raise RuntimeError("Invalid compose type (%s), must be one of %s" %
(compose_type, compose_types(share_dir)))
# Some image types (live-iso) need extra packages for composer to execute the output template
with dnflock.lock:
extra_pkgs = get_extra_pkgs(dnflock.dbo, share_dir, compose_type)
log.debug("Extra packages needed for %s: %s", compose_type, extra_pkgs)
with gitlock.lock:
(commit_id, recipe) = read_recipe_and_id(gitlock.repo, branch,
recipe_name)
# Combine modules and packages and depsolve the list
module_nver = recipe.module_nver
package_nver = recipe.package_nver
package_nver.extend([(name, '*') for name in extra_pkgs])
projects = sorted(set(module_nver + package_nver),
key=lambda p: p[0].lower())
deps = []
log.info("depsolving %s", recipe["name"])
try:
# This can possibly update repodata and reset the YumBase object.
with dnflock.lock_check:
(installed_size,
deps) = projects_depsolve_with_size(dnflock.dbo,
projects,
recipe.group_names,
with_core=False)
except ProjectsError as e:
log.error("start_build depsolve: %s", str(e))
raise RuntimeError("Problem depsolving %s: %s" %
(recipe["name"], str(e)))
# Read the kickstart template for this type
ks_template_path = joinpaths(share_dir, "composer", compose_type) + ".ks"
ks_template = open(ks_template_path, "r").read()
# How much space will the packages in the default template take?
ks_version = makeVersion()
ks = KickstartParser(ks_version,
errorsAreFatal=False,
missingIncludeIsFatal=False)
ks.readKickstartFromString(ks_template + "\n%end\n")
pkgs = [(name, "*") for name in ks.handler.packages.packageList]
grps = [grp.name for grp in ks.handler.packages.groupList]
try:
with dnflock.lock:
(template_size, _) = projects_depsolve_with_size(
dnflock.dbo,
pkgs,
grps,
with_core=not ks.handler.packages.nocore)
except ProjectsError as e:
log.error("start_build depsolve: %s", str(e))
raise RuntimeError("Problem depsolving %s: %s" %
(recipe["name"], str(e)))
log.debug("installed_size = %d, template_size=%d", installed_size,
template_size)
# Minimum LMC disk size is 1GiB, and anaconda bumps the estimated size up by 10% (which doesn't always work).
installed_size = int((installed_size + template_size)) * 1.2
log.debug("/ partition size = %d", installed_size)
# Create the results directory
build_id = str(uuid4())
results_dir = joinpaths(lib_dir, "results", build_id)
os.makedirs(results_dir)
# Write the recipe commit hash
commit_path = joinpaths(results_dir, "COMMIT")
with open(commit_path, "w") as f:
f.write(commit_id)
# Write the original recipe
recipe_path = joinpaths(results_dir, "blueprint.toml")
with open(recipe_path, "w") as f:
f.write(recipe.toml())
# Write the frozen recipe
frozen_recipe = recipe.freeze(deps)
recipe_path = joinpaths(results_dir, "frozen.toml")
with open(recipe_path, "w") as f:
f.write(frozen_recipe.toml())
# Write out the dependencies to the results dir
deps_path = joinpaths(results_dir, "deps.toml")
with open(deps_path, "w") as f:
f.write(toml.dumps({"packages": deps}))
# Save a copy of the original kickstart
shutil.copy(ks_template_path, results_dir)
with dnflock.lock:
repos = list(dnflock.dbo.repos.iter_enabled())
if not repos:
raise RuntimeError("No enabled repos, canceling build.")
# Create the git rpms, if any, and return the path to the repo under results_dir
gitrpm_repo = create_gitrpm_repo(results_dir, recipe)
# Create the final kickstart with repos and package list
ks_path = joinpaths(results_dir, "final-kickstart.ks")
with open(ks_path, "w") as f:
ks_url = repo_to_ks(repos[0], "url")
log.debug("url = %s", ks_url)
f.write('url %s\n' % ks_url)
for idx, r in enumerate(repos[1:]):
ks_repo = repo_to_ks(r, "baseurl")
log.debug("repo composer-%s = %s", idx, ks_repo)
f.write('repo --name="composer-%s" %s\n' % (idx, ks_repo))
if gitrpm_repo:
log.debug("repo gitrpms = %s", gitrpm_repo)
f.write('repo --name="gitrpms" --baseurl="file://%s"\n' %
gitrpm_repo)
# Setup the disk for booting
# TODO Add GPT and UEFI boot support
f.write('clearpart --all --initlabel\n')
# Write the root partition and it's size in MB (rounded up)
f.write('part / --size=%d\n' % ceil(installed_size / 1024**2))
# Some customizations modify the template before writing it
f.write(customize_ks_template(ks_template, recipe))
for d in deps:
f.write(dep_nevra(d) + "\n")
# Include the rpms from the gitrpm repo directory
if gitrpm_repo:
for rpm in glob(os.path.join(gitrpm_repo, "*.rpm")):
f.write(os.path.basename(rpm)[:-4] + "\n")
f.write("%end\n")
# Other customizations can be appended to the kickstart
add_customizations(f, recipe)
# Setup the config to pass to novirt_install
log_dir = joinpaths(results_dir, "logs/")
cfg_args = compose_args(compose_type)
# Get the title, project, and release version from the host
if not os.path.exists("/etc/os-release"):
log.error(
"/etc/os-release is missing, cannot determine product or release version"
)
os_release = flatconfig("/etc/os-release")
log.debug("os_release = %s", dict(os_release.items()))
cfg_args["title"] = os_release.get("PRETTY_NAME", "")
cfg_args["project"] = os_release.get("NAME", "")
cfg_args["releasever"] = os_release.get("VERSION_ID", "")
cfg_args["volid"] = ""
cfg_args["extra_boot_args"] = get_kernel_append(recipe)
cfg_args.update({
"compression": "xz",
"compress_args": [],
"ks": [ks_path],
"logfile": log_dir,
"timeout": 60, # 60 minute timeout
})
with open(joinpaths(results_dir, "config.toml"), "w") as f:
f.write(toml.dumps(cfg_args))
# Set the initial status
open(joinpaths(results_dir, "STATUS"), "w").write("WAITING")
# Set the test mode, if requested
if test_mode > 0:
open(joinpaths(results_dir, "TEST"), "w").write("%s" % test_mode)
write_timestamp(results_dir, TS_CREATED)
log.info("Adding %s (%s %s) to compose queue", build_id, recipe["name"],
compose_type)
os.symlink(results_dir, joinpaths(lib_dir, "queue/new/", build_id))
return build_id
def make_compose(cfg, results_dir):
"""Run anaconda with the final-kickstart.ks from results_dir
:param cfg: Configuration settings
:type cfg: DataHolder
:param results_dir: The directory containing the metadata and results for the build
:type results_dir: str
:returns: Nothing
:raises: May raise various exceptions
This takes the final-kickstart.ks, and the settings in config.toml and runs Anaconda
in no-virt mode (directly on the host operating system). Exceptions should be caught
at the higer level.
If there is a failure, the build artifacts will be cleaned up, and any logs will be
moved into logs/anaconda/ and their ownership will be set to the user from the cfg
object.
"""
# Check on the ks's presence
ks_path = joinpaths(results_dir, "final-kickstart.ks")
if not os.path.exists(ks_path):
raise RuntimeError("Missing kickstart file at %s" % ks_path)
# Load the compose configuration
cfg_path = joinpaths(results_dir, "config.toml")
if not os.path.exists(cfg_path):
raise RuntimeError("Missing config.toml for %s" % results_dir)
cfg_dict = toml.loads(open(cfg_path, "r").read())
# The keys in cfg_dict correspond to the arguments setup in livemedia-creator
# keys that define what to build should be setup in compose_args, and keys with
# defaults should be setup here.
# Make sure that image_name contains no path components
cfg_dict["image_name"] = os.path.basename(cfg_dict["image_name"])
# Only support novirt installation, set some other defaults
cfg_dict["no_virt"] = True
cfg_dict["disk_image"] = None
cfg_dict["fs_image"] = None
cfg_dict["keep_image"] = False
cfg_dict["domacboot"] = False
cfg_dict["anaconda_args"] = ""
cfg_dict["proxy"] = ""
cfg_dict["armplatform"] = ""
cfg_dict["squashfs_args"] = None
cfg_dict["lorax_templates"] = find_templates(cfg.share_dir)
cfg_dict["tmp"] = cfg.tmp
cfg_dict["dracut_args"] = None # Use default args for dracut
# TODO How to support other arches?
cfg_dict["arch"] = None
# Compose things in a temporary directory inside the results directory
cfg_dict["result_dir"] = joinpaths(results_dir, "compose")
os.makedirs(cfg_dict["result_dir"])
install_cfg = DataHolder(**cfg_dict)
# Some kludges for the 99-copy-logs %post, failure in it will crash the build
for f in ["/tmp/NOSAVE_INPUT_KS", "/tmp/NOSAVE_LOGS"]:
open(f, "w")
# Placing a CANCEL file in the results directory will make execWithRedirect send anaconda a SIGTERM
def cancel_build():
return os.path.exists(joinpaths(results_dir, "CANCEL"))
log.debug("cfg = %s", install_cfg)
try:
test_path = joinpaths(results_dir, "TEST")
write_timestamp(results_dir, TS_STARTED)
if os.path.exists(test_path):
# Pretend to run the compose
time.sleep(5)
try:
test_mode = int(open(test_path, "r").read())
except Exception:
test_mode = 1
if test_mode == 1:
raise RuntimeError("TESTING FAILED compose")
else:
open(joinpaths(results_dir, install_cfg.image_name),
"w").write("TEST IMAGE")
else:
run_creator(install_cfg, cancel_func=cancel_build)
# Extract the results of the compose into results_dir and cleanup the compose directory
move_compose_results(install_cfg, results_dir)
finally:
# Make sure any remaining temporary directories are removed (eg. if there was an exception)
for d in glob(joinpaths(cfg.tmp, "lmc-*")):
if os.path.isdir(d):
shutil.rmtree(d)
elif os.path.isfile(d):
os.unlink(d)
# Make sure that everything under the results directory is owned by the user
user = pwd.getpwuid(cfg.uid).pw_name
group = grp.getgrgid(cfg.gid).gr_name
log.debug("Install finished, chowning results to %s:%s", user, group)
subprocess.call(["chown", "-R", "%s:%s" % (user, group), results_dir])
def monitor(cfg):
"""Monitor the queue for new compose requests
:param cfg: Configuration settings
:type cfg: DataHolder
:returns: Does not return
The queue has 2 subdirectories, new and run. When a compose is ready to be run
a symlink to the uniquely named results directory should be placed in ./queue/new/
When the it is ready to be run (it is checked every 30 seconds or after a previous
compose is finished) the symlink will be moved into ./queue/run/ and a STATUS file
will be created in the results directory.
STATUS can contain one of: WAITING, RUNNING, FINISHED, FAILED
If the system is restarted while a compose is running it will move any old symlinks
from ./queue/run/ to ./queue/new/ and rerun them.
"""
def queue_sort(uuid):
"""Sort the queue entries by their mtime, not their names"""
return os.stat(joinpaths(cfg.composer_dir, "queue/new", uuid)).st_mtime
check_queues(cfg)
while True:
uuids = sorted(os.listdir(joinpaths(cfg.composer_dir, "queue/new")),
key=queue_sort)
# Pick the oldest and move it into ./run/
if not uuids:
# No composes left to process, sleep for a bit
time.sleep(5)
else:
src = joinpaths(cfg.composer_dir, "queue/new", uuids[0])
dst = joinpaths(cfg.composer_dir, "queue/run", uuids[0])
try:
os.rename(src, dst)
except OSError:
# The symlink may vanish if uuid_cancel() has been called
continue
# The anaconda logs are also copied into ./anaconda/ in this directory
os.makedirs(joinpaths(dst, "logs"), exist_ok=True)
def open_handler(loggers, file_name):
handler = logging.FileHandler(joinpaths(
dst, "logs", file_name))
handler.setLevel(logging.DEBUG)
handler.setFormatter(
logging.Formatter(
"%(asctime)s %(levelname)s: %(message)s"))
for logger in loggers:
logger.addHandler(handler)
return (handler, loggers)
loggers = (((log, program_log, dnf_log),
"combined.log"), ((log, ), "composer.log"),
((program_log, ), "program.log"), ((dnf_log, ),
"dnf.log"))
handlers = [
open_handler(loggers, file_name)
for loggers, file_name in loggers
]
log.info("Starting new compose: %s", dst)
open(joinpaths(dst, "STATUS"), "w").write("RUNNING\n")
try:
make_compose(cfg, os.path.realpath(dst))
log.info("Finished building %s, results are in %s", dst,
os.path.realpath(dst))
open(joinpaths(dst, "STATUS"), "w").write("FINISHED\n")
write_timestamp(dst, TS_FINISHED)
except Exception:
import traceback
log.error("traceback: %s", traceback.format_exc())
# TODO - Write the error message to an ERROR-LOG file to include with the status
# log.error("Error running compose: %s", e)
open(joinpaths(dst, "STATUS"), "w").write("FAILED\n")
write_timestamp(dst, TS_FINISHED)
finally:
for handler, loggers in handlers:
for logger in loggers:
logger.removeHandler(handler)
handler.close()
os.unlink(dst)