def create_image(arch, repository_names, repository_paths, kickstart_file_path,
mic_options, specific_packages):
"""
Creates an image using MIC tool, from given repository and given kickstart
file. It creates a copy of kickstart file and replaces "repo" to given
repository path.
@param arch The architecture of the image
@param repository_names The names of repositorues
@param repository_paths The repository paths
@param kickstart_file The kickstart file to be used
@param mic_options Additional options for MIC.
@param specific_packages Packages that must be additionally
installed.
"""
if repository_names is None or len(repository_names) == 0:
raise Exception("Repository names are not given! "
"{0}".format(repository_names))
modified_kickstart_file_path = temporaries.create_temporary_file("mod.ks")
shutil.copy(kickstart_file_path, modified_kickstart_file_path)
kickstart_file = KickstartFile(modified_kickstart_file_path)
kickstart_file.replace_repository_paths(repository_names, repository_paths)
kickstart_file.add_packages(specific_packages)
# Now create the image using the "mic" tool:
global mic_config_path
mic_command = [
"sudo", "mic", "create", "loop", modified_kickstart_file_path, "-A",
arch, "--config", mic_config_path, "--tmpfs"
]
if logging.getLogger().getEffectiveLevel() <= logging.DEBUG:
mic_options.extend(["--debug", "--verbose"])
if mic_options is not None:
mic_command.extend(mic_options)
hidden_subprocess.call("Building the image", mic_command)
def __deploy_packages(self):
"""
Deploys packages to chroot clones and generates makefiles for them.
"""
self._tasks.sort(key=lambda task: os.stat(task[1]).st_size)
for i in range(repository_combiner.jobs_number):
tasks = []
i_task = i
while i_task < len(self._tasks):
tasks.append(self._tasks[i_task])
i_task += repository_combiner.jobs_number
if len(tasks) == 0:
continue
directories = {}
for task in tasks:
package_name, package_path, target, _, _ = task
self._targets[package_name] = target
basename = os.path.basename(target)
self._package_names[basename] = package_name
hidden_subprocess.call(
"Copying to patcher",
["sudo", "cp", package_path, self.patching_root_clones[i]])
self._generate_makefile(self.patching_root_clones[i], tasks)
def __process_results(self):
"""
Processes final results of patcher.
"""
results = self._get_results()
for info in results:
name, path, _ = info
target = self._targets[name]
hidden_subprocess.call("Copying to repo",
["sudo", "cp", path, target])
def __install_rpmrebuild(self, queue):
"""
Chroots to the given path and installs rpmrebuild in it.
@param queue The queue where the result will be put.
"""
make_command = [
"sudo", "chroot", self.patching_root, "bash", "-c",
"""chmod a+x /usr/bin/*; cd /rpmrebuild/src && make && make install"""
]
hidden_subprocess.call("Make and install the rpmrebuild.",
make_command)
queue.put(True)
def __preprocess_cache(self):
"""
Preprocesses the patching RPMs cache.
"""
global drop_patching_cache
if drop_patching_cache:
global patching_cache_path
hidden_subprocess.call("Drop patching cache",
["sudo", "rm", "-rf", patching_cache_path])
os.makedirs(patching_cache_path)
return
ready_rpms = files.find_fast(patching_cache_path, ".*\.rpm")
info_items = {}
for rpm in ready_rpms:
info_path = "{0}.info.txt".format(rpm)
if os.path.isfile(info_path):
with open(info_path, "r") as info_file:
lines = []
for line in info_file:
lines.append(line)
info_item = lines[0]
info_items[info_item] = rpm
for info_item in info_items.keys():
logging.info("Found item {0} at location "
"{1}".format(info_item, info_items[info_item]))
copy_tasks = []
tasks_undone = []
for i_task in range(len(self._tasks)):
task = self._tasks[i_task]
name, path, destination, release, updates = task
info = "{0}".format((name, path, release, updates))
logging.info("Searching for {0}".format(info))
if_cached = False
for key in info_items.keys():
if key == info:
cached_package_path = info_items[key]
logging.info("Found already patched RPM at "
"{0}".format(cached_package_path))
copy_tasks.append((name, cached_package_path, destination))
if_cached = True
break
if not if_cached:
tasks_undone.append(task)
self._tasks = tasks_undone
if len(copy_tasks) > 0:
hidden_subprocess.function_call_list("Copying from cache",
shutil.copy, copy_tasks)
def __install_rpmrebuild(self, queue):
"""
Chroots to the given path and installs rpmrebuild in it.
@param queue The queue where the result will be put.
"""
os.chroot(self.patching_root)
os.chdir("/")
os.chdir("/rpmrebuild/src")
hidden_subprocess.call("Making the rpmrebuild.", ["make"])
hidden_subprocess.call("Installing the rpmrebuild.",
["make", "install"])
if not check.command_exists("rpmrebuild"):
sys.exit("Error.")
queue.put(True)
def __prepare(self):
"""
Prepares the patching root ready for RPM patching.
"""
global developer_disable_patching
if developer_disable_patching:
logging.debug("RPM patcher will not be prepared.")
return
graphs = self._graphs
self.__prepare_image(graphs)
self.patching_root = temporaries.mount_firmware(self.images_directory)
host_arch = platform.machine()
host_arches = self.__produce_architecture_synonyms_list(host_arch)
if self.architecture not in host_arches:
self.__deploy_qemu_package()
combirepo_dir = os.path.abspath(os.path.dirname(__file__))
rpmrebuild_file = os.path.join(combirepo_dir, 'data/rpmrebuild.tar')
already_present_rpmrebuilds = files.find_fast(self.patching_root,
"rpmrebuild.*")
for already_present_rpmrebuild in already_present_rpmrebuilds:
if os.path.isdir(already_present_rpmrebuild):
shutil.rmtree(already_present_rpmrebuild)
elif os.path.isfile(already_present_rpmrebuild):
os.remove(already_present_rpmrebuild)
hidden_subprocess.call(
"Extracting the rpmrebuild ",
["tar", "xf", rpmrebuild_file, "-C", self.patching_root])
queue = multiprocessing.Queue()
child = multiprocessing.Process(target=self.__install_rpmrebuild,
args=(queue, ))
child.start()
child.join()
if queue.empty():
logging.error("Failed to install rpmrebuild into chroot.")
sys.exit("Error.")
else:
result = queue.get()
if result:
logging.debug("Installation of rpmrebuild successfully "
"completed.")
else:
raise Exception("Impossible happened.")
def __use_cached_root_or_prepare(self):
"""
Tries to find cached root and uses it in case it exists and prepares
it otherwise.
"""
image_info = "{0}".format(
(self.names, self.repositories, self.architecture,
os.path.basename(self.kickstart_file_path)))
cached_images_info_paths = files.find_fast(
patching_cache_path, ".*preliminary_image.info.txt")
matching_images_path = None
self.__prepare()
for info_path in cached_images_info_paths:
cached_images_path = info_path.replace(".info.txt", "")
if not os.path.isdir(cached_images_path):
logging.error("Directory {0} not "
"found!".format(cached_images_path))
continue
lines = []
with open(info_path, "r") as info_file:
for line in info_file:
lines.append(line)
if lines[0] == image_info:
matching_images_path = cached_images_path
break
if matching_images_path is not None:
self.patching_root = matching_images_path
logging.info("Found already prepared patching root: "
"{0}".format(matching_images_path))
else:
cached_chroot_path = os.path.join(
patching_cache_path,
os.path.basename(self.patching_root) + "preliminary_image")
hidden_subprocess.call("Saving chroot to cache", [
"sudo", "cp", "-Z", "-P", "-a", self.patching_root,
cached_chroot_path
])
info_path = cached_chroot_path + ".info.txt"
with open(info_path, "wb") as info_file:
info_file.write(image_info)
def __postprocess_cache(self):
"""
Postprocesses the patching RPMs cache.
"""
results = self._get_results()
for result in results:
name, path, _ = result
global patching_cache_path
destination_path = os.path.join(patching_cache_path,
os.path.basename(path))
matching_task = None
for task in self._tasks:
if task[0] == name:
name, marked_rpm_path, _, release, updates = task
matching_task = (name, marked_rpm_path, release, updates)
if matching_task is None:
raise Exception("Cannot match task for {0}".format(name))
info_path = "{0}.info.txt".format(destination_path)
info = "{0}".format(matching_task)
with open(info_path, "wb") as info_file:
info_file.write(info)
hidden_subprocess.call("Copying to cache",
["sudo", "cp", path, destination_path])
def create_patched_packages(queue):
"""
Patches the given package using rpmrebuild and the patching root.
@param root The root to be used.
"""
root = queue.get()
if not os.path.isfile(root + "/Makefile"):
logging.info("Chroot has no jobs to perform.")
queue.task_done()
return
logging.debug("Chrooting to {0}".format(root))
make_command = [
"sudo", "chroot", root, "bash", "-c", """chmod a+x /usr/bin/*;
rm -f /var/lib/rpm/__db.*;
make --silent"""
]
hidden_subprocess.call("Start rpm patching", make_command)
logging.debug("Exiting from {0}".format(root))
queue.task_done()
def generate_derived_data(self):
"""
Generates the automatically generated data of the repository.
"""
self._path = os.path.abspath(self._path)
initial_directory = os.getcwd()
os.chdir(self._path)
repodata_path = os.path.join(self._path, "repodata")
if os.path.isdir(repodata_path):
logging.warning("The repository data already exists in "
"{0}! It will be removed and "
"re-generated".format(repodata_path))
shutil.rmtree(repodata_path)
os.mkdir(repodata_path)
else:
os.mkdir(repodata_path)
createrepo_command = [
"createrepo", self._path, "--database", "--unique-md-filenames"
]
if self.data.groups_data is not None:
groups_file_path = os.path.join(repodata_path, "group.xml")
with open(groups_file_path, "w") as groups_file:
groups_file.writelines(self.data.groups_data)
createrepo_command.extend(["-g", "repodata/group.xml"])
exit_value = hidden_subprocess.call("Creating repository.",
createrepo_command)
if exit_value != 0:
raise Exception("createrepo failed with exit value = "
"{0}".format(exit_value))
if self.data.patterns_data is not None:
patterns_file_path = os.path.join(repodata_path, "patterns.xml")
with open(patterns_file_path, "w") as patterns_file:
patterns_file.writelines(self.data.patterns_data)
exit_value = hidden_subprocess.silent_call(
["modifyrepo", patterns_file_path, repodata_path])
if exit_value != 0:
raise Exception("modifyrepo failed with exit value = "
"{0}".format(exit_value))
self.__workaround_repodata_open_checksum_bug()
os.chdir(initial_directory)
def _generate_makefile(self, root, tasks):
"""
Generates makefile for given tasks.
@param root The root to be used.
@param tasks The list of tasks.
"""
makefile_path = os.path.join(root, "Makefile")
results_path = os.path.join(root, "rpmrebuild_results")
if os.path.isfile(makefile_path):
hidden_subprocess.call("Remove Makefile.",
["sudo", "rm", makefile_path])
hidden_subprocess.call("Create Makefile.",
["sudo", "touch", makefile_path])
hidden_subprocess.call("Change mode of Makefile.",
["sudo", "chmod", "a+rw", makefile_path])
if os.path.isdir(results_path):
hidden_subprocess.call("Remove results_path directory.",
["sudo", "rm", "-rf", results_path])
hidden_subprocess.call("Create results_path directory.",
["sudo", "mkdir", "-m", "777", results_path])
with open(makefile_path, "ab") as makefile:
makefile.write("all:")
for task in tasks:
package_name, _, _, _, _ = task
makefile.write(" {0}".format(package_name))
makefile.write("\n")
for task in tasks:
package_name, package_path, _, release, updates = task
package_file_name = os.path.basename(package_path)
makefile.write("\n")
makefile.write("{0}: {1}\n".format(package_name,
package_file_name))
sed_command = ""
if (updates):
sed_command = "-f \'sed"
for update in updates:
command = build_requirement_command(update)
sed_command += " -e \"{0}\"".format(command)
sed_command += "\'"
spec_commands = []
# skip %buildroot and basic.target.wants files in spec
spec_commands.append(
"--change-spec-files=\'sed -e \"/\.build-id/d\" -e \"/basic\.target\.wants/d\"\'"
)
# remove -p option from %posttrans
spec_commands.append(
"--change-spec-posttrans=\'sed -e \"s/-p .*//g\"\'")
commands_subpackages = build_subpackages_commands(
package_path, release)
spec_commands.extend(commands_subpackages)
spec_command = ""
for command in spec_commands:
spec_command += (" " + command)
makefile.write("\trpmrebuild {0} {1} --release={2} -p -n -d "
"/rpmrebuild_results "
"{3}".format(spec_command, sed_command, release,
package_file_name))
if logging.getLogger().getEffectiveLevel() != logging.DEBUG:
makefile.write(" >/dev/null 2>/dev/null")
makefile.write("\n")
makefile.write("\nall:\n\trm -rf /home/*\n")
if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
subprocess.call(["cat", makefile_path])