def __postprocess_cache(self): """ Postprocesses the patching RPMs cache. """ results = self._get_results() copy_tasks = [] for result in results: name, path, _ = result global patching_cache_path destination_path = os.path.join(patching_cache_path, os.path.basename(path)) matching_task = None for task in self._tasks: if task[0] == name: name, marked_rpm_path, _, release, updates = task matching_task = (name, marked_rpm_path, release, updates) if matching_task is None: raise Exception("Cannot match task for {0}".format(name)) info_path = "{0}.info.txt".format(destination_path) info = "{0}".format(matching_task) with open(info_path, "wb") as info_file: info_file.write(info) copy_tasks.append((name, path, destination_path)) hidden_subprocess.function_call_list("Copying to cache", shutil.copy, copy_tasks)
def __deploy_packages(self): """ Deploys packages to chroot clones and generates makefiles for them. """ self._tasks.sort(key=lambda task: os.stat(task[1]).st_size) copy_tasks = [] for i in range(repository_combiner.jobs_number): tasks = [] i_task = i while i_task < len(self._tasks): tasks.append(self._tasks[i_task]) i_task += repository_combiner.jobs_number if len(tasks) == 0: continue directories = {} for task in tasks: package_name, package_path, target, _, _ = task copy_tasks.append( (package_name, package_path, self.patching_root_clones[i])) self._targets[package_name] = target basename = os.path.basename(target) self._package_names[basename] = package_name self._generate_makefile(self.patching_root_clones[i], tasks) hidden_subprocess.function_call_list("Copying to patcher", shutil.copy, copy_tasks)
def __do_idle_tasks(self): """ Does idle tasks, i. e. just copies RPMs in case when RPM patching is totally disabled. """ tasks = [] for task in self._tasks: package_name, package_path, target, _, _ = task check.file_exists(package_path) tasks.append((package_name, package_path, target)) hidden_subprocess.function_call_list("Copying w/o patching", shutil.copy, tasks)
def __process_results(self): """ Processes final results of patcher. """ results = self._get_results() copy_tasks = [] for info in results: name, path, _ = info target = self._targets[name] copy_tasks.append((name, path, target)) hidden_subprocess.function_call_list("Copying to repo", shutil.copy, copy_tasks)
def __build_dependency_graph_vertices(self, yum_base): """ Builds vertices of repository dependency graph. @param yum_base The YUM base. @return Forward and backward dependency graphs (with vertices only). """ graph = DependencyGraph() back_graph = DependencyGraph() yum_sack = yum_base.pkgSack # Remember IDs of packages in the hash. id_packages = {} i = 0 packages = yum_sack.returnPackages() graph.add_vertices(len(packages)) back_graph.add_vertices(len(packages)) tasks = [] names = [] full_names = [] locations = [] versions = [] releases = [] requirements = [] added_packages = [] for package in packages: task = (package.name, package, names, full_names, locations, versions, releases, requirements, added_packages, yum_sack, graph, back_graph) tasks.append(task) hidden_subprocess.function_call_list("Building vertices", self.__build_vertex, tasks) for i in range(len(names)): name_id = graph.get_name_id(names[i]) if i != name_id: raise Exception("name id = {0} for package #{1}".format( name_id, i)) graph.vs["name"] = names graph.vs["full_name"] = full_names graph.vs["location"] = locations graph.vs["version"] = versions graph.vs["release"] = releases graph.vs["requirements"] = requirements back_graph.vs["name"] = names back_graph.vs["full_name"] = full_names back_graph.vs["location"] = locations back_graph.vs["version"] = versions back_graph.vs["release"] = releases back_graph.vs["requirements"] = requirements return graph, back_graph
def __preprocess_cache(self): """ Preprocesses the patching RPMs cache. """ global drop_patching_cache if drop_patching_cache: global patching_cache_path hidden_subprocess.call("Drop patching cache", ["sudo", "rm", "-rf", patching_cache_path]) os.makedirs(patching_cache_path) return ready_rpms = files.find_fast(patching_cache_path, ".*\.rpm") info_items = {} for rpm in ready_rpms: info_path = "{0}.info.txt".format(rpm) if os.path.isfile(info_path): with open(info_path, "r") as info_file: lines = [] for line in info_file: lines.append(line) info_item = lines[0] info_items[info_item] = rpm for info_item in info_items.keys(): logging.info("Found item {0} at location " "{1}".format(info_item, info_items[info_item])) copy_tasks = [] tasks_undone = [] for i_task in range(len(self._tasks)): task = self._tasks[i_task] name, path, destination, release, updates = task info = "{0}".format((name, path, release, updates)) logging.info("Searching for {0}".format(info)) if_cached = False for key in info_items.keys(): if key == info: cached_package_path = info_items[key] logging.info("Found already patched RPM at " "{0}".format(cached_package_path)) copy_tasks.append((name, cached_package_path, destination)) if_cached = True break if not if_cached: tasks_undone.append(task) self._tasks = tasks_undone if len(copy_tasks) > 0: hidden_subprocess.function_call_list("Copying from cache", shutil.copy, copy_tasks)
def __clone_chroots(self): """ Clones patching chroot to several clones. """ clone_tasks = [] for i in range(repository_combiner.jobs_number): clone_path = temporaries.create_temporary_directory( "patching_root_clone.{0}".format(i)) shutil.rmtree(clone_path) self.patching_root_clones.append(clone_path) clone_tasks.append(("chroot #{0}".format(i), ["cp", "-a", self.patching_root, clone_path])) hidden_subprocess.function_call_list("Cloning chroot", subprocess.call, clone_tasks)
def construct_combined_repository(graph, marked_graph, marked_packages, if_mirror, rpm_patcher): """ Constructs the temporary repository that consists of symbolic links to packages from non-marked and marked repositories. @param graph Dependency graph of the non-marked repository @param marked_graph Dependency graph of the marked repository @param marked_packages Set of marked package names @param if_mirror Whether to mirror not found marked packages from non-marked repository @param rpm_patcher The patcher of RPMs. @return The path to the constructed combined repository. """ check_rpm_versions(graph, marked_graph, marked_packages) repository_path = temporaries.create_temporary_directory("combirepo") packages_not_found = [] copy_tasks = [] for package in marked_packages: marked_package_id = marked_graph.get_name_id(package) if marked_package_id is None: packages_not_found.append(package) continue location_from = marked_graph.vs[marked_package_id]["location"] release_marked = marked_graph.vs[marked_package_id]["release"] package_id = graph.get_name_id(package) if package_id is None: copy_tasks.append((package, location_from, repository_path)) else: release = graph.vs[package_id]["release"] location_original = graph.vs[package_id]["location"] new_name = os.path.basename(location_original) location_to = os.path.join(repository_path, new_name) if_patching_needed = False if release != release_marked: logging.debug("Release numbers of package {0} differ: " "{1} and {2}".format(package, release, release_marked)) if_patching_needed = True updates = get_requirements_updates( package, graph.vs[package_id]["requirements"], marked_graph.vs[marked_package_id]["requirements"]) if len(updates) > 0: logging.debug("Requirements updates are necessary.") if_patching_needed = True if if_patching_needed: rpm_patcher.add_task(package, location_from, location_to, release, updates) else: copy_tasks.append((package, location_from, repository_path)) if len(packages_not_found) != 0: for package in packages_not_found: logging.warning("Marked package {0} not found in marked " "repository".format(package)) if not if_mirror: logging.error("The above listed packages were not found in " "marked repository.\n" "HINT: use option -m to use non-marked packages " "instead of them.") sys.exit("Error.") packages = Set(graph.vs["name"]) for package in packages: if package in marked_packages: if package not in packages_not_found: continue if package in packages_not_found: logging.info("Package {0} from original repository will be " "used (mirror mode is on).".format(package)) package_id = graph.get_name_id(package) location_from = graph.vs[package_id]["location"] copy_tasks.append((package, location_from, repository_path)) hidden_subprocess.function_call_list("Copying", shutil.copy, copy_tasks) if logging.getLogger().getEffectiveLevel() == logging.DEBUG: hidden_subprocess.silent_call(["ls", "-l", repository_path]) return repository_path