def __init__(self, rebuild_metadata, pkg_source): self.pkg_source = pkg_source self.packages = set(rebuild_metadata['packages']) if 'metapackage' in rebuild_metadata: self.metapackage = rebuild_metadata['metapackage'] self.repo = rebuild_metadata['repo'] self.prefix = rebuild_metadata['prefix'] self.koji_tag = rebuild_metadata['koji_tag'] self.path = tempfile.mkdtemp() self.built_packages = set() self.num_of_deps = {} self.circular_deps = [] self.get_files() self.graph = PackageGraph(self.repo, self.pkg_source) try: self.recipes = rebuild_metadata['recipes'] except IOError: logger.error("Failed to load recipe {0}.".format(rebuild_metadata['recipes']))
class Builder(metaclass=ABCMeta): ''' Abstract superclass of builder classes. ''' def __init__(self, rebuild_metadata, pkg_source): self.pkg_source = pkg_source self.packages = set(rebuild_metadata['packages']) if 'metapackage' in rebuild_metadata: self.metapackage = rebuild_metadata['metapackage'] self.repo = rebuild_metadata['repo'] self.prefix = rebuild_metadata['prefix'] self.koji_tag = rebuild_metadata['koji_tag'] self.path = tempfile.mkdtemp() self.built_packages = set() self.num_of_deps = {} self.circular_deps = [] self.get_files() self.graph = PackageGraph(self.repo, self.pkg_source) try: self.recipes = rebuild_metadata['recipes'] except IOError: logger.error("Failed to load recipe {0}.".format(rebuild_metadata['recipes'])) def __del__(self): shutil.rmtree(self.path) shutil.rmtree(self.__tempdir) @property def path(self): return self.__path @path.setter def path(self, value): self.__tempdir = value value += '/rebuild_tool-{0}/'.format(self.repo) if not os.path.isdir(value): os.makedirs(value) self.__path = value @property def recipes(self): return self.__recipes @recipes.setter def recipes(self, recipe_files): if not recipe_files: self.__recipes = None else: self.__recipes = [Recipe(recipe) for recipe in recipe_files] def get_relations(self): ''' Runs graph analysis and get dependance tree and circular_deps ''' self.graph.make_graph() self.circular_deps = self.graph.get_cycles() if self.circular_deps and not self.recipes: raise MissingRecipeException( "Missing recipes to resolve circular dependencies in graph.") def deps_satisfied(self, package): ''' Compares package deps with self.build_packages to check if are all dependancies already built ''' if set(self.graph.G.successors(package)) <= self.built_packages: return True return False def recipe_deps_satisfied(self, recipe): ''' Checks if all packages in recipe have satisfied their dependencies on packages that are not in recipe ''' deps = set() for pkg in recipe.packages: if not pkg in self.packages: raise KeyError("Package {} from recipe missing in packages list".format(pkg)) deps |= set(self.graph.G.successors(pkg)) if (deps - recipe.packages) <= self.built_packages: return True return False @check_build def build(self, pkgs, verbose=True): for pkg in pkgs: if verbose: print("Building {0}...".format(pkg)) return True def run_building(self): ''' First builds all packages without deps, then iterates over num_of_deps and simulate building of packages in right order ''' # Build and add metapackage to chroots when rebuilding scl if hasattr(self, 'metapackage'): self.build([self.metapackage]) self.add_chroot_pkg([self.metapackage]) while self.packages > self.built_packages: zero_deps = self.graph.get_leaf_nodes() if zero_deps: self.build(zero_deps) else: for recipe in self.recipes: if list(self.packages - self.built_packages)[1] in recipe.packages and\ self.recipe_deps_satisfied(recipe): self.build_following_recipe(recipe) break else: sys.stderr.write("Recipe to resolve circular dependencies not found.\n") raise SystemExit(1) def find_recipe(self, package): ''' Search for recipe including package in self.recipes ''' for recipe in self.recipes: if package in recipe.packages: return recipe raise MissingRecipeException("Recipe for package {0} not found".format(package)) def build_following_recipe(self, recipe): ''' Builds packages in order and macro values discribed in given recipe. ''' for step in recipe.order: if len(step) == 1: print("Building package {0}".format(step[0])) else: (name, macro_value) = step print("Building package {0} {1}".format(name, macro_value)) (macro, value) = macro_value.split(' ') utils.check_bootstrap_macro(self.pkg_source[name].full_path_spec, macro) utils.edit_bootstrap(self.pkg_source[name].full_path_spec, macro, value) self.pkg_source[name].pack() self.build([step[0]], False) for pkg in {step[0] for step in recipe.order}: self.graph.G.remove_node(pkg) self.recipes.remove(recipe) def get_files(self): ''' Creates SrpmArchive object and downloads files for each package ''' with utils.ChangeDir(self.path): for package in self.packages: pkg_dir = self.path + package + "_files/" if not os.path.exists(pkg_dir): os.mkdir(pkg_dir) print("Getting files of {0}.".format(package)) logger.debug("Getting files of {0}.".format(package)) self.pkg_source.add(package, pkg_dir, self.repo, self.prefix, self.koji_tag)
def test_get_leaf_nodes(self, pkg_source, expected): graph = PackageGraph("rawhide", pkg_source) graph.make_graph() assert set(graph.get_leaf_nodes()) == expected
def tests_get_cycles(self, pkg_source, expected): graph = PackageGraph("rawhide", pkg_source) graph.make_graph() assert graph.get_cycles() == expected
def test_rpms(self, pkg_source, expected): graph = PackageGraph("rawhide", pkg_source) graph.make_graph() assert graph.rpms == expected
class TestGraph(object): fake_python = flexmock(package='python3', rpms={ 'python3-devel', 'python3-test', 'python3-debug', 'python3', 'python3-debuginfo', 'python3-tools', 'python3-tkinter', 'python3-libs' }, dependencies={ 'valgrind-devel', 'python-macros', 'tk-devel', 'gdbm-devel', 'python3-pip', 'tar', 'tix-devel', 'python3-setuptools' }) fake_setuptools = flexmock( package='python-setuptools', rpms={'python3-setuptools', 'python-setuptools'}, dependencies={ 'python3-pytest', 'python3-mock', 'python3-pip', 'python-mock', 'python-pip', 'python2-devel', 'pytest', 'python3-devel' }) fake_pip = flexmock(package='python_pip', rpms={'python-pip', 'python3-pip'}, dependencies={ 'python3-pip', 'python-pip', 'python-wheel', 'python-devel', 'python3-wheel', 'python-setuptools', 'python3-devel', 'python3-setuptools' }) fake_nodeps = flexmock(package='python_six', rpms={'python-nodeps', 'python3-six'}, dependencies=set()) class_pkg_source = { 'python3': fake_python, 'python-setuptools': fake_setuptools, 'python-pip': fake_pip, 'python-nodeps': fake_nodeps } class_graph = PackageGraph("rawhide", class_pkg_source) class_graph.make_graph() @pytest.mark.parametrize( ('pkg_source', 'expected'), [(class_pkg_source, { 'python3-devel', 'python3-test', 'python3-debug', 'python3', 'python3-debuginfo', 'python3-tools', 'python3-tkinter', 'python3-libs', 'python3-setuptools', 'python-setuptools', 'python-pip', 'python3-pip', 'python-nodeps', 'python3-six' }), ({}, set()), ({ 'python-setuptools': fake_setuptools }, {'python3-setuptools', 'python-setuptools'})]) def test_rpms(self, pkg_source, expected): graph = PackageGraph("rawhide", pkg_source) graph.make_graph() assert graph.rpms == expected @pytest.mark.parametrize( ('pkg', 'expected'), [('python3', {'python-pip', 'python-setuptools'}), ('python-setuptools', {'python3', 'python-pip'}), ('python-pip', {'python-pip', 'python3', 'python-setuptools'}), ('python-nodeps', set())]) def test_successors(self, pkg, expected): assert set(TestGraph.class_graph.G.successors(pkg)) == expected @pytest.mark.parametrize( ('pkg', 'expected'), [('python3', {'python-pip', 'python-setuptools'}), ('python-setuptools', {'python3', 'python-pip'}), ('python-pip', {'python-pip', 'python3', 'python-setuptools'}), ('python-nodeps', set())]) def test_predecessors(self, pkg, expected): assert set(TestGraph.class_graph.G.predecessors(pkg)) == expected @pytest.mark.parametrize( ('pkg_source', 'expected'), [(class_pkg_source, {'python-nodeps'}), ({}, set()), ({ 'python-setuptools': fake_setuptools, 'python-pip': fake_pip, 'python3': fake_python }, set())]) def test_get_leaf_nodes(self, pkg_source, expected): graph = PackageGraph("rawhide", pkg_source) graph.make_graph() assert set(graph.get_leaf_nodes()) == expected @pytest.mark.parametrize( ('pkg_source', 'expected'), [(class_pkg_source, [{'python3', 'python-setuptools', 'python-pip'}]), ({}, []), ({ 'python-setuptools': fake_setuptools, 'python-pip': fake_pip }, [{'python-pip', 'python-setuptools'}])]) def tests_get_cycles(self, pkg_source, expected): graph = PackageGraph("rawhide", pkg_source) graph.make_graph() assert graph.get_cycles() == expected