def __init__(self, rebuild_metadata, pkg_source):
self.pkg_source = pkg_source
self.packages = set(rebuild_metadata['packages'])
if 'metapackage' in rebuild_metadata:
self.metapackage = rebuild_metadata['metapackage']
self.repo = rebuild_metadata['repo']
self.prefix = rebuild_metadata['prefix']
self.koji_tag = rebuild_metadata['koji_tag']
self.path = tempfile.mkdtemp()
self.built_packages = set()
self.num_of_deps = {}
self.circular_deps = []
self.get_files()
self.graph = PackageGraph(self.repo, self.pkg_source)
try:
self.recipes = rebuild_metadata['recipes']
except IOError:
logger.error("Failed to load recipe {0}.".format(rebuild_metadata['recipes']))
class Builder(metaclass=ABCMeta):
'''
Abstract superclass of builder classes.
'''
def __init__(self, rebuild_metadata, pkg_source):
self.pkg_source = pkg_source
self.packages = set(rebuild_metadata['packages'])
if 'metapackage' in rebuild_metadata:
self.metapackage = rebuild_metadata['metapackage']
self.repo = rebuild_metadata['repo']
self.prefix = rebuild_metadata['prefix']
self.koji_tag = rebuild_metadata['koji_tag']
self.path = tempfile.mkdtemp()
self.built_packages = set()
self.num_of_deps = {}
self.circular_deps = []
self.get_files()
self.graph = PackageGraph(self.repo, self.pkg_source)
try:
self.recipes = rebuild_metadata['recipes']
except IOError:
logger.error("Failed to load recipe {0}.".format(rebuild_metadata['recipes']))
def __del__(self):
shutil.rmtree(self.path)
shutil.rmtree(self.__tempdir)
@property
def path(self):
return self.__path
@path.setter
def path(self, value):
self.__tempdir = value
value += '/rebuild_tool-{0}/'.format(self.repo)
if not os.path.isdir(value):
os.makedirs(value)
self.__path = value
@property
def recipes(self):
return self.__recipes
@recipes.setter
def recipes(self, recipe_files):
if not recipe_files:
self.__recipes = None
else:
self.__recipes = [Recipe(recipe) for recipe in recipe_files]
def get_relations(self):
'''
Runs graph analysis and get dependance tree and circular_deps
'''
self.graph.make_graph()
self.circular_deps = self.graph.get_cycles()
if self.circular_deps and not self.recipes:
raise MissingRecipeException(
"Missing recipes to resolve circular dependencies in graph.")
def deps_satisfied(self, package):
'''
Compares package deps with self.build_packages to
check if are all dependancies already built
'''
if set(self.graph.G.successors(package)) <= self.built_packages:
return True
return False
def recipe_deps_satisfied(self, recipe):
'''
Checks if all packages in recipe have satisfied their
dependencies on packages that are not in recipe
'''
deps = set()
for pkg in recipe.packages:
if not pkg in self.packages:
raise KeyError("Package {} from recipe missing in packages list".format(pkg))
deps |= set(self.graph.G.successors(pkg))
if (deps - recipe.packages) <= self.built_packages:
return True
return False
@check_build
def build(self, pkgs, verbose=True):
for pkg in pkgs:
if verbose:
print("Building {0}...".format(pkg))
return True
def run_building(self):
'''
First builds all packages without deps, then iterates over num_of_deps
and simulate building of packages in right order
'''
# Build and add metapackage to chroots when rebuilding scl
if hasattr(self, 'metapackage'):
self.build([self.metapackage])
self.add_chroot_pkg([self.metapackage])
while self.packages > self.built_packages:
zero_deps = self.graph.get_leaf_nodes()
if zero_deps:
self.build(zero_deps)
else:
for recipe in self.recipes:
if list(self.packages - self.built_packages)[1] in recipe.packages and\
self.recipe_deps_satisfied(recipe):
self.build_following_recipe(recipe)
break
else:
sys.stderr.write("Recipe to resolve circular dependencies not found.\n")
raise SystemExit(1)
def find_recipe(self, package):
'''
Search for recipe including package in self.recipes
'''
for recipe in self.recipes:
if package in recipe.packages:
return recipe
raise MissingRecipeException("Recipe for package {0} not found".format(package))
def build_following_recipe(self, recipe):
'''
Builds packages in order and macro values discribed in given
recipe.
'''
for step in recipe.order:
if len(step) == 1:
print("Building package {0}".format(step[0]))
else:
(name, macro_value) = step
print("Building package {0} {1}".format(name, macro_value))
(macro, value) = macro_value.split(' ')
utils.check_bootstrap_macro(self.pkg_source[name].full_path_spec, macro)
utils.edit_bootstrap(self.pkg_source[name].full_path_spec, macro, value)
self.pkg_source[name].pack()
self.build([step[0]], False)
for pkg in {step[0] for step in recipe.order}:
self.graph.G.remove_node(pkg)
self.recipes.remove(recipe)
def get_files(self):
'''
Creates SrpmArchive object and downloads files for each package
'''
with utils.ChangeDir(self.path):
for package in self.packages:
pkg_dir = self.path + package + "_files/"
if not os.path.exists(pkg_dir):
os.mkdir(pkg_dir)
print("Getting files of {0}.".format(package))
logger.debug("Getting files of {0}.".format(package))
self.pkg_source.add(package, pkg_dir, self.repo, self.prefix, self.koji_tag)
def test_get_leaf_nodes(self, pkg_source, expected):
graph = PackageGraph("rawhide", pkg_source)
graph.make_graph()
assert set(graph.get_leaf_nodes()) == expected
def tests_get_cycles(self, pkg_source, expected):
graph = PackageGraph("rawhide", pkg_source)
graph.make_graph()
assert graph.get_cycles() == expected
def test_rpms(self, pkg_source, expected):
graph = PackageGraph("rawhide", pkg_source)
graph.make_graph()
assert graph.rpms == expected
def test_get_leaf_nodes(self, pkg_source, expected):
graph = PackageGraph("rawhide", pkg_source)
graph.make_graph()
assert set(graph.get_leaf_nodes()) == expected
class TestGraph(object):
fake_python = flexmock(package='python3',
rpms={
'python3-devel', 'python3-test',
'python3-debug', 'python3', 'python3-debuginfo',
'python3-tools', 'python3-tkinter',
'python3-libs'
},
dependencies={
'valgrind-devel', 'python-macros', 'tk-devel',
'gdbm-devel', 'python3-pip', 'tar', 'tix-devel',
'python3-setuptools'
})
fake_setuptools = flexmock(
package='python-setuptools',
rpms={'python3-setuptools', 'python-setuptools'},
dependencies={
'python3-pytest', 'python3-mock', 'python3-pip', 'python-mock',
'python-pip', 'python2-devel', 'pytest', 'python3-devel'
})
fake_pip = flexmock(package='python_pip',
rpms={'python-pip', 'python3-pip'},
dependencies={
'python3-pip', 'python-pip', 'python-wheel',
'python-devel', 'python3-wheel',
'python-setuptools', 'python3-devel',
'python3-setuptools'
})
fake_nodeps = flexmock(package='python_six',
rpms={'python-nodeps', 'python3-six'},
dependencies=set())
class_pkg_source = {
'python3': fake_python,
'python-setuptools': fake_setuptools,
'python-pip': fake_pip,
'python-nodeps': fake_nodeps
}
class_graph = PackageGraph("rawhide", class_pkg_source)
class_graph.make_graph()
@pytest.mark.parametrize(
('pkg_source', 'expected'),
[(class_pkg_source, {
'python3-devel', 'python3-test', 'python3-debug', 'python3',
'python3-debuginfo', 'python3-tools', 'python3-tkinter',
'python3-libs', 'python3-setuptools', 'python-setuptools',
'python-pip', 'python3-pip', 'python-nodeps', 'python3-six'
}), ({}, set()),
({
'python-setuptools': fake_setuptools
}, {'python3-setuptools', 'python-setuptools'})])
def test_rpms(self, pkg_source, expected):
graph = PackageGraph("rawhide", pkg_source)
graph.make_graph()
assert graph.rpms == expected
@pytest.mark.parametrize(
('pkg', 'expected'),
[('python3', {'python-pip', 'python-setuptools'}),
('python-setuptools', {'python3', 'python-pip'}),
('python-pip', {'python-pip', 'python3', 'python-setuptools'}),
('python-nodeps', set())])
def test_successors(self, pkg, expected):
assert set(TestGraph.class_graph.G.successors(pkg)) == expected
@pytest.mark.parametrize(
('pkg', 'expected'),
[('python3', {'python-pip', 'python-setuptools'}),
('python-setuptools', {'python3', 'python-pip'}),
('python-pip', {'python-pip', 'python3', 'python-setuptools'}),
('python-nodeps', set())])
def test_predecessors(self, pkg, expected):
assert set(TestGraph.class_graph.G.predecessors(pkg)) == expected
@pytest.mark.parametrize(
('pkg_source', 'expected'), [(class_pkg_source, {'python-nodeps'}),
({}, set()),
({
'python-setuptools': fake_setuptools,
'python-pip': fake_pip,
'python3': fake_python
}, set())])
def test_get_leaf_nodes(self, pkg_source, expected):
graph = PackageGraph("rawhide", pkg_source)
graph.make_graph()
assert set(graph.get_leaf_nodes()) == expected
@pytest.mark.parametrize(
('pkg_source', 'expected'),
[(class_pkg_source, [{'python3', 'python-setuptools', 'python-pip'}]),
({}, []),
({
'python-setuptools': fake_setuptools,
'python-pip': fake_pip
}, [{'python-pip', 'python-setuptools'}])])
def tests_get_cycles(self, pkg_source, expected):
graph = PackageGraph("rawhide", pkg_source)
graph.make_graph()
assert graph.get_cycles() == expected
def test_rpms(self, pkg_source, expected):
graph = PackageGraph("rawhide", pkg_source)
graph.make_graph()
assert graph.rpms == expected
def tests_get_cycles(self, pkg_source, expected):
graph = PackageGraph("rawhide", pkg_source)
graph.make_graph()
assert graph.get_cycles() == expected
