def build_rpm(self, srpm, distro, arch, output_dir):
self.build_ret_code = 0
def move_files(output, files):
if not output.exists() and not output.is_dir():
Command("mkdir " + path_to_str(output)).execute()
for _out in files:
try:
Command("mv " + path_to_str(_out) + " " +
path_to_str(output)).execute()
except:
pass
def check_output(proc):
while proc.poll() is None:
line = proc.stdout.readline().decode("utf-8")
if PackageBuilder.regex.search(line):
yield line
self.build_ret_code = proc.returncode
_ret = list(
check_output(
subprocess.Popen([
"mock", "--no-clean", "--verbose", "--root",
distro + '-' + arch, "--rebuild",
path_to_str(srpm),
"--resultdir=" + path_to_str(self.temp_dir)
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT))) + list(
self.check_logs(self.temp_dir))
move_files(output_dir, self.temp_dir.glob("*.rpm"))
self.mock_logs = output_dir / "mock_logs"
move_files(self.mock_logs, self.temp_dir.glob("*.log"))
raise BuildException(_ret, self.build_ret_code)
def _compute_checksum(sources):
if sources.is_dir():
cmd = "find %s -type f -print0 | sort -z | xargs " \
"-0 sha1sum | sha1sum" % path_to_str(sources.resolve())
else:
cmd = "sha1sum %s" % path_to_str(sources.resolve())
return Command([cmd]).execute()[:7]
def _compute_checksum(sources):
if sources.is_dir():
cmd = "find %s -type f -print0 | sort -z | xargs " \
"-0 sha1sum | sha1sum" % path_to_str(sources.resolve())
else:
cmd = "sha1sum %s" % path_to_str(sources.resolve())
return cmd_output([cmd])[:7]
def build_rpm(self, srpm, distro, arch, output_dir):
""" builds rpm from source RPM to distro and architecture
binary RPM into output_dir """
self.build_ret_code = 0
def check_output(proc):
while proc.poll() is None:
line = proc.stdout.readline().decode("utf-8")
if self._regex.search(line):
yield line
self.build_ret_code = proc.returncode
_ret = list(
check_output(
subprocess.Popen(
[
"mock", "--no-clean",
"--verbose",
"--root", distro + '-' + arch,
"--rebuild", path_to_str(srpm),
"--resultdir=" + path_to_str(self.temp_dir)
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT
)
)
) + list(self._check_logs(self.temp_dir))
self._move_files(output_dir, self.temp_dir.glob("*.rpm"))
self.mock_logs = output_dir / "mock_logs"
self._move_files(self.mock_logs, self.temp_dir.glob("*.log"))
raise BuildException(_ret, self.build_ret_code)
def build_rpm(self, srpm, distro, arch, output_dir):
""" builds rpm from source RPM to distro and architecture
binary RPM into output_dir """
self.build_ret_code = 0
def check_output(proc):
while proc.poll() is None:
line = proc.stdout.readline().decode("utf-8")
if self._regex.search(line):
yield line
self.build_ret_code = proc.returncode
_ret = list(
check_output(
subprocess.Popen([
"mock", "--no-clean", "--verbose", "--root",
distro + '-' + arch, "--rebuild",
path_to_str(srpm),
"--resultdir=" + path_to_str(self.temp_dir)
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT))) + list(
self._check_logs(self.temp_dir))
self._move_files(output_dir, self.temp_dir.glob("*.rpm"))
self.mock_logs = output_dir / "mock_logs"
self._move_files(self.mock_logs, self.temp_dir.glob("*.log"))
raise BuildException(_ret, self.build_ret_code)
def extraction(self, source, dest):
source = path_to_str(source)
if is_zipfile(source):
Command("unzip {} -d {}".format(
source, path_to_str(dest))).execute()
return True
else:
return False
def _compute_checksum(sources):
if sources.is_dir():
cmd = "find %s -type f -print0 | sort -z | xargs " \
"-0 sha1sum | sha1sum" % path_to_str(sources.resolve())
else:
cmd = "cat %s | sha1sum" % path_to_str(sources.resolve())
logging.error(str(cmd))
return Command([cmd]).execute()[:7]
def extraction(self, source, dest):
source = path_to_str(source)
if is_tarfile(source):
Command("tar xf {} -C {}".format(source,
path_to_str(dest))).execute()
return True
else:
return False
def test_command_concat(self):
cmd = Command("cd %s" % path_to_str(self.test_project_dir))
cmd.append("cmake ..")
cmd.append(["make", "make test"])
self.assertRaises(TypeError, cmd.append, 4)
expected = "cd %s\ncmake ..\nmake\nmake test" %\
path_to_str(self.test_project_dir)
self.assertEqual(expected, str(cmd))
def test_command_concat(self):
cmd = Command("cd %s" % path_to_str(self.test_project_dir))
cmd.append("cmake ..")
cmd.append(["make", "make test"])
self.assertRaises(TypeError, cmd.append, 4)
expected = "cd %s\ncmake ..\nmake\nmake test" %\
path_to_str(self.test_project_dir)
self.assertEqual(expected, str(cmd))
def create_archive(self):
""" Creates archive (archvie_path) from Source folder """
self.spec.Source = self.spec.Name + "-" + self.spec.Version + ".tar.gz"
_tar = Command("tar zcf " + path_to_str(self.archive_path) + " -C " +
path_to_str(self.extracted_dir) +
" . --transform='s/^\./" + self.spec.Name + "-" +
self.spec.Version + "/g'")
_tar.execute()
logging.debug(str(_tar))
def build_srpm(spec_file, tarball, output_dir):
""" Builds source rpm from spec and tarball and moves it to the
output directory """
Command("rpmdev-setuptree").execute()
Command("cp " + path_to_str(tarball) +
' $(rpm --eval "%{_topdir}")/SOURCES').execute()
output = Command("rpmbuild -bs " + path_to_str(spec_file)).execute()
Command("mv " + path_to_str(output.split()[-1]) + " " +
path_to_str(output_dir)).execute()
def move_files(output, files):
if not output.exists() and not output.is_dir():
Command("mkdir " + path_to_str(output)).execute()
for _out in files:
try:
Command("mv " + path_to_str(_out) + " " +
path_to_str(output)).execute()
except:
pass
def build_srpm(spec_file, tarball, output_dir):
""" Builds source rpm from spec and tarball and moves it to the
output directory """
Command("rpmdev-setuptree").execute()
Command("cp " + path_to_str(tarball) +
' $(rpm --eval "%{_topdir}")/SOURCES').execute()
output = Command("rpmbuild -bs " + path_to_str(spec_file)).execute()
Command("mv " + path_to_str(output.split()[-1]) +
" " + path_to_str(output_dir)).execute()
def move_files(output, files):
if not output.exists() and not output.is_dir():
Command("mkdir " + path_to_str(output)).execute()
for _out in files:
try:
Command("mv " + path_to_str(_out) +
" " + path_to_str(output)).execute()
except:
pass
def create_archive(self):
""" Creates archive (archvie_path) from Source folder """
self.spec.Source = self.spec.Name + "-" + self.spec.Version + ".tar.gz"
_tar = Command("tar zcf " + path_to_str(self.archive_path) +
" -C " + path_to_str(self.extracted_dir) +
" . --transform='s/^\./" +
self.spec.Name + "-" + self.spec.Version + "/g'")
_tar.execute()
logging.debug(str(_tar))
def build_srpm(spec_file, tarball, output_dir):
"""builds RPM package with build_srpm and build_rpm"""
# Build srpm pakcage from given spec_file and tarball
Command("rpmdev-setuptree").execute()
Command("cp " + path_to_str(tarball) +
' $(rpm --eval "%{_topdir}")/SOURCES').execute()
output = Command("rpmbuild -bs " + path_to_str(spec_file)).execute()
Command("mv " + path_to_str(output.split()[-1]) + " " +
path_to_str(output_dir)).execute()
def test_tar_xz_extract(self):
self._source_loader.load_sources(
self._tar_xz,
self._tar_extracted)
self.assertTrue(
path.isdir(path_to_str(self._tar_extracted)))
self.assertEqual(
self.md5Tar(path_to_str(self._tar_xz)),
self.md5Dir(path_to_str(self._tar_extracted)))
self.assertExistInDir(["file1", "file2"], self._tar_extracted)
def build_srpm(spec_file, tarball, output_dir):
"""builds RPM package with build_srpm and build_rpm"""
# Build srpm pakcage from given spec_file and tarball
Command("rpmdev-setuptree").execute()
Command("cp " + path_to_str(tarball) +
' $(rpm --eval "%{_topdir}")/SOURCES').execute()
output = Command("rpmbuild -bs " + path_to_str(spec_file)).execute()
Command("mv " + path_to_str(output.split()[-1]) +
" " + path_to_str(output_dir)).execute()
def setUp(self):
self._source_loader = SourceLoader()
self._tar = None
self._tar_dir = self.test_project_dir / "archives"
self._tar_gz = self.test_project_dir / "archives" / "sample.tar.gz"
self._tar_xz = self.test_project_dir / "archives" / "sample.tar.xz"
self._tar_temp = Path("/var/tmp/rpg_test/")
self._tar_extracted = self._tar_temp / "extracted"
self._download = self._tar_temp / "download.tar.gz"
if path.isdir(path_to_str(self._tar_temp)):
rmtree(path_to_str(self._tar_temp))
makedirs(path_to_str(self._tar_temp))
makedirs(path_to_str(self._tar_extracted))
def setUp(self):
self._source_loader = SourceLoader()
self._tar = None
self._tar_dir = self.test_project_dir / "archives"
self._tar_gz = self.test_project_dir / "archives" / "sample.tar.gz"
self._tar_xz = self.test_project_dir / "archives" / "sample.tar.xz"
self._tar_temp = Path("/var/tmp/rpg_test/")
self._tar_extracted = self._tar_temp / "extracted"
self._download = self._tar_temp / "download.tar.gz"
if path.isdir(path_to_str(self._tar_temp)):
rmtree(path_to_str(self._tar_temp))
makedirs(path_to_str(self._tar_temp))
makedirs(path_to_str(self._tar_extracted))
def build_rpm_recover(self, distro, arch):
""" Repeatedly build rpm with mock and finds all build errors.
May raise RuntimeError on failed recover. """
def build():
self.build_srpm()
self.build_rpm(distro, arch)
def analyse():
_files_to_pkgs.installed(self.base_dir, self.spec, self.sack)
self.write_spec()
_files_to_pkgs = FilesToPkgsPlugin()
analyse()
while True:
try:
build()
except BuildException as be:
if not self._plugin_engine.execute_mock_recover(be.errors):
if be.return_code:
raise RuntimeError(
"Build failed! See logs in '{}'".format(
self._package_builder.mock_logs))
break
Command("rm -rf {}".format(path_to_str(self.spec_path))).execute()
analyse()
def patched(self, project_dir, spec, sack):
""" Finds dependencies via makedepend - This is not garanteed to be
all of them. Makedepend uses macro preprocessor and if it throws
and error makedepend didn't print deps. """
out = Command([
"find " + path_to_str(project_dir) + " -name " +
" -o -name ".join(
["'*." + ex + "'" for ex in self.EXT_CPP]
)
]).execute()
cc_makedep = ""
cc_included_files = []
for _f in out.splitlines():
try:
cc_makedep = Command("makedepend -w 1000 " + str(_f) +
" -f- 2>/dev/null").execute()
except CalledProcessError as e:
logging.warn(str(e.cmd) + "\n" + str(e.output))
continue
cc_included_files += [
s for s in cc_makedep.split()
if (s.startswith("/usr") or s.startswith("/include"))
and str(project_dir) not in s]
spec.required_files.update(cc_included_files)
spec.build_required_files.update(cc_included_files)
def md5Dir(self, d):
md5sum = Command(
"find " + path_to_str(d) +
r' -type f | sed -e "s/.*\///gm" | sort | md5sum'
).execute()
self.assertNotEqual(self.FNF_MD5, md5sum)
return md5sum
def build_rpm_recover(self, distro, arch):
""" Repeatedly build rpm with mock and finds all build errors.
May raise RuntimeError on failed recover. """
def build():
self.build_srpm()
self.build_rpm(distro, arch)
def analyse():
_files_to_pkgs.installed(self.base_dir, self.spec, self.sack)
self.write_spec()
_files_to_pkgs = FilesToPkgsPlugin()
analyse()
while True:
try:
build()
except BuildException as be:
if not self._plugin_engine.execute_mock_recover(be.errors):
if be.return_code:
raise RuntimeError(
"Build failed! See logs in '{}'"
.format(self._package_builder.mock_logs))
break
Command("rm -rf {}".format(path_to_str(self.spec_path))).execute()
analyse()
def test_command_execute_from(self):
cmd = Command("pwd\ncd c 2>/dev/null\npwd")
output = cmd.execute_from(self.test_project_dir)
path = path_to_str(self.test_project_dir.resolve())
expected = "%s\n%s/c\n" % (path, path)
self.assertEqual(expected, output)
# doesn't add 'cd work_dir' during execute to command lines
cur_dir = Path('.')
with self.assertRaises(subprocess.CalledProcessError) as ctx:
cmd.execute_from(cur_dir)
expected = "Command '['/bin/sh', '-c', 'cd %s "\
"&& pwd && cd c 2>/dev/null && pwd']' "\
"returned non-zero exit status 1"\
% path_to_str(cur_dir.resolve())
self.assertEqual(expected, str(ctx.exception))
def test_command_execute_from(self):
cmd = Command("pwd\ncd c 2>/dev/null\npwd")
output = cmd.execute(self.test_project_dir)
path = path_to_str(self.test_project_dir.resolve())
expected = "%s\n%s/c\n" % (path, path)
self.assertEqual(expected, output)
# doesn't add 'cd work_dir' during execute to command lines
cur_dir = Path('.')
with self.assertRaises(subprocess.CalledProcessError) as ctx:
cmd.execute(cur_dir)
expected = "Command '['/bin/sh', '-c', 'cd %s "\
"&& pwd && cd c 2>/dev/null && pwd']' "\
"returned non-zero exit status 1"\
% path_to_str(cur_dir.resolve())
self.assertEqual(expected, str(ctx.exception))
def execute_from(self, work_dir):
"""executes command in work_dir (instance of pathlib.Path),
can raise CalledProcessError"""
cd_workdir = ["cd %s" % path_to_str(work_dir.resolve())]
command_lines = self._assign_rpm_variables() + cd_workdir + \
[rpm.expandMacro(x) for x in self._command_lines]
return cmd_output(command_lines)
def execute_from(self, work_dir):
"""executes command in work_dir (instance of pathlib.Path),
can raise CalledProcessError"""
cd_workdir = ["cd %s" % path_to_str(work_dir.resolve())]
command_lines = self._assign_rpm_variables() + cd_workdir + \
[rpm.expandMacro(x) for x in self._command_lines]
return cmd_output(command_lines)
def md5Tar(self, t):
mdsum = Command(
"tar --list -f " + path_to_str(t) + " 2>/dev/null | "
"awk -F/ '{ if($NF != \"\") print $NF }' | "
r'sed -e "s/.*\///gm" | sort | md5sum'
).execute()
self.assertNotEqual(self.FNF_MD5, mdsum)
return mdsum
def load_sources(self, source_path, extraction_dir):
""" If source_path is a directory, path tree will be
copied to extraction_dir. If it is archive
May raise IOError """
logging.debug('load_sources({}, {}) called'
.format(str(source_path), str(extraction_dir)))
path = path_to_str(source_path)
esc_extr_dir = path_to_str(extraction_dir)
if isfile(path):
compression = self._get_compression_method(path)
if not compression:
raise IOError("Input source archive '{}' is incompatible!"
.format(path))
self._extract(path, esc_extr_dir, compression)
self._process_dir(esc_extr_dir)
elif isdir(str(source_path)):
self._copy_dir(str(source_path), str(extraction_dir))
else:
raise IOError("Input source archive/directory '{}' doesn't exists!"
.format(path))
def build_rpm(self, srpm, distro, arch, output_dir):
self.build_ret_code = 0
def move_files(output, files):
if not output.exists() and not output.is_dir():
Command("mkdir " + path_to_str(output)).execute()
for _out in files:
try:
Command("mv " + path_to_str(_out) +
" " + path_to_str(output)).execute()
except:
pass
def check_output(proc):
while proc.poll() is None:
line = proc.stdout.readline().decode("utf-8")
if PackageBuilder.regex.search(line):
yield line
self.build_ret_code = proc.returncode
_ret = list(
check_output(
subprocess.Popen(
[
"mock", "--no-clean",
"--verbose",
"--root", distro + '-' + arch,
"--rebuild", path_to_str(srpm),
"--resultdir=" + path_to_str(self.temp_dir)
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT
)
)
) + list(self.check_logs(self.temp_dir))
move_files(output_dir, self.temp_dir.glob("*.rpm"))
self.mock_logs = output_dir / "mock_logs"
move_files(self.mock_logs, self.temp_dir.glob("*.log"))
raise BuildException(_ret, self.build_ret_code)
def load_sources(self, source_path, extraction_dir):
""" If source_path is a directory, path tree will be
copied to extraction_dir. If it is archive
May raise IOError """
logging.debug('load_sources({}, {}) called'.format(
str(source_path), str(extraction_dir)))
path = path_to_str(source_path)
esc_extr_dir = path_to_str(extraction_dir)
if isfile(path):
compression = self._get_compression_method(path)
if not compression:
raise IOError(
"Input source archive '{}' is incompatible!".format(path))
self._extract(path, esc_extr_dir, compression)
self._process_dir(esc_extr_dir)
elif isdir(str(source_path)):
self._copy_dir(str(source_path), str(extraction_dir))
else:
raise IOError(
"Input source archive/directory '{}' doesn't exists!".format(
path))
def patched(self, project_dir, spec, sack):
out = Command([
"find " + path_to_str(project_dir) + " -name " +
" -o -name ".join(["'*." + ex + "'" for ex in self.EXT_CPP])
]).execute()
cc_makedep = ""
cc_included_files = []
for _f in out.splitlines():
try:
cc_makedep = Command("makedepend -w 1000 " + str(_f) +
" -f- 2>/dev/null").execute()
except CalledProcessError as e:
logging.warn(str(e.cmd) + "\n" + str(e.output))
continue
cc_included_files += [
s for s in cc_makedep.split()
if (s.startswith("/usr") or s.startswith("/include"))
and str(project_dir) not in s
]
spec.required_files.update(cc_included_files)
spec.build_required_files.update(cc_included_files)
def patched(self, project_dir, spec, sack):
out = Command([
"find " + path_to_str(project_dir) + " -name " +
" -o -name ".join(
["'*." + ex + "'" for ex in self.EXT_CPP]
)
]).execute()
cc_makedep = ""
cc_included_files = []
for _f in out.splitlines():
try:
cc_makedep = Command("makedepend -w 1000 " + str(_f) +
" -f- 2>/dev/null").execute()
except CalledProcessError as e:
logging.warn(str(e.cmd) + "\n" + str(e.output))
continue
cc_included_files += [
s for s in cc_makedep.split()
if (s.startswith("/usr") or s.startswith("/include"))
and str(project_dir) not in s]
spec.required_files.update(cc_included_files)
spec.build_required_files.update(cc_included_files)
def patched(self, project_dir, spec, sack):
""" Finds dependencies via makedepend - This is not garanteed to be
all of them. Makedepend uses macro preprocessor and if it throws
and error makedepend didn't print deps. """
out = Command([
"find " + path_to_str(project_dir) + " -name " +
" -o -name ".join(["'*." + ex + "'" for ex in self.EXT_CPP])
]).execute()
cc_makedep = ""
cc_included_files = []
for _f in out.splitlines():
try:
cc_makedep = Command("makedepend -w 1000 " + str(_f) +
" -f- 2>/dev/null").execute()
except CalledProcessError as e:
logging.warn(str(e.cmd) + "\n" + str(e.output))
continue
cc_included_files += [
s for s in cc_makedep.split()
if (s.startswith("/usr") or s.startswith("/include"))
and str(project_dir) not in s
]
spec.required_files.update(cc_included_files)
spec.build_required_files.update(cc_included_files)
def tearDown(self):
Command("rm -rf " + path_to_str(self.srpm) + "/mock_logs " +
path_to_str(self.package_builder.temp_dir / "*.rpm")).execute()
def write_spec(self):
""" Creates spec file or rewrites old one. """
with open(path_to_str(self.spec_path), 'w') as spec_file:
spec_file.write(str(self.spec))
def tearDown(self):
if self._tar:
remove(path_to_str(self._tar))
if path.isdir(path_to_str(self._tar_temp)):
rmtree(path_to_str(self._tar_temp))
def md5Tar(self, t):
mdsum = Command("tar --list -f " + path_to_str(t) + " 2>/dev/null | "
"awk -F/ '{ if($NF != \"\") print $NF }' | "
r'sed -e "s/.*\///gm" | sort | md5sum').execute()
self.assertNotEqual(self.FNF_MD5, mdsum)
return mdsum
def test_tar_xz_method(self):
self.assertEqual(
self._source_loader._get_compression_method(
path_to_str(self._tar_xz)),
("tar", "xz"))
def md5Dir(self, d):
md5sum = Command(
"find " + path_to_str(d) +
r' -type f | sed -e "s/.*\///gm" | sort | md5sum').execute()
self.assertNotEqual(self.FNF_MD5, md5sum)
return md5sum
def test_tar_xz_method(self):
self.assertEqual(
self._source_loader._get_compression_method(
path_to_str(self._tar_xz)), ("tar", "xz"))
def test_tar_xz_extract(self):
self._source_loader.load_sources(self._tar_xz, self._tar_extracted)
self.assertTrue(path.isdir(path_to_str(self._tar_extracted)))
self.assertEqual(self.md5Tar(path_to_str(self._tar_xz)),
self.md5Dir(path_to_str(self._tar_extracted)))
self.assertExistInDir(["file1", "file2"], self._tar_extracted)
def write_spec(self):
""" Creates spec file or rewrites old one. """
with open(path_to_str(self.spec_path), 'w') as spec_file:
spec_file.write(str(self.spec))
def tearDown(self):
Command("rm -rf " + path_to_str(self.srpm) + "/mock_logs " +
path_to_str(self.package_builder.temp_dir /
"*.rpm")).execute()
def tearDown(self):
if self._tar:
remove(path_to_str(self._tar))
if path.isdir(path_to_str(self._tar_temp)):
rmtree(path_to_str(self._tar_temp))
