def build_rpm(self, srpm, distro, arch, output_dir): self.build_ret_code = 0 def move_files(output, files): if not output.exists() and not output.is_dir(): Command("mkdir " + path_to_str(output)).execute() for _out in files: try: Command("mv " + path_to_str(_out) + " " + path_to_str(output)).execute() except: pass def check_output(proc): while proc.poll() is None: line = proc.stdout.readline().decode("utf-8") if PackageBuilder.regex.search(line): yield line self.build_ret_code = proc.returncode _ret = list( check_output( subprocess.Popen([ "mock", "--no-clean", "--verbose", "--root", distro + '-' + arch, "--rebuild", path_to_str(srpm), "--resultdir=" + path_to_str(self.temp_dir) ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT))) + list( self.check_logs(self.temp_dir)) move_files(output_dir, self.temp_dir.glob("*.rpm")) self.mock_logs = output_dir / "mock_logs" move_files(self.mock_logs, self.temp_dir.glob("*.log")) raise BuildException(_ret, self.build_ret_code)
def _compute_checksum(sources): if sources.is_dir(): cmd = "find %s -type f -print0 | sort -z | xargs " \ "-0 sha1sum | sha1sum" % path_to_str(sources.resolve()) else: cmd = "sha1sum %s" % path_to_str(sources.resolve()) return Command([cmd]).execute()[:7]
def _compute_checksum(sources): if sources.is_dir(): cmd = "find %s -type f -print0 | sort -z | xargs " \ "-0 sha1sum | sha1sum" % path_to_str(sources.resolve()) else: cmd = "sha1sum %s" % path_to_str(sources.resolve()) return cmd_output([cmd])[:7]
def build_rpm(self, srpm, distro, arch, output_dir): """ builds rpm from source RPM to distro and architecture binary RPM into output_dir """ self.build_ret_code = 0 def check_output(proc): while proc.poll() is None: line = proc.stdout.readline().decode("utf-8") if self._regex.search(line): yield line self.build_ret_code = proc.returncode _ret = list( check_output( subprocess.Popen( [ "mock", "--no-clean", "--verbose", "--root", distro + '-' + arch, "--rebuild", path_to_str(srpm), "--resultdir=" + path_to_str(self.temp_dir) ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) ) ) + list(self._check_logs(self.temp_dir)) self._move_files(output_dir, self.temp_dir.glob("*.rpm")) self.mock_logs = output_dir / "mock_logs" self._move_files(self.mock_logs, self.temp_dir.glob("*.log")) raise BuildException(_ret, self.build_ret_code)
def build_rpm(self, srpm, distro, arch, output_dir): """ builds rpm from source RPM to distro and architecture binary RPM into output_dir """ self.build_ret_code = 0 def check_output(proc): while proc.poll() is None: line = proc.stdout.readline().decode("utf-8") if self._regex.search(line): yield line self.build_ret_code = proc.returncode _ret = list( check_output( subprocess.Popen([ "mock", "--no-clean", "--verbose", "--root", distro + '-' + arch, "--rebuild", path_to_str(srpm), "--resultdir=" + path_to_str(self.temp_dir) ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT))) + list( self._check_logs(self.temp_dir)) self._move_files(output_dir, self.temp_dir.glob("*.rpm")) self.mock_logs = output_dir / "mock_logs" self._move_files(self.mock_logs, self.temp_dir.glob("*.log")) raise BuildException(_ret, self.build_ret_code)
def extraction(self, source, dest): source = path_to_str(source) if is_zipfile(source): Command("unzip {} -d {}".format( source, path_to_str(dest))).execute() return True else: return False
def _compute_checksum(sources): if sources.is_dir(): cmd = "find %s -type f -print0 | sort -z | xargs " \ "-0 sha1sum | sha1sum" % path_to_str(sources.resolve()) else: cmd = "cat %s | sha1sum" % path_to_str(sources.resolve()) logging.error(str(cmd)) return Command([cmd]).execute()[:7]
def extraction(self, source, dest): source = path_to_str(source) if is_tarfile(source): Command("tar xf {} -C {}".format(source, path_to_str(dest))).execute() return True else: return False
def test_command_concat(self): cmd = Command("cd %s" % path_to_str(self.test_project_dir)) cmd.append("cmake ..") cmd.append(["make", "make test"]) self.assertRaises(TypeError, cmd.append, 4) expected = "cd %s\ncmake ..\nmake\nmake test" %\ path_to_str(self.test_project_dir) self.assertEqual(expected, str(cmd))
def create_archive(self): """ Creates archive (archvie_path) from Source folder """ self.spec.Source = self.spec.Name + "-" + self.spec.Version + ".tar.gz" _tar = Command("tar zcf " + path_to_str(self.archive_path) + " -C " + path_to_str(self.extracted_dir) + " . --transform='s/^\./" + self.spec.Name + "-" + self.spec.Version + "/g'") _tar.execute() logging.debug(str(_tar))
def build_srpm(spec_file, tarball, output_dir): """ Builds source rpm from spec and tarball and moves it to the output directory """ Command("rpmdev-setuptree").execute() Command("cp " + path_to_str(tarball) + ' $(rpm --eval "%{_topdir}")/SOURCES').execute() output = Command("rpmbuild -bs " + path_to_str(spec_file)).execute() Command("mv " + path_to_str(output.split()[-1]) + " " + path_to_str(output_dir)).execute()
def move_files(output, files): if not output.exists() and not output.is_dir(): Command("mkdir " + path_to_str(output)).execute() for _out in files: try: Command("mv " + path_to_str(_out) + " " + path_to_str(output)).execute() except: pass
def build_srpm(spec_file, tarball, output_dir): """builds RPM package with build_srpm and build_rpm""" # Build srpm pakcage from given spec_file and tarball Command("rpmdev-setuptree").execute() Command("cp " + path_to_str(tarball) + ' $(rpm --eval "%{_topdir}")/SOURCES').execute() output = Command("rpmbuild -bs " + path_to_str(spec_file)).execute() Command("mv " + path_to_str(output.split()[-1]) + " " + path_to_str(output_dir)).execute()
def test_tar_xz_extract(self): self._source_loader.load_sources( self._tar_xz, self._tar_extracted) self.assertTrue( path.isdir(path_to_str(self._tar_extracted))) self.assertEqual( self.md5Tar(path_to_str(self._tar_xz)), self.md5Dir(path_to_str(self._tar_extracted))) self.assertExistInDir(["file1", "file2"], self._tar_extracted)
def setUp(self): self._source_loader = SourceLoader() self._tar = None self._tar_dir = self.test_project_dir / "archives" self._tar_gz = self.test_project_dir / "archives" / "sample.tar.gz" self._tar_xz = self.test_project_dir / "archives" / "sample.tar.xz" self._tar_temp = Path("/var/tmp/rpg_test/") self._tar_extracted = self._tar_temp / "extracted" self._download = self._tar_temp / "download.tar.gz" if path.isdir(path_to_str(self._tar_temp)): rmtree(path_to_str(self._tar_temp)) makedirs(path_to_str(self._tar_temp)) makedirs(path_to_str(self._tar_extracted))
def build_rpm_recover(self, distro, arch): """ Repeatedly build rpm with mock and finds all build errors. May raise RuntimeError on failed recover. """ def build(): self.build_srpm() self.build_rpm(distro, arch) def analyse(): _files_to_pkgs.installed(self.base_dir, self.spec, self.sack) self.write_spec() _files_to_pkgs = FilesToPkgsPlugin() analyse() while True: try: build() except BuildException as be: if not self._plugin_engine.execute_mock_recover(be.errors): if be.return_code: raise RuntimeError( "Build failed! See logs in '{}'".format( self._package_builder.mock_logs)) break Command("rm -rf {}".format(path_to_str(self.spec_path))).execute() analyse()
def patched(self, project_dir, spec, sack): """ Finds dependencies via makedepend - This is not garanteed to be all of them. Makedepend uses macro preprocessor and if it throws and error makedepend didn't print deps. """ out = Command([ "find " + path_to_str(project_dir) + " -name " + " -o -name ".join( ["'*." + ex + "'" for ex in self.EXT_CPP] ) ]).execute() cc_makedep = "" cc_included_files = [] for _f in out.splitlines(): try: cc_makedep = Command("makedepend -w 1000 " + str(_f) + " -f- 2>/dev/null").execute() except CalledProcessError as e: logging.warn(str(e.cmd) + "\n" + str(e.output)) continue cc_included_files += [ s for s in cc_makedep.split() if (s.startswith("/usr") or s.startswith("/include")) and str(project_dir) not in s] spec.required_files.update(cc_included_files) spec.build_required_files.update(cc_included_files)
def md5Dir(self, d): md5sum = Command( "find " + path_to_str(d) + r' -type f | sed -e "s/.*\///gm" | sort | md5sum' ).execute() self.assertNotEqual(self.FNF_MD5, md5sum) return md5sum
def build_rpm_recover(self, distro, arch): """ Repeatedly build rpm with mock and finds all build errors. May raise RuntimeError on failed recover. """ def build(): self.build_srpm() self.build_rpm(distro, arch) def analyse(): _files_to_pkgs.installed(self.base_dir, self.spec, self.sack) self.write_spec() _files_to_pkgs = FilesToPkgsPlugin() analyse() while True: try: build() except BuildException as be: if not self._plugin_engine.execute_mock_recover(be.errors): if be.return_code: raise RuntimeError( "Build failed! See logs in '{}'" .format(self._package_builder.mock_logs)) break Command("rm -rf {}".format(path_to_str(self.spec_path))).execute() analyse()
def test_command_execute_from(self): cmd = Command("pwd\ncd c 2>/dev/null\npwd") output = cmd.execute_from(self.test_project_dir) path = path_to_str(self.test_project_dir.resolve()) expected = "%s\n%s/c\n" % (path, path) self.assertEqual(expected, output) # doesn't add 'cd work_dir' during execute to command lines cur_dir = Path('.') with self.assertRaises(subprocess.CalledProcessError) as ctx: cmd.execute_from(cur_dir) expected = "Command '['/bin/sh', '-c', 'cd %s "\ "&& pwd && cd c 2>/dev/null && pwd']' "\ "returned non-zero exit status 1"\ % path_to_str(cur_dir.resolve()) self.assertEqual(expected, str(ctx.exception))
def test_command_execute_from(self): cmd = Command("pwd\ncd c 2>/dev/null\npwd") output = cmd.execute(self.test_project_dir) path = path_to_str(self.test_project_dir.resolve()) expected = "%s\n%s/c\n" % (path, path) self.assertEqual(expected, output) # doesn't add 'cd work_dir' during execute to command lines cur_dir = Path('.') with self.assertRaises(subprocess.CalledProcessError) as ctx: cmd.execute(cur_dir) expected = "Command '['/bin/sh', '-c', 'cd %s "\ "&& pwd && cd c 2>/dev/null && pwd']' "\ "returned non-zero exit status 1"\ % path_to_str(cur_dir.resolve()) self.assertEqual(expected, str(ctx.exception))
def execute_from(self, work_dir): """executes command in work_dir (instance of pathlib.Path), can raise CalledProcessError""" cd_workdir = ["cd %s" % path_to_str(work_dir.resolve())] command_lines = self._assign_rpm_variables() + cd_workdir + \ [rpm.expandMacro(x) for x in self._command_lines] return cmd_output(command_lines)
def md5Tar(self, t): mdsum = Command( "tar --list -f " + path_to_str(t) + " 2>/dev/null | " "awk -F/ '{ if($NF != \"\") print $NF }' | " r'sed -e "s/.*\///gm" | sort | md5sum' ).execute() self.assertNotEqual(self.FNF_MD5, mdsum) return mdsum
def load_sources(self, source_path, extraction_dir): """ If source_path is a directory, path tree will be copied to extraction_dir. If it is archive May raise IOError """ logging.debug('load_sources({}, {}) called' .format(str(source_path), str(extraction_dir))) path = path_to_str(source_path) esc_extr_dir = path_to_str(extraction_dir) if isfile(path): compression = self._get_compression_method(path) if not compression: raise IOError("Input source archive '{}' is incompatible!" .format(path)) self._extract(path, esc_extr_dir, compression) self._process_dir(esc_extr_dir) elif isdir(str(source_path)): self._copy_dir(str(source_path), str(extraction_dir)) else: raise IOError("Input source archive/directory '{}' doesn't exists!" .format(path))
def build_rpm(self, srpm, distro, arch, output_dir): self.build_ret_code = 0 def move_files(output, files): if not output.exists() and not output.is_dir(): Command("mkdir " + path_to_str(output)).execute() for _out in files: try: Command("mv " + path_to_str(_out) + " " + path_to_str(output)).execute() except: pass def check_output(proc): while proc.poll() is None: line = proc.stdout.readline().decode("utf-8") if PackageBuilder.regex.search(line): yield line self.build_ret_code = proc.returncode _ret = list( check_output( subprocess.Popen( [ "mock", "--no-clean", "--verbose", "--root", distro + '-' + arch, "--rebuild", path_to_str(srpm), "--resultdir=" + path_to_str(self.temp_dir) ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) ) ) + list(self.check_logs(self.temp_dir)) move_files(output_dir, self.temp_dir.glob("*.rpm")) self.mock_logs = output_dir / "mock_logs" move_files(self.mock_logs, self.temp_dir.glob("*.log")) raise BuildException(_ret, self.build_ret_code)
def load_sources(self, source_path, extraction_dir): """ If source_path is a directory, path tree will be copied to extraction_dir. If it is archive May raise IOError """ logging.debug('load_sources({}, {}) called'.format( str(source_path), str(extraction_dir))) path = path_to_str(source_path) esc_extr_dir = path_to_str(extraction_dir) if isfile(path): compression = self._get_compression_method(path) if not compression: raise IOError( "Input source archive '{}' is incompatible!".format(path)) self._extract(path, esc_extr_dir, compression) self._process_dir(esc_extr_dir) elif isdir(str(source_path)): self._copy_dir(str(source_path), str(extraction_dir)) else: raise IOError( "Input source archive/directory '{}' doesn't exists!".format( path))
def patched(self, project_dir, spec, sack): out = Command([ "find " + path_to_str(project_dir) + " -name " + " -o -name ".join(["'*." + ex + "'" for ex in self.EXT_CPP]) ]).execute() cc_makedep = "" cc_included_files = [] for _f in out.splitlines(): try: cc_makedep = Command("makedepend -w 1000 " + str(_f) + " -f- 2>/dev/null").execute() except CalledProcessError as e: logging.warn(str(e.cmd) + "\n" + str(e.output)) continue cc_included_files += [ s for s in cc_makedep.split() if (s.startswith("/usr") or s.startswith("/include")) and str(project_dir) not in s ] spec.required_files.update(cc_included_files) spec.build_required_files.update(cc_included_files)
def patched(self, project_dir, spec, sack): out = Command([ "find " + path_to_str(project_dir) + " -name " + " -o -name ".join( ["'*." + ex + "'" for ex in self.EXT_CPP] ) ]).execute() cc_makedep = "" cc_included_files = [] for _f in out.splitlines(): try: cc_makedep = Command("makedepend -w 1000 " + str(_f) + " -f- 2>/dev/null").execute() except CalledProcessError as e: logging.warn(str(e.cmd) + "\n" + str(e.output)) continue cc_included_files += [ s for s in cc_makedep.split() if (s.startswith("/usr") or s.startswith("/include")) and str(project_dir) not in s] spec.required_files.update(cc_included_files) spec.build_required_files.update(cc_included_files)
def patched(self, project_dir, spec, sack): """ Finds dependencies via makedepend - This is not garanteed to be all of them. Makedepend uses macro preprocessor and if it throws and error makedepend didn't print deps. """ out = Command([ "find " + path_to_str(project_dir) + " -name " + " -o -name ".join(["'*." + ex + "'" for ex in self.EXT_CPP]) ]).execute() cc_makedep = "" cc_included_files = [] for _f in out.splitlines(): try: cc_makedep = Command("makedepend -w 1000 " + str(_f) + " -f- 2>/dev/null").execute() except CalledProcessError as e: logging.warn(str(e.cmd) + "\n" + str(e.output)) continue cc_included_files += [ s for s in cc_makedep.split() if (s.startswith("/usr") or s.startswith("/include")) and str(project_dir) not in s ] spec.required_files.update(cc_included_files) spec.build_required_files.update(cc_included_files)
def tearDown(self): Command("rm -rf " + path_to_str(self.srpm) + "/mock_logs " + path_to_str(self.package_builder.temp_dir / "*.rpm")).execute()
def write_spec(self): """ Creates spec file or rewrites old one. """ with open(path_to_str(self.spec_path), 'w') as spec_file: spec_file.write(str(self.spec))
def tearDown(self): if self._tar: remove(path_to_str(self._tar)) if path.isdir(path_to_str(self._tar_temp)): rmtree(path_to_str(self._tar_temp))
def md5Tar(self, t): mdsum = Command("tar --list -f " + path_to_str(t) + " 2>/dev/null | " "awk -F/ '{ if($NF != \"\") print $NF }' | " r'sed -e "s/.*\///gm" | sort | md5sum').execute() self.assertNotEqual(self.FNF_MD5, mdsum) return mdsum
def test_tar_xz_method(self): self.assertEqual( self._source_loader._get_compression_method( path_to_str(self._tar_xz)), ("tar", "xz"))
def md5Dir(self, d): md5sum = Command( "find " + path_to_str(d) + r' -type f | sed -e "s/.*\///gm" | sort | md5sum').execute() self.assertNotEqual(self.FNF_MD5, md5sum) return md5sum
def test_tar_xz_extract(self): self._source_loader.load_sources(self._tar_xz, self._tar_extracted) self.assertTrue(path.isdir(path_to_str(self._tar_extracted))) self.assertEqual(self.md5Tar(path_to_str(self._tar_xz)), self.md5Dir(path_to_str(self._tar_extracted))) self.assertExistInDir(["file1", "file2"], self._tar_extracted)