def __init__(self): self.conf = Conf() self._setup_logging() self._project_builder = ProjectBuilder() self.spec = Spec() self.sack = None self._package_builder = PackageBuilder() self._source_loader = SourceLoader()
def __init__(self): self._setup_logging() self._project_builder = ProjectBuilder() self.spec = Spec() self.sack = None # TODO dnf sack self._package_builder = PackageBuilder() self._plugin_engine = PluginEngine(self.spec, self.sack) self._source_loader = SourceLoader() self._copr_uploader = CoprUploader() self._plugin_engine.load_plugins(Path('rpg/plugins'))
def setUp(self): self._source_loader = SourceLoader() self._tar = None self._tar_dir = self.test_project_dir / "archives" self._tar_gz = self.test_project_dir / "archives" / "sample.tar.gz" self._tar_xz = self.test_project_dir / "archives" / "sample.tar.xz" self._tar_temp = Path("/var/tmp/rpg_test/") self._tar_extracted = self._tar_temp / "extracted" self._download = self._tar_temp / "download.tar.gz" if path.isdir(path_to_str(self._tar_temp)): rmtree(path_to_str(self._tar_temp)) makedirs(path_to_str(self._tar_temp)) makedirs(path_to_str(self._tar_extracted))
def setUp(self): self._source_loader = SourceLoader() self._tar_temp = Path("/var/tmp/rpg_test/") self._download = self._tar_temp / "download.tar.gz" if path.isdir(str(self._tar_temp)): rmtree(str(self._tar_temp)) makedirs(str(self._tar_temp))
def __init__(self): self.conf = Conf() self._setup_logging() self._project_builder = ProjectBuilder() self.spec = Spec() self.sack = None # TODO dnf sack self._package_builder = PackageBuilder() self._plugin_engine = PluginEngine(self.spec, self.sack) self._source_loader = SourceLoader() self._copr_uploader = CoprUploader()
def setUp(self): self._source_loader = SourceLoader() self._tar = None self._tar_dir = self.test_project_dir / "archives" self._tar_gz = self.test_project_dir / "archives" / "sample.tar.gz" self._tar_xz = self.test_project_dir / "archives" / "sample.tar.xz" self._tar_temp = Path("/var/tmp/rpg_test/") self._tar_extracted = self._tar_temp / "extracted" self._download = self._tar_temp / "download.tar.gz" if path.isdir(str(self._tar_temp)): rmtree(str(self._tar_temp)) makedirs(str(self._tar_temp)) makedirs(str(self._tar_extracted))
def setUp(self): self._source_loader = SourceLoader() self._tar = False self._tar_dir = self.test_project_dir self._tar_gz = self.test_project_dir / "archives" / "sample.tar.gz" self._tar_xz = self.test_project_dir / "archives" / "sample.tar.xz" self._tar_temp = "/var/tmp/rpg_test/" self._tar_extracted = self._tar_temp + "extracted" self._archive = self._tar_temp + "sample" self._hasher = None if path.isdir(self._tar_extracted): rmtree(self._tar_extracted) if path.isdir(self._tar_temp): rmtree(self._tar_temp) makedirs(self._tar_temp) makedirs(self._tar_extracted)
class Base(object): """Base class that is controlled by RPM GUI""" def __init__(self): self.conf = Conf() self._setup_logging() self._project_builder = ProjectBuilder() self.spec = Spec() self.sack = None self._package_builder = PackageBuilder() self._source_loader = SourceLoader() def dnf_load_sack(self): logging.info('DNF sack is loading') import dnf with dnf.Base() as self._dnf_base: self._dnf_base.conf.releasever = dnf.rpm.detect_releasever( self._dnf_base.conf.installroot) self._dnf_base.read_all_repos() self._dnf_base.fill_sack() return self._dnf_base.sack def _setup_logging(self): if geteuid() == 0: log_dir = "/var/log/rpg/" else: log_dir = "/var/tmp/rpg/" if not isdir(log_dir): makedirs(log_dir) logging.basicConfig(level=logging.DEBUG, format='[%(asctime)s] {%(pathname)s:%(lineno)d} ' '%(levelname)s - %(message)s', handlers=[logging.FileHandler(log_dir + "rpg.log"), logging.StreamHandler()], datefmt='%H:%M:%S') def load_plugins(self): self._plugin_engine = PluginEngine(self.spec, self.sack) self._plugin_engine.load_plugins( Path('rpg/plugins'), self.conf.exclude) for directory in self.conf.directories: self._plugin_engine.load_plugins( Path(directory), self.conf.exclude) def create_archive(self): """ Creates archive (archvie_path) from Source folder """ self.spec.Source = self.spec.Name + "-" + self.spec.Version + ".tar.gz" _tar = Command("tar zcf " + str(self.archive_path) + " -C " + str(self.extracted_dir) + " . --transform='s/^\./" + self.spec.Name + "-" + self.spec.Version + "/g'") _tar.execute() logging.debug(str(_tar)) @property def base_dir(self): try: return Path("/tmp/rpg-%s-%s" % (self._input_name, self._hash)) except AttributeError: msg = "`load_project_from_url` method needs to be called first" raise RuntimeError(msg) @property def extracted_dir(self): return self.base_dir / "extracted" @property def compiled_dir(self): return self.base_dir / "compiled" @property def installed_dir(self): return self.base_dir / "installed" @property def project_name(self): return self.spec.Name @property def spec_path(self): return self.base_dir / (self.project_name + ".spec") @property def archive_path(self): return self.base_dir / self.spec.Source @property def srpm_path(self): try: return next(self.base_dir.glob(self.project_name + "*src.rpm")) except StopIteration: raise RuntimeError( "Can't find '{}'! You need to call build_srpm first." .format(str(self.base_dir / (self.project_name + "*src.rpm")))) def load_project_from_url(self, path): """executed in background after dir/tarball/SRPM selection""" path = Path(path) temp_arch = "downloaded_archive.tar.gz" if search(r"github\.com/[^/]+/[^/]+/?$", str(path)): self._source_loader.download_git_repo(str(path), temp_arch) elif str(path).startswith("http"): self._source_loader.download_archive(str(path), temp_arch) else: temp_arch = None self._hash = self.compute_checksum(path) self._input_name = path.name self._setup_workspace() self._source_loader.load_sources(path, self.extracted_dir) self.spec.prep = Command("%autosetup") if temp_arch: remove(temp_arch) def run_raw_sources_analysis(self): """executed in background after dir/tarball/SRPM selection""" self._plugin_engine.execute_phase(phases[0], self.extracted_dir) def apply_patches(self, ordered_patches): """executed in background after patch selection and reordering""" self._project_builder.apply_patches(ordered_patches) def run_patched_sources_analysis(self): """executed in background after patches are applied""" self._plugin_engine.execute_phase(phases[1], self.extracted_dir) def run_compiled_analysis(self): """executed in background after patches are applied""" self._plugin_engine.execute_phase(phases[2], self.extracted_dir) def install_project(self): """executed in background after filled requires screen""" self._project_builder.install(self.compiled_dir, self.installed_dir, self.spec.install) def run_installed_analysis(self): """executed in background after successful project build""" self._plugin_engine.execute_phase(phases[3], self.installed_dir) def write_spec(self): with open(str(self.spec_path), 'w') as spec_file: spec_file.write(str(self.spec)) def build_srpm(self): if not self.spec.Source or not self.archive_path.exists(): self.create_archive() self.write_spec() self._package_builder.build_srpm( self.spec_path, self.archive_path, self.base_dir) def build_rpm(self, target_distro, target_arch): self._package_builder.build_rpm(str(self.srpm_path), target_distro, target_arch) def fetch_repos(self, dist, arch): self._package_builder.fetch_repos(dist, arch) def build_project(self): """executed in background after filled requires screen""" self._project_builder.build(self.extracted_dir, self.compiled_dir, self.spec.build) def copr_set_config(self, username, login, token): self.cl = CoprClient( username, login, token, copr_url="http://copr.fedoraproject.org") def copr_create_project(self, name, chroots, desc, intro): self.cl.create_project( name, chroots=chroots, description=desc, instructions=intro) def copr_build(self, name, url): self.cl.create_new_build(name, pkgs=[url, ]) @staticmethod def compute_checksum(sources): if sources.is_dir(): cmd = "find %s -type f -print0 | sort -z | xargs " \ "-0 sha1sum | sha1sum" % sources.resolve() else: cmd = "sha1sum %s" % sources.resolve() return cmd_output([cmd])[:7] @property def all_dirs(self): return [ self.extracted_dir, self.compiled_dir, self.installed_dir ] def _setup_workspace(self): """make sure all directories used later will exist""" shutil.rmtree(str(self.base_dir), True) for d in self.all_dirs: d.mkdir(parents=True) # predictor methods are used for autocompletion of the field, # every guess_* method return list of strings matched ordered # by their rank def guess_name(self): name = str(self._input_name) if isdir(name): return name else: if name[-4:] == ".zip": return name[:-4] else: if "tar" in name: return name.split(".tar")[0] return "" def guess_provide(self): # returns list of all known provides provides = set() for pkg in self.sack.query(): provides.update(pkg.provides) return sorted(provides) def guess_changelog_data(self): # returns list of tuples (author, email) from git pass def guess_dependency(self): # returns guess_provide() + all package names from repos names = map(lambda pkg: pkg.name, self.sack.query()) return sorted(set(names).union(set(self.guess_provide()))) def guess_license(self): # returns list of all known licenses licenses = set() for pkg in self.sack.query(): licenses.update(pkg.license) return sorted(licenses) def build_rpm_recover(self, distro, arch): def build(): self.build_srpm() return self._package_builder.build_rpm( self.srpm_path, distro, arch) + list(self._package_builder.check_logs()) _files_to_pkgs = FilesToPkgsPlugin() while True: _file = "" _files_to_pkgs.installed(self.base_dir, self.spec, self.sack) self.write_spec() for err in build(): match = search( r"DEBUG\:\s*[^:]+\:\s*([^:]+)\:\s*" + r"[cC][oO][mM][mM][aA][nN][dD]\s*[nN][oO][tT]\s*" + r"[fF][oO][uU][nN][dD]", err) if match: if match.group(1) == _file or\ "/usr/bin/" + match.group(1) == _file: logging.info("Couldn't resolve '{}'!" .format(match.group(1))) return else: if "/" in match.group(1): _file = match.group(1) else: _file = "/usr/bin/" + match.group(1) break if _file == "": break self.spec.required_files.add(_file) self.spec.build_required_files.add(_file) Command("rm -rf " + str(self._package_builder.temp_dir) + "/*.log")
class Base(object): """Base class that is controlled by RPM GUI""" def __init__(self): self.conf = Conf() self._setup_logging() self._project_builder = ProjectBuilder() self.spec = Spec() self.sack = None self._package_builder = PackageBuilder() self._source_loader = SourceLoader() self._copr_uploader = CoprUploader() def dnf_load_sack(self): logging.info('DNF sack is loading') import dnf with dnf.Base() as self._dnf_base: self._dnf_base.conf.releasever = dnf.rpm.detect_releasever( self._dnf_base.conf.installroot) self._dnf_base.read_all_repos() self._dnf_base.fill_sack() return self._dnf_base.sack def _setup_logging(self): if geteuid() == 0: log_dir = "/var/log/rpg/" else: log_dir = "/var/tmp/rpg/" if not isdir(log_dir): makedirs(log_dir) logging.basicConfig(level=logging.DEBUG, format='[%(asctime)s] {%(pathname)s:%(lineno)d} ' '%(levelname)s - %(message)s', handlers=[logging.FileHandler(log_dir + "rpg.log"), logging.StreamHandler()], datefmt='%H:%M:%S') def load_plugins(self): self._plugin_engine = PluginEngine(self.spec, self.sack) self._plugin_engine.load_plugins( Path('rpg/plugins'), self.conf.exclude) for directory in self.conf.directories: self._plugin_engine.load_plugins( Path(directory), self.conf.exclude) @property def base_dir(self): try: return Path("/tmp/rpg-%s-%s" % (self._input_name, self._hash)) except AttributeError: msg = "`process_archive_or_dir` method needs to be called first" raise RuntimeError(msg) @property def extracted_dir(self): return self.base_dir / "extracted" @property def compiled_dir(self): return self.base_dir / "compiled" @property def installed_dir(self): return self.base_dir / "installed" @property def project_name(self): return self.spec.Name @property def spec_path(self): return self.base_dir / (self.project_name + ".spec") @property def archive_path(self): return self.base_dir / self.spec.Source @property def srpm_path(self): return next(self.base_dir.glob(self.project_name + "*src.rpm")) def process_archive_or_dir(self, path): """executed in background after dir/tarball/SRPM selection""" p = Path(path) self._hash = self.compute_checksum(p) self._input_name = p.name self._setup_workspace() self._source_loader.load_sources(p, self.extracted_dir) def run_raw_sources_analysis(self): """executed in background after dir/tarball/SRPM selection""" self._plugin_engine.execute_phase(phases[0], self.extracted_dir) def apply_patches(self, ordered_patches): """executed in background after patch selection and reordering""" self._project_builder.apply_patches(ordered_patches) def run_patched_sources_analysis(self): """executed in background after patches are applied""" self._plugin_engine.execute_phase(phases[1], self.extracted_dir) def build_project(self): """executed in background after filled requires screen""" self._project_builder.build(self.extracted_dir, self.compiled_dir, self.spec.build) def run_compiled_analysis(self): """executed in background after patches are applied""" self._plugin_engine.execute_phase(phases[2], self.extracted_dir) def install_project(self): """executed in background after filled requires screen""" self._project_builder.install(self.compiled_dir, self.installed_dir, self.spec.install) def run_installed_analysis(self): """executed in background after successful project build""" self._plugin_engine.execute_phase(phases[3], self.installed_dir) def write_spec(self): with open(str(self.spec_path), 'w') as spec_file: spec_file.write(str(self.spec)) def build_srpm(self): if not self.spec_path.exists(): self.write_spec() self._package_builder.build_srpm( self.spec_path, self.archive_path, self.base_dir) def build_packages(self, distros, archs=platform.machine()): """builds packages for desired distributions""" for arch in archs: for distro in distros: self._package_builder.build(self.spec_path, self.archive_path, distro, arch) @staticmethod def compute_checksum(sources): if sources.is_dir(): cmd = "find %s -type f -print0 | sort -z | xargs " \ "-0 sha1sum | sha1sum" % sources.resolve() else: cmd = "sha1sum %s" % sources.resolve() return cmd_output([cmd])[:7] @property def all_dirs(self): return [ self.extracted_dir, self.compiled_dir, self.installed_dir ] def _setup_workspace(self): """make sure all directories used later will exist""" try: shutil.rmtree(str(self.base_dir)) except FileNotFoundError: pass for d in self.all_dirs: d.mkdir(parents=True) # predictor methods are used for autocompletion of the field, # every guess_* method return list of strings matched ordered # by their rank def guess_name(self): name = str(self._input_name) if isdir(name): return name else: if name[-4:] == ".zip": return name[:-4] else: if "tar" in name: return name.split(".tar")[0] return "" def guess_provide(self): # returns list of all known provides provides = set() for pkg in self.sack.query(): provides.update(pkg.provides) return sorted(provides) def guess_changelog_data(self): # returns list of tuples (author, email) from git pass def guess_dependency(self): # returns guess_provide() + all package names from repos names = map(lambda pkg: pkg.name, self.sack.query()) return sorted(set(names).union(set(self.guess_provide()))) def guess_license(self): # returns list of all known licenses licenses = set() for pkg in self.sack.query(): licenses.update(pkg.license) return sorted(licenses)
class SourceLoaderTest(RpgTestCase): # FileNotFound hash FNF_MD5 = "d41d8cd98f00b204e9800998ecf8427e" def setUp(self): self._source_loader = SourceLoader() self._tar = False self._tar_dir = self.test_project_dir self._tar_gz = self.test_project_dir / "archives" / "sample.tar.gz" self._tar_xz = self.test_project_dir / "archives" / "sample.tar.xz" self._tar_temp = "/var/tmp/rpg_test/" self._tar_extracted = self._tar_temp + "extracted" self._archive = self._tar_temp + "sample" self._hasher = None if path.isdir(self._tar_extracted): rmtree(self._tar_extracted) if path.isdir(self._tar_temp): rmtree(self._tar_temp) makedirs(self._tar_temp) makedirs(self._tar_extracted) def tearDown(self): if self._tar: remove(str(self._tar)) def md5TarXz(self, t): mdsum = Command( "tar -J -xOf " + str(t) + " 2>/dev/null | md5sum | cut -b-32" ).execute()[:-1] self.assertNotEqual(self.FNF_MD5, mdsum) return mdsum def md5TarGz(self, t): mdsum = Command( "tar -xOzf " + str(t) + " 2>/dev/null | md5sum | cut -b-32" ).execute()[:-1] self.assertNotEqual(self.FNF_MD5, mdsum) return mdsum def md5Dir(self, d): mdsum = md5( Command( "find " + d + r" -type f -exec cat {} \;" ).execute(True)).hexdigest() self.assertNotEqual(self.FNF_MD5, mdsum) return mdsum def test_tar_gz_method(self): self.assertEqual( self._source_loader.get_compression_method( str(self._tar_gz)), ("tar", "gz")) def test_tar_xz_method(self): self.assertEqual( self._source_loader.get_compression_method( str(self._tar_xz)), ("tar", "xz")) def test_tar_gz_extract(self): self._source_loader.load_sources( self._tar_gz, self._tar_extracted) self.assertTrue( path.isdir(str(self._tar_extracted))) self.assertEqual( self.md5TarGz(str(self._tar_gz)), self.md5Dir(str(self._tar_extracted))) def test_tar_xz_extract(self): self._source_loader.load_sources( self._tar_xz, self._tar_extracted) self.assertTrue( path.isdir(str(self._tar_extracted))) self.assertEqual( self.md5TarXz(str(self._tar_xz)), self.md5Dir(str(self._tar_extracted))) def test_dir_source_loader(self): self._source_loader.load_sources( str(self.test_project_dir), self._tar_extracted) self.assertEqual( self.md5Dir(self._tar_extracted), self.md5Dir(str(self.test_project_dir))) def test_create_archive(self): self._tar = self._source_loader.create_archive( str(self._archive), str(self._tar_dir)) self.assertEqual( self.md5TarGz(str(self._tar)), self.md5Dir(str(self._tar_dir))) @expectedFailure def test_tar_xz_method_fail(self): self._tar_xz = self._tar_gz self.test_tar_xz_method() @expectedFailure def test_tar_gz_method_fail(self): self._tar_gz = self._tar_xz self.test_tar_gz_method() @expectedFailure def test_tar_gz_extract_fail(self): self._tar_gz = "NotAnArchive" self.test_tar_gz_extract() @expectedFailure def test_tar_xz_extract_fail(self): self._tar_xz = "NotAnArchive" self.test_tar_xz_extract() @expectedFailure def test_create_archive_fail(self): self._tar_dir = "NonExistingDir" self.test_create_archive()
class SourceLoaderTest(RpgTestCase): # FileNotFound hash FNF_MD5 = "d41d8cd98f00b204e9800998ecf8427e" def setUp(self): self._source_loader = SourceLoader() self._tar = None self._tar_dir = self.test_project_dir / "archives" self._tar_gz = self.test_project_dir / "archives" / "sample.tar.gz" self._tar_xz = self.test_project_dir / "archives" / "sample.tar.xz" self._tar_temp = Path("/var/tmp/rpg_test/") self._tar_extracted = self._tar_temp / "extracted" self._download = self._tar_temp / "download.tar.gz" if path.isdir(path_to_str(self._tar_temp)): rmtree(path_to_str(self._tar_temp)) makedirs(path_to_str(self._tar_temp)) makedirs(path_to_str(self._tar_extracted)) def tearDown(self): if self._tar: remove(path_to_str(self._tar)) if path.isdir(path_to_str(self._tar_temp)): rmtree(path_to_str(self._tar_temp)) def md5Tar(self, t): mdsum = Command("tar --list -f " + path_to_str(t) + " 2>/dev/null | " "awk -F/ '{ if($NF != \"\") print $NF }' | " r'sed -e "s/.*\///gm" | sort | md5sum').execute() self.assertNotEqual(self.FNF_MD5, mdsum) return mdsum def md5Dir(self, d): md5sum = Command( "find " + path_to_str(d) + r' -type f | sed -e "s/.*\///gm" | sort | md5sum').execute() self.assertNotEqual(self.FNF_MD5, md5sum) return md5sum def test_tar_gz_method(self): self.assertEqual( self._source_loader._get_compression_method( path_to_str(self._tar_gz)), ("tar", "gz")) def test_tar_xz_method(self): self.assertEqual( self._source_loader._get_compression_method( path_to_str(self._tar_xz)), ("tar", "xz")) def test_tar_gz_extract(self): self._source_loader.load_sources(self._tar_gz, self._tar_extracted) self.assertTrue(path.isdir(path_to_str(self._tar_extracted))) self.assertEqual(self.md5Tar(path_to_str(self._tar_gz)), self.md5Dir(path_to_str(self._tar_extracted))) self.assertExistInDir(["file"], self._tar_extracted) def test_tar_xz_extract(self): self._source_loader.load_sources(self._tar_xz, self._tar_extracted) self.assertTrue(path.isdir(path_to_str(self._tar_extracted))) self.assertEqual(self.md5Tar(path_to_str(self._tar_xz)), self.md5Dir(path_to_str(self._tar_extracted))) self.assertExistInDir(["file1", "file2"], self._tar_extracted) def test_dir_source_loader(self): self._source_loader.load_sources(self._tar_dir, self._tar_extracted) self.assertEqual(self.md5Dir(self._tar_extracted), self.md5Dir(self._tar_dir)) self.assertExistInDir(["sample.tar.gz", "sample.tar.xz"], self._tar_extracted) def test_extensions(self): self.assertEqual( ("zip", None), self._source_loader._get_compression_method("sample.zip")) self.assertEqual( ("zip", None), self._source_loader._get_compression_method("sample.sample.zip")) self.assertEqual( ("zip", None), self._source_loader._get_compression_method("sample.tar.zip")) self.assertEqual( ("tar", "xz"), self._source_loader._get_compression_method("sample.tar.xz")) @expectedFailure def test_extensions_fail(self): self._source_loader._get_compression_method("sample.tar.zip") self._source_loader._get_compression_method("sample.xz.tar") self._source_loader._get_compression_method("sample.tar.") self._source_loader._get_compression_method("sample.zip.xz") @expectedFailure def test_tar_xz_method_fail(self): self._tar_xz = self._tar_gz self.test_tar_xz_method() @expectedFailure def test_tar_gz_method_fail(self): self._tar_gz = self._tar_xz self.test_tar_gz_method() @expectedFailure def test_tar_gz_extract_fail(self): self._tar_gz = "NotAnArchive" self.test_tar_gz_extract() @expectedFailure def test_tar_xz_extract_fail(self): self._tar_xz = "NotAnArchive" self.test_tar_xz_extract()
class Base(object): """Base class that is controlled by RPM GUI""" def __init__(self): self._setup_logging() self._project_builder = ProjectBuilder() self.spec = Spec() self.sack = None # TODO dnf sack self._package_builder = PackageBuilder() self._plugin_engine = PluginEngine(self.spec, self.sack) self._source_loader = SourceLoader() self._copr_uploader = CoprUploader() self._plugin_engine.load_plugins(Path('rpg/plugins')) def _setup_logging(self): logging.basicConfig(level=logging.DEBUG, format='[%(asctime)s] {%(pathname)s:%(lineno)d} ' '%(levelname)s - %(message)s', handlers=[logging.FileHandler("rpg.log"), logging.StreamHandler()], datefmt='%H:%M:%S') @property def base_dir(self): try: return Path("/tmp/rpg-%s-%s" % (self._input_name, self._hash)) except AttributeError: msg = "`process_archive_or_dir` method needs to be called first" raise RuntimeError(msg) @property def extracted_dir(self): return self.base_dir / "extracted" @property def compiled_dir(self): return self.base_dir / "compiled" @property def installed_dir(self): return self.base_dir / "installed" @property def project_name(self): return self.spec.tags["name"] @property def spec_path(self): return self.base_dir / (self.project_name + ".spec") @property def tarball_path(self): return self.base_dir / (self.project_name + ".tar.gz") @property def rpm_path(self): return next(self.base_dir.glob(self.project_name + "*.rpm")) def process_archive_or_dir(self, path): """executed in background after dir/tarball/SRPM selection""" p = Path(path) self._hash = self.compute_checksum(p) self._input_name = p.name self.setup_workspace() self._source_loader.load_sources(p, self.extracted_dir) def run_raw_sources_analysis(self): """executed in background after dir/tarball/SRPM selection""" self._plugin_engine.execute_phase(phases[0], self.extracted_dir) def apply_patches(self, ordered_patches): """executed in background after patch selection and reordering""" self._project_builder.apply_patches(ordered_patches) def run_patched_sources_analysis(self): """executed in background after patches are applied""" self._plugin_engine.execute_phase(phases[1], self.extracted_dir) def build_project(self): """executed in background after filled requires screen""" self._project_builder.build(self.extracted_dir, self.compiled_dir, self.spec.scripts["%build"]) def run_compiled_analysis(self): """executed in background after patches are applied""" self._plugin_engine.execute_phase(phases[2], self.extracted_dir) def install_project(self): """executed in background after filled requires screen""" self._project_builder.install(self.compiled_dir, self.installed_dir, self.spec.scripts["%install"]) def run_installed_analysis(self): """executed in background after successful project build""" self._plugin_engine.execute_phase(phases[3], self.compiled_dir) def build_packages(self, *distros): """builds packages for desired distributions""" for distro in distros: self._package_builder.build(self.spec_path, self.tarball_path, self.distro) @staticmethod def compute_checksum(sources): if sources.is_dir(): cmd = "find %s -type f -print0 | sort -z | xargs " \ "-0 sha1sum | sha1sum" % sources.resolve() else: cmd = "sha1sum %s" % sources.resolve() return cmd_output([cmd])[:7] @property def all_dirs(self): return [ self.extracted_dir, self.compiled_dir, self.installed_dir ] def setup_workspace(self): """make sure all directories used later will exist""" try: shutil.rmtree(str(self.base_dir)) except FileNotFoundError: pass for d in self.all_dirs: d.mkdir(parents=True) # predictor methods are used for autocompletion of the field, # every guess_* method return list of strings matched ordered # by their rank def guess_name(self): pass def guess_provide(self): pass def guess_group(self): pass def guess_chagelog_data(self): # returns list of tuples (author, email) pass def guess_build_dependency(self): pass def guess_dependency(self): pass def guess_license(self): pass
class Base(object): """Base class that is controlled by RPM GUI :Example: >>> from rpg import Base >>> base = Base() >>> base.sack = base.load_dnf_sack() >>> base.load_plugins() >>> base.load_project_from_url("https://github.com/example/ex_repo") >>> base.spec.Name = "Example" >>> base.spec.Version = "0.6.11" >>> base.spec.Release = "1%{?snapshot}%{?dist}" >>> base.spec.License = "GPLv2" >>> base.spec.Summary = "Example ..." >>> base.spec.description = ("Example ...") >>> base.spec.URL = "https://github.com/example/ex_repo" >>> base.target_arch = "x86_64" >>> base.target_distro = "fedora-22" >>> base.fetch_repos(base.target_distro, base.target_arch) >>> base.run_extracted_source_analysis() >>> base.run_patched_source_analysis() >>> base.build_project() >>> base.run_compiled_source_analysis() >>> base.install_project() >>> base.run_installed_source_analysis() >>> base.build_srpm() >>> base.build_rpm_recover(self.base.target_distro, self.base.target_arch) """ def __init__(self): self.conf = Conf() self._setup_logging() self._project_builder = ProjectBuilder() self.spec = Spec() self.sack = None self._package_builder = PackageBuilder() self._source_loader = SourceLoader() def load_dnf_sack(self): logging.info('DNF sack is loading') import dnf with dnf.Base() as self._dnf_base: self._dnf_base.conf.releasever = dnf.rpm.detect_releasever( self._dnf_base.conf.installroot) self._dnf_base.read_all_repos() self._dnf_base.fill_sack() return self._dnf_base.sack def _setup_logging(self): if geteuid() == 0: log_dir = "/var/log/rpg/" else: log_dir = "/var/tmp/rpg/" if not isdir(log_dir): makedirs(log_dir) logging.basicConfig(level=logging.DEBUG, format='[%(asctime)s] {%(pathname)s:%(lineno)d} ' '%(levelname)s - %(message)s', handlers=[logging.FileHandler(log_dir + "rpg.log"), logging.StreamHandler()], datefmt='%H:%M:%S') def load_plugins(self): """ This method sets up plugin engine and loads them """ self._plugin_engine = PluginEngine(self.spec, self.sack) self._plugin_engine.load_plugins( Path('rpg/plugins'), self.conf.exclude) for directory in self.conf.directories: self._plugin_engine.load_plugins( Path(directory), self.conf.exclude) def create_archive(self): """ Creates archive (archvie_path) from Source folder """ self.spec.Source = self.spec.Name + "-" + self.spec.Version + ".tar.gz" _tar = Command("tar zcf " + path_to_str(self.archive_path) + " -C " + path_to_str(self.extracted_dir) + " . --transform='s/^\./" + self.spec.Name + "-" + self.spec.Version + "/g'") _tar.execute() logging.debug(str(_tar)) @property def base_dir(self): """ Returns path where compiled, extracted, installed directories are """ try: return Path("/tmp/rpg-%s-%s" % (self._input_name, self._hash)) except AttributeError: msg = "`load_project_from_url` method needs to be called first" raise RuntimeError(msg) @property def extracted_dir(self): return self.base_dir / "extracted" @property def compiled_dir(self): return self.base_dir / "compiled" @property def installed_dir(self): return self.base_dir / "installed" @property def project_name(self): return self.spec.Name @property def spec_path(self): return self.base_dir / (self.project_name + ".spec") @property def archive_path(self): return self.base_dir / self.spec.Source @property def srpm_path(self): """ Returns path to SRPM only, if it is created. You have to build srpm first. """ try: return next(self.base_dir.glob(self.project_name + "*.src.rpm")) except StopIteration: raise RuntimeError( "Can't find '{}'! You need to call build_srpm first." .format(str(self.base_dir / (self.project_name + "*.src.rpm")))) @property def rpm_path(self): """ This is the same as it is in srpm_path. But this returns list of rpms - there may be severals rpm packages like debuginfo, binary rpm and so on. """ try: _ret = [ _path for _path in self.base_dir.glob(self.project_name + "*.rpm") if not str(_path).endswith(".src.rpm") ] if not _ret: raise StopIteration return _ret except StopIteration: raise RuntimeError( "Can't find '{}'! You need to call build_rpm first." .format(str(self.base_dir / (self.project_name + "*.rpm")))) def load_project_from_url(self, path): """executed in background after dir/tarball/SRPM selection""" temp_arch = "downloaded_archive.tar.gz" if search(r"github\.com/[^/]+/[^/]+/?$", str(path)): self._source_loader.download_git_repo(path, temp_arch) path = Path(temp_arch) elif str(path).startswith("http"): temp_arch = search(r"([^/]+\.[^/]+(?:\.[^/]+)?)$", str(path))\ .group(0) self._source_loader.download_archive(path, temp_arch) path = Path(temp_arch) else: temp_arch = None path = Path(path) self._hash = self._compute_checksum(path) self._input_name = path.name self._setup_workspace() self._source_loader.load_sources(path, self.extracted_dir) self.spec.prep = Command("%autosetup") if temp_arch: remove(temp_arch) def run_extracted_source_analysis(self): """executed in background after dir/tarball/SRPM selection""" self._plugin_engine.execute_phase(PluginEngine.phases[0], self.extracted_dir) def run_patched_source_analysis(self): """executed in background after patches are applied""" self._plugin_engine.execute_phase(PluginEngine.phases[1], self.extracted_dir) def run_compiled_source_analysis(self): """executed in background after patches are applied""" self._plugin_engine.execute_phase(PluginEngine.phases[2], self.compiled_dir) def install_project(self): """executed in background after filled requires screen""" self._project_builder.install(self.compiled_dir, self.installed_dir, self.spec.install) def run_installed_source_analysis(self): """executed in background after successful project build""" self._plugin_engine.execute_phase(PluginEngine.phases[3], self.installed_dir) def write_spec(self): """ Creates spec file or rewrites old one. """ with open(str(self.spec_path), 'w') as spec_file: spec_file.write(str(self.spec)) def build_srpm(self): """ Builds srpm into base directory. """ if not self.spec.Source or not self.archive_path.exists(): self.create_archive() self.write_spec() self._package_builder.build_srpm( self.spec_path, self.archive_path, self.base_dir) def build_rpm(self, target_distro, target_arch): """ Build rpm from srpm. If srpm does not exists, it will be created. """ try: self.srpm_path except RuntimeError: self.build_srpm() self._package_builder.build_rpm( str(self.srpm_path), target_distro, target_arch, self.base_dir) def build_rpm_recover(self, distro, arch): """ Repeatedly build rpm with mock and finds all build errors. May raise RuntimeError on failed recover. """ def build(): self.build_srpm() self.build_rpm(distro, arch) def analyse(): _files_to_pkgs.installed(self.base_dir, self.spec, self.sack) self.write_spec() _files_to_pkgs = FilesToPkgsPlugin() analyse() while True: try: build() except BuildException as be: if not self._plugin_engine.execute_mock_recover(be.errors): if be.return_code: raise RuntimeError( "Build failed! See logs in '{}'" .format(self._package_builder.mock_logs)) break analyse() def fetch_repos(self, dist, arch): """ Initialize mock - should be called before build_rpm_recover """ self._package_builder.fetch_repos(dist, arch) def build_project(self): """ Executed in background after filled requires screen """ self._project_builder.build(self.extracted_dir, self.compiled_dir, self.spec.build) def copr_set_config(self, username, login, token): """ Logs into copr with username, login and token. This has to be called before copr_create_project and copr_build To sign up on copr go here: http://copr.fedoraproject.org """ self.cl = CoprClient( username, login, token, copr_url="http://copr.fedoraproject.org") def copr_create_project(self, name, chroots, desc, intro): """ Creates metadata about project - won't build until copr_build would be called """ self.cl.create_project( name, chroots=chroots, description=desc, instructions=intro) def copr_build(self, name, url): """ Builds project on fedora copr server """ self.cl.create_new_build(name, pkgs=[url, ]) @staticmethod def _compute_checksum(sources): if sources.is_dir(): cmd = "find %s -type f -print0 | sort -z | xargs " \ "-0 sha1sum | sha1sum" % path_to_str(sources.resolve()) else: cmd = "sha1sum %s" % path_to_str(sources.resolve()) return Command([cmd]).execute()[:7] @property def all_dirs(self): """ Returns Extracted, Compiled and Installed direcotry paths. """ return [ self.extracted_dir, self.compiled_dir, self.installed_dir ] def _setup_workspace(self): """make sure all directories used later will exist""" shutil.rmtree(str(self.base_dir), True) for d in self.all_dirs: d.mkdir(parents=True) # predictor methods are used for autocompletion of the field, # every guess_* method return list of strings matched ordered # by their rank def guess_name(self): """ Returns guessed name from source path """ suffixes = [".zip", ".tar", ".rar", ".tgz", ".lzma"] name = str(self._input_name) if isdir(name): return basename(name) else: for suffix in suffixes: if suffix in name: return suffix.join(name.split(suffix)[:-1]) return "" def guess_provide(self): """ returns list of all known provides """ provides = set() for pkg in self.sack.query(): provides.update(pkg.provides) return sorted(provides) def guess_changelog_data(self): """ returns list of tuples (author, email) from git """ pass def guess_dependency(self): """ returns guess_provide() + all package names from repos """ names = map(lambda pkg: pkg.name, self.sack.query()) return sorted(set(names).union(set(self.guess_provide()))) def guess_license(self): """ returns list of all known licenses """ licenses = set() for pkg in self.sack.query(): licenses.update(pkg.license) return sorted(licenses)
class Base(object): """Base class that is controlled by RPM GUI""" def __init__(self): self.conf = Conf() self._setup_logging() self._project_builder = ProjectBuilder() self.spec = Spec() self.sack = None self._package_builder = PackageBuilder() self._source_loader = SourceLoader() def load_dnf_sack(self): logging.info('DNF sack is loading') import dnf with dnf.Base() as self._dnf_base: self._dnf_base.conf.releasever = dnf.rpm.detect_releasever( self._dnf_base.conf.installroot) self._dnf_base.read_all_repos() self._dnf_base.fill_sack() return self._dnf_base.sack def _setup_logging(self): if geteuid() == 0: log_dir = "/var/log/rpg/" else: log_dir = "/var/tmp/rpg/" if not isdir(log_dir): makedirs(log_dir) logging.basicConfig(level=logging.DEBUG, format='[%(asctime)s] {%(pathname)s:%(lineno)d} ' '%(levelname)s - %(message)s', handlers=[ logging.FileHandler(log_dir + "rpg.log"), logging.StreamHandler() ], datefmt='%H:%M:%S') def load_plugins(self): self._plugin_engine = PluginEngine(self.spec, self.sack) self._plugin_engine.load_plugins(Path('rpg/plugins'), self.conf.exclude) for directory in self.conf.directories: self._plugin_engine.load_plugins(Path(directory), self.conf.exclude) def create_archive(self): """ Creates archive (archvie_path) from Source folder """ self.spec.Source = self.spec.Name + "-" + self.spec.Version + ".tar.gz" _tar = Command("tar zcf " + path_to_str(self.archive_path) + " -C " + path_to_str(self.extracted_dir) + " . --transform='s/^\./" + self.spec.Name + "-" + self.spec.Version + "/g'") _tar.execute() logging.debug(str(_tar)) @property def base_dir(self): try: return Path("/tmp/rpg-%s-%s" % (self._input_name, self._hash)) except AttributeError: msg = "`load_project_from_url` method needs to be called first" raise RuntimeError(msg) @property def extracted_dir(self): return self.base_dir / "extracted" @property def compiled_dir(self): return self.base_dir / "compiled" @property def installed_dir(self): return self.base_dir / "installed" @property def project_name(self): return self.spec.Name @property def spec_path(self): return self.base_dir / (self.project_name + ".spec") @property def archive_path(self): return self.base_dir / self.spec.Source @property def srpm_path(self): try: return next(self.base_dir.glob(self.project_name + "*.src.rpm")) except StopIteration: raise RuntimeError( "Can't find '{}'! You need to call build_srpm first.".format( str(self.base_dir / (self.project_name + "*.src.rpm")))) @property def rpm_path(self): try: _ret = [ _path for _path in self.base_dir.glob(self.project_name + "*.rpm") if not str(_path).endswith(".src.rpm") ] if not _ret: raise StopIteration return _ret except StopIteration: raise RuntimeError( "Can't find '{}'! You need to call build_rpm first.".format( str(self.base_dir / (self.project_name + "*.rpm")))) def load_project_from_url(self, path): """executed in background after dir/tarball/SRPM selection""" temp_arch = "downloaded_archive.tar.gz" if search(r"github\.com/[^/]+/[^/]+/?$", str(path)): self._source_loader.download_git_repo(path, temp_arch) path = Path(temp_arch) elif str(path).startswith("http"): temp_arch = search(r"([^/]+\.[^/]+(?:\.[^/]+)?)$", str(path))\ .group(0) self._source_loader.download_archive(path, temp_arch) path = Path(temp_arch) else: temp_arch = None path = Path(path) self._hash = self._compute_checksum(path) self._input_name = path.name self._setup_workspace() self._source_loader.load_sources(path, self.extracted_dir) self.spec.prep = Command("%autosetup") if temp_arch: remove(temp_arch) def run_extracted_source_analysis(self): """executed in background after dir/tarball/SRPM selection""" self._plugin_engine.execute_phase(phases[0], self.extracted_dir) def run_patched_source_analysis(self): """executed in background after patches are applied""" self._plugin_engine.execute_phase(phases[1], self.extracted_dir) def run_compiled_source_analysis(self): """executed in background after patches are applied""" self._plugin_engine.execute_phase(phases[2], self.compiled_dir) def install_project(self): """executed in background after filled requires screen""" self._project_builder.install(self.compiled_dir, self.installed_dir, self.spec.install) def run_installed_source_analysis(self): """executed in background after successful project build""" self._plugin_engine.execute_phase(phases[3], self.installed_dir) def write_spec(self): with open(str(self.spec_path), 'w') as spec_file: spec_file.write(str(self.spec)) def build_srpm(self): if not self.spec.Source or not self.archive_path.exists(): self.create_archive() self.write_spec() self._package_builder.build_srpm(self.spec_path, self.archive_path, self.base_dir) def build_rpm(self, target_distro, target_arch): try: self.srpm_path except RuntimeError: self.build_srpm() return self._package_builder.build_rpm(str(self.srpm_path), target_distro, target_arch, self.base_dir) def build_rpm_recover(self, distro, arch): def build(): self.build_srpm() self.build_rpm(distro, arch) def analyse(): _files_to_pkgs.installed(self.base_dir, self.spec, self.sack) self.write_spec() _files_to_pkgs = FilesToPkgsPlugin() analyse() while True: try: build() except BuildException as be: if not self._plugin_engine.execute_mock_recover(be.errors): if be.return_code: raise RuntimeError( "Build failed! See logs in '{}'".format( self._package_builder.mock_logs)) break analyse() def fetch_repos(self, dist, arch): self._package_builder.fetch_repos(dist, arch) def build_project(self): """executed in background after filled requires screen""" self._project_builder.build(self.extracted_dir, self.compiled_dir, self.spec.build) def copr_set_config(self, username, login, token): self.cl = CoprClient(username, login, token, copr_url="http://copr.fedoraproject.org") def copr_create_project(self, name, chroots, desc, intro): self.cl.create_project(name, chroots=chroots, description=desc, instructions=intro) def copr_build(self, name, url): self.cl.create_new_build(name, pkgs=[ url, ]) @staticmethod def _compute_checksum(sources): if sources.is_dir(): cmd = "find %s -type f -print0 | sort -z | xargs " \ "-0 sha1sum | sha1sum" % path_to_str(sources.resolve()) else: cmd = "sha1sum %s" % path_to_str(sources.resolve()) return cmd_output([cmd])[:7] @property def all_dirs(self): return [self.extracted_dir, self.compiled_dir, self.installed_dir] def _setup_workspace(self): """make sure all directories used later will exist""" shutil.rmtree(str(self.base_dir), True) for d in self.all_dirs: d.mkdir(parents=True) # predictor methods are used for autocompletion of the field, # every guess_* method return list of strings matched ordered # by their rank def guess_name(self): name = str(self._input_name) if isdir(name): return name else: if name[-4:] == ".zip": return name[:-4] else: if "tar" in name: return name.split(".tar")[0] return "" def guess_provide(self): # returns list of all known provides provides = set() for pkg in self.sack.query(): provides.update(pkg.provides) return sorted(provides) def guess_changelog_data(self): # returns list of tuples (author, email) from git pass def guess_dependency(self): # returns guess_provide() + all package names from repos names = map(lambda pkg: pkg.name, self.sack.query()) return sorted(set(names).union(set(self.guess_provide()))) def guess_license(self): # returns list of all known licenses licenses = set() for pkg in self.sack.query(): licenses.update(pkg.license) return sorted(licenses)
def test_git_download(self): SourceLoader.download_git_repo( "https://github.com/rh-lab-q/rpg", self._download) self.assertTrue(Path(str(self._download)).exists())
def test_download(self): SourceLoader.download_archive( "https://github.com/rh-lab-q/rpg/archive/master.tar.gz", self._download) self.assertTrue(Path(str(self._download)).exists())
def test_git_download(self): SourceLoader.download_git_repo("https://github.com/rh-lab-q/rpg", self._download) self.assertTrue(Path(str(self._download)).exists())
class SourceLoaderTest(RpgTestCase): # FileNotFound hash FNF_MD5 = "d41d8cd98f00b204e9800998ecf8427e" def setUp(self): self._source_loader = SourceLoader() self._tar = None self._tar_dir = self.test_project_dir / "archives" self._tar_gz = self.test_project_dir / "archives" / "sample.tar.gz" self._tar_xz = self.test_project_dir / "archives" / "sample.tar.xz" self._tar_temp = Path("/var/tmp/rpg_test/") self._tar_extracted = self._tar_temp / "extracted" self._download = self._tar_temp / "download.tar.gz" if path.isdir(str(self._tar_temp)): rmtree(str(self._tar_temp)) makedirs(str(self._tar_temp)) makedirs(str(self._tar_extracted)) def tearDown(self): if self._tar: remove(str(self._tar)) if path.isdir(str(self._tar_temp)): rmtree(str(self._tar_temp)) def md5Tar(self, t): mdsum = Command( "tar --list -f " + str(t) + " 2>/dev/null | " "awk -F/ '{ if($NF != \"\") print $NF }' | " r'sed -e "s/.*\///gm" | sort | md5sum' ).execute() self.assertNotEqual(self.FNF_MD5, mdsum) return mdsum def md5Dir(self, d): md5sum = Command( "find " + str(d) + r' -type f | sed -e "s/.*\///gm" | sort | md5sum' ).execute() self.assertNotEqual(self.FNF_MD5, md5sum) return md5sum def test_tar_gz_method(self): self.assertEqual( self._source_loader._get_compression_method( str(self._tar_gz)), ("tar", "gz")) def test_tar_xz_method(self): self.assertEqual( self._source_loader._get_compression_method( str(self._tar_xz)), ("tar", "xz")) def test_tar_gz_extract(self): self._source_loader.load_sources( self._tar_gz, self._tar_extracted) self.assertTrue( path.isdir(str(self._tar_extracted))) self.assertEqual( self.md5Tar(str(self._tar_gz)), self.md5Dir(str(self._tar_extracted))) self.assertExistInDir(["file"], self._tar_extracted) def test_tar_xz_extract(self): self._source_loader.load_sources( self._tar_xz, self._tar_extracted) self.assertTrue( path.isdir(str(self._tar_extracted))) self.assertEqual( self.md5Tar(str(self._tar_xz)), self.md5Dir(str(self._tar_extracted))) self.assertExistInDir(["file1", "file2"], self._tar_extracted) def test_dir_source_loader(self): self._source_loader.load_sources( self._tar_dir, self._tar_extracted) self.assertEqual( self.md5Dir(self._tar_extracted), self.md5Dir(self._tar_dir)) self.assertExistInDir(["sample.tar.gz", "sample.tar.xz"], self._tar_extracted) @expectedFailure def test_tar_xz_method_fail(self): self._tar_xz = self._tar_gz self.test_tar_xz_method() @expectedFailure def test_tar_gz_method_fail(self): self._tar_gz = self._tar_xz self.test_tar_gz_method() @expectedFailure def test_tar_gz_extract_fail(self): self._tar_gz = "NotAnArchive" self.test_tar_gz_extract() @expectedFailure def test_tar_xz_extract_fail(self): self._tar_xz = "NotAnArchive" self.test_tar_xz_extract()