class SourceLoaderTest(RpgTestCase):
# FileNotFound hash
FNF_MD5 = "d41d8cd98f00b204e9800998ecf8427e"
def setUp(self):
self._source_loader = SourceLoader()
self._tar = False
self._tar_dir = self.test_project_dir
self._tar_gz = self.test_project_dir / "archives" / "sample.tar.gz"
self._tar_xz = self.test_project_dir / "archives" / "sample.tar.xz"
self._tar_temp = "/var/tmp/rpg_test/"
self._tar_extracted = self._tar_temp + "extracted"
self._archive = self._tar_temp + "sample"
self._hasher = None
if path.isdir(self._tar_extracted):
rmtree(self._tar_extracted)
if path.isdir(self._tar_temp):
rmtree(self._tar_temp)
makedirs(self._tar_temp)
makedirs(self._tar_extracted)
def tearDown(self):
if self._tar:
remove(str(self._tar))
def md5TarXz(self, t):
mdsum = Command(
"tar -J -xOf " + str(t) +
" 2>/dev/null | md5sum | cut -b-32"
).execute()[:-1]
self.assertNotEqual(self.FNF_MD5, mdsum)
return mdsum
def md5TarGz(self, t):
mdsum = Command(
"tar -xOzf " + str(t) +
" 2>/dev/null | md5sum | cut -b-32"
).execute()[:-1]
self.assertNotEqual(self.FNF_MD5, mdsum)
return mdsum
def md5Dir(self, d):
mdsum = md5(
Command(
"find " + d + r" -type f -exec cat {} \;"
).execute(True)).hexdigest()
self.assertNotEqual(self.FNF_MD5, mdsum)
return mdsum
def test_tar_gz_method(self):
self.assertEqual(
self._source_loader.get_compression_method(
str(self._tar_gz)),
("tar", "gz"))
def test_tar_xz_method(self):
self.assertEqual(
self._source_loader.get_compression_method(
str(self._tar_xz)),
("tar", "xz"))
def test_tar_gz_extract(self):
self._source_loader.load_sources(
self._tar_gz,
self._tar_extracted)
self.assertTrue(
path.isdir(str(self._tar_extracted)))
self.assertEqual(
self.md5TarGz(str(self._tar_gz)),
self.md5Dir(str(self._tar_extracted)))
def test_tar_xz_extract(self):
self._source_loader.load_sources(
self._tar_xz,
self._tar_extracted)
self.assertTrue(
path.isdir(str(self._tar_extracted)))
self.assertEqual(
self.md5TarXz(str(self._tar_xz)),
self.md5Dir(str(self._tar_extracted)))
def test_dir_source_loader(self):
self._source_loader.load_sources(
str(self.test_project_dir),
self._tar_extracted)
self.assertEqual(
self.md5Dir(self._tar_extracted),
self.md5Dir(str(self.test_project_dir)))
def test_create_archive(self):
self._tar = self._source_loader.create_archive(
str(self._archive),
str(self._tar_dir))
self.assertEqual(
self.md5TarGz(str(self._tar)),
self.md5Dir(str(self._tar_dir)))
@expectedFailure
def test_tar_xz_method_fail(self):
self._tar_xz = self._tar_gz
self.test_tar_xz_method()
@expectedFailure
def test_tar_gz_method_fail(self):
self._tar_gz = self._tar_xz
self.test_tar_gz_method()
@expectedFailure
def test_tar_gz_extract_fail(self):
self._tar_gz = "NotAnArchive"
self.test_tar_gz_extract()
@expectedFailure
def test_tar_xz_extract_fail(self):
self._tar_xz = "NotAnArchive"
self.test_tar_xz_extract()
@expectedFailure
def test_create_archive_fail(self):
self._tar_dir = "NonExistingDir"
self.test_create_archive()
class Base(object):
"""Base class that is controlled by RPM GUI"""
def __init__(self):
self.conf = Conf()
self._setup_logging()
self._project_builder = ProjectBuilder()
self.spec = Spec()
self.sack = None
self._package_builder = PackageBuilder()
self._source_loader = SourceLoader()
self._copr_uploader = CoprUploader()
def dnf_load_sack(self):
logging.info('DNF sack is loading')
import dnf
with dnf.Base() as self._dnf_base:
self._dnf_base.conf.releasever = dnf.rpm.detect_releasever(
self._dnf_base.conf.installroot)
self._dnf_base.read_all_repos()
self._dnf_base.fill_sack()
return self._dnf_base.sack
def _setup_logging(self):
if geteuid() == 0:
log_dir = "/var/log/rpg/"
else:
log_dir = "/var/tmp/rpg/"
if not isdir(log_dir):
makedirs(log_dir)
logging.basicConfig(level=logging.DEBUG,
format='[%(asctime)s] {%(pathname)s:%(lineno)d} '
'%(levelname)s - %(message)s',
handlers=[logging.FileHandler(log_dir + "rpg.log"),
logging.StreamHandler()],
datefmt='%H:%M:%S')
def load_plugins(self):
self._plugin_engine = PluginEngine(self.spec, self.sack)
self._plugin_engine.load_plugins(
Path('rpg/plugins'),
self.conf.exclude)
for directory in self.conf.directories:
self._plugin_engine.load_plugins(
Path(directory),
self.conf.exclude)
@property
def base_dir(self):
try:
return Path("/tmp/rpg-%s-%s" % (self._input_name, self._hash))
except AttributeError:
msg = "`process_archive_or_dir` method needs to be called first"
raise RuntimeError(msg)
@property
def extracted_dir(self):
return self.base_dir / "extracted"
@property
def compiled_dir(self):
return self.base_dir / "compiled"
@property
def installed_dir(self):
return self.base_dir / "installed"
@property
def project_name(self):
return self.spec.Name
@property
def spec_path(self):
return self.base_dir / (self.project_name + ".spec")
@property
def archive_path(self):
return self.base_dir / self.spec.Source
@property
def srpm_path(self):
return next(self.base_dir.glob(self.project_name + "*src.rpm"))
def process_archive_or_dir(self, path):
"""executed in background after dir/tarball/SRPM selection"""
p = Path(path)
self._hash = self.compute_checksum(p)
self._input_name = p.name
self._setup_workspace()
self._source_loader.load_sources(p, self.extracted_dir)
def run_raw_sources_analysis(self):
"""executed in background after dir/tarball/SRPM selection"""
self._plugin_engine.execute_phase(phases[0],
self.extracted_dir)
def apply_patches(self, ordered_patches):
"""executed in background after patch selection and reordering"""
self._project_builder.apply_patches(ordered_patches)
def run_patched_sources_analysis(self):
"""executed in background after patches are applied"""
self._plugin_engine.execute_phase(phases[1],
self.extracted_dir)
def build_project(self):
"""executed in background after filled requires screen"""
self._project_builder.build(self.extracted_dir,
self.compiled_dir,
self.spec.build)
def run_compiled_analysis(self):
"""executed in background after patches are applied"""
self._plugin_engine.execute_phase(phases[2],
self.extracted_dir)
def install_project(self):
"""executed in background after filled requires screen"""
self._project_builder.install(self.compiled_dir,
self.installed_dir,
self.spec.install)
def run_installed_analysis(self):
"""executed in background after successful project build"""
self._plugin_engine.execute_phase(phases[3],
self.installed_dir)
def write_spec(self):
with open(str(self.spec_path), 'w') as spec_file:
spec_file.write(str(self.spec))
def build_srpm(self):
if not self.spec_path.exists():
self.write_spec()
self._package_builder.build_srpm(
self.spec_path, self.archive_path, self.base_dir)
def build_packages(self, distros, archs=platform.machine()):
"""builds packages for desired distributions"""
for arch in archs:
for distro in distros:
self._package_builder.build(self.spec_path, self.archive_path,
distro, arch)
@staticmethod
def compute_checksum(sources):
if sources.is_dir():
cmd = "find %s -type f -print0 | sort -z | xargs " \
"-0 sha1sum | sha1sum" % sources.resolve()
else:
cmd = "sha1sum %s" % sources.resolve()
return cmd_output([cmd])[:7]
@property
def all_dirs(self):
return [
self.extracted_dir,
self.compiled_dir,
self.installed_dir
]
def _setup_workspace(self):
"""make sure all directories used later will exist"""
try:
shutil.rmtree(str(self.base_dir))
except FileNotFoundError:
pass
for d in self.all_dirs:
d.mkdir(parents=True)
# predictor methods are used for autocompletion of the field,
# every guess_* method return list of strings matched ordered
# by their rank
def guess_name(self):
name = str(self._input_name)
if isdir(name):
return name
else:
if name[-4:] == ".zip":
return name[:-4]
else:
if "tar" in name:
return name.split(".tar")[0]
return ""
def guess_provide(self):
# returns list of all known provides
provides = set()
for pkg in self.sack.query():
provides.update(pkg.provides)
return sorted(provides)
def guess_changelog_data(self):
# returns list of tuples (author, email) from git
pass
def guess_dependency(self):
# returns guess_provide() + all package names from repos
names = map(lambda pkg: pkg.name, self.sack.query())
return sorted(set(names).union(set(self.guess_provide())))
def guess_license(self):
# returns list of all known licenses
licenses = set()
for pkg in self.sack.query():
licenses.update(pkg.license)
return sorted(licenses)
class Base(object):
"""Base class that is controlled by RPM GUI"""
def __init__(self):
self.conf = Conf()
self._setup_logging()
self._project_builder = ProjectBuilder()
self.spec = Spec()
self.sack = None
self._package_builder = PackageBuilder()
self._source_loader = SourceLoader()
def dnf_load_sack(self):
logging.info('DNF sack is loading')
import dnf
with dnf.Base() as self._dnf_base:
self._dnf_base.conf.releasever = dnf.rpm.detect_releasever(
self._dnf_base.conf.installroot)
self._dnf_base.read_all_repos()
self._dnf_base.fill_sack()
return self._dnf_base.sack
def _setup_logging(self):
if geteuid() == 0:
log_dir = "/var/log/rpg/"
else:
log_dir = "/var/tmp/rpg/"
if not isdir(log_dir):
makedirs(log_dir)
logging.basicConfig(level=logging.DEBUG,
format='[%(asctime)s] {%(pathname)s:%(lineno)d} '
'%(levelname)s - %(message)s',
handlers=[logging.FileHandler(log_dir + "rpg.log"),
logging.StreamHandler()],
datefmt='%H:%M:%S')
def load_plugins(self):
self._plugin_engine = PluginEngine(self.spec, self.sack)
self._plugin_engine.load_plugins(
Path('rpg/plugins'),
self.conf.exclude)
for directory in self.conf.directories:
self._plugin_engine.load_plugins(
Path(directory),
self.conf.exclude)
def create_archive(self):
""" Creates archive (archvie_path) from Source folder """
self.spec.Source = self.spec.Name + "-" + self.spec.Version + ".tar.gz"
_tar = Command("tar zcf " + str(self.archive_path) +
" -C " + str(self.extracted_dir) +
" . --transform='s/^\./" +
self.spec.Name + "-" + self.spec.Version + "/g'")
_tar.execute()
logging.debug(str(_tar))
@property
def base_dir(self):
try:
return Path("/tmp/rpg-%s-%s" % (self._input_name, self._hash))
except AttributeError:
msg = "`load_project_from_url` method needs to be called first"
raise RuntimeError(msg)
@property
def extracted_dir(self):
return self.base_dir / "extracted"
@property
def compiled_dir(self):
return self.base_dir / "compiled"
@property
def installed_dir(self):
return self.base_dir / "installed"
@property
def project_name(self):
return self.spec.Name
@property
def spec_path(self):
return self.base_dir / (self.project_name + ".spec")
@property
def archive_path(self):
return self.base_dir / self.spec.Source
@property
def srpm_path(self):
try:
return next(self.base_dir.glob(self.project_name + "*src.rpm"))
except StopIteration:
raise RuntimeError(
"Can't find '{}'! You need to call build_srpm first."
.format(str(self.base_dir / (self.project_name + "*src.rpm"))))
def load_project_from_url(self, path):
"""executed in background after dir/tarball/SRPM selection"""
path = Path(path)
temp_arch = "downloaded_archive.tar.gz"
if search(r"github\.com/[^/]+/[^/]+/?$", str(path)):
self._source_loader.download_git_repo(str(path), temp_arch)
elif str(path).startswith("http"):
self._source_loader.download_archive(str(path), temp_arch)
else:
temp_arch = None
self._hash = self.compute_checksum(path)
self._input_name = path.name
self._setup_workspace()
self._source_loader.load_sources(path, self.extracted_dir)
self.spec.prep = Command("%autosetup")
if temp_arch:
remove(temp_arch)
def run_raw_sources_analysis(self):
"""executed in background after dir/tarball/SRPM selection"""
self._plugin_engine.execute_phase(phases[0],
self.extracted_dir)
def apply_patches(self, ordered_patches):
"""executed in background after patch selection and reordering"""
self._project_builder.apply_patches(ordered_patches)
def run_patched_sources_analysis(self):
"""executed in background after patches are applied"""
self._plugin_engine.execute_phase(phases[1],
self.extracted_dir)
def run_compiled_analysis(self):
"""executed in background after patches are applied"""
self._plugin_engine.execute_phase(phases[2],
self.extracted_dir)
def install_project(self):
"""executed in background after filled requires screen"""
self._project_builder.install(self.compiled_dir,
self.installed_dir,
self.spec.install)
def run_installed_analysis(self):
"""executed in background after successful project build"""
self._plugin_engine.execute_phase(phases[3],
self.installed_dir)
def write_spec(self):
with open(str(self.spec_path), 'w') as spec_file:
spec_file.write(str(self.spec))
def build_srpm(self):
if not self.spec.Source or not self.archive_path.exists():
self.create_archive()
self.write_spec()
self._package_builder.build_srpm(
self.spec_path, self.archive_path, self.base_dir)
def build_rpm(self, target_distro, target_arch):
self._package_builder.build_rpm(str(self.srpm_path),
target_distro, target_arch)
def fetch_repos(self, dist, arch):
self._package_builder.fetch_repos(dist, arch)
def build_project(self):
"""executed in background after filled requires screen"""
self._project_builder.build(self.extracted_dir,
self.compiled_dir,
self.spec.build)
def copr_set_config(self, username, login, token):
self.cl = CoprClient(
username, login, token, copr_url="http://copr.fedoraproject.org")
def copr_create_project(self, name, chroots, desc, intro):
self.cl.create_project(
name, chroots=chroots, description=desc, instructions=intro)
def copr_build(self, name, url):
self.cl.create_new_build(name, pkgs=[url, ])
@staticmethod
def compute_checksum(sources):
if sources.is_dir():
cmd = "find %s -type f -print0 | sort -z | xargs " \
"-0 sha1sum | sha1sum" % sources.resolve()
else:
cmd = "sha1sum %s" % sources.resolve()
return cmd_output([cmd])[:7]
@property
def all_dirs(self):
return [
self.extracted_dir,
self.compiled_dir,
self.installed_dir
]
def _setup_workspace(self):
"""make sure all directories used later will exist"""
shutil.rmtree(str(self.base_dir), True)
for d in self.all_dirs:
d.mkdir(parents=True)
# predictor methods are used for autocompletion of the field,
# every guess_* method return list of strings matched ordered
# by their rank
def guess_name(self):
name = str(self._input_name)
if isdir(name):
return name
else:
if name[-4:] == ".zip":
return name[:-4]
else:
if "tar" in name:
return name.split(".tar")[0]
return ""
def guess_provide(self):
# returns list of all known provides
provides = set()
for pkg in self.sack.query():
provides.update(pkg.provides)
return sorted(provides)
def guess_changelog_data(self):
# returns list of tuples (author, email) from git
pass
def guess_dependency(self):
# returns guess_provide() + all package names from repos
names = map(lambda pkg: pkg.name, self.sack.query())
return sorted(set(names).union(set(self.guess_provide())))
def guess_license(self):
# returns list of all known licenses
licenses = set()
for pkg in self.sack.query():
licenses.update(pkg.license)
return sorted(licenses)
def build_rpm_recover(self, distro, arch):
def build():
self.build_srpm()
return self._package_builder.build_rpm(
self.srpm_path,
distro, arch) + list(self._package_builder.check_logs())
_files_to_pkgs = FilesToPkgsPlugin()
while True:
_file = ""
_files_to_pkgs.installed(self.base_dir,
self.spec,
self.sack)
self.write_spec()
for err in build():
match = search(
r"DEBUG\:\s*[^:]+\:\s*([^:]+)\:\s*" +
r"[cC][oO][mM][mM][aA][nN][dD]\s*[nN][oO][tT]\s*" +
r"[fF][oO][uU][nN][dD]", err)
if match:
if match.group(1) == _file or\
"/usr/bin/" + match.group(1) == _file:
logging.info("Couldn't resolve '{}'!"
.format(match.group(1)))
return
else:
if "/" in match.group(1):
_file = match.group(1)
else:
_file = "/usr/bin/" + match.group(1)
break
if _file == "":
break
self.spec.required_files.add(_file)
self.spec.build_required_files.add(_file)
Command("rm -rf " + str(self._package_builder.temp_dir) + "/*.log")
class SourceLoaderTest(RpgTestCase):
# FileNotFound hash
FNF_MD5 = "d41d8cd98f00b204e9800998ecf8427e"
def setUp(self):
self._source_loader = SourceLoader()
self._tar = None
self._tar_dir = self.test_project_dir / "archives"
self._tar_gz = self.test_project_dir / "archives" / "sample.tar.gz"
self._tar_xz = self.test_project_dir / "archives" / "sample.tar.xz"
self._tar_temp = Path("/var/tmp/rpg_test/")
self._tar_extracted = self._tar_temp / "extracted"
self._download = self._tar_temp / "download.tar.gz"
if path.isdir(path_to_str(self._tar_temp)):
rmtree(path_to_str(self._tar_temp))
makedirs(path_to_str(self._tar_temp))
makedirs(path_to_str(self._tar_extracted))
def tearDown(self):
if self._tar:
remove(path_to_str(self._tar))
if path.isdir(path_to_str(self._tar_temp)):
rmtree(path_to_str(self._tar_temp))
def md5Tar(self, t):
mdsum = Command("tar --list -f " + path_to_str(t) + " 2>/dev/null | "
"awk -F/ '{ if($NF != \"\") print $NF }' | "
r'sed -e "s/.*\///gm" | sort | md5sum').execute()
self.assertNotEqual(self.FNF_MD5, mdsum)
return mdsum
def md5Dir(self, d):
md5sum = Command(
"find " + path_to_str(d) +
r' -type f | sed -e "s/.*\///gm" | sort | md5sum').execute()
self.assertNotEqual(self.FNF_MD5, md5sum)
return md5sum
def test_tar_gz_method(self):
self.assertEqual(
self._source_loader._get_compression_method(
path_to_str(self._tar_gz)), ("tar", "gz"))
def test_tar_xz_method(self):
self.assertEqual(
self._source_loader._get_compression_method(
path_to_str(self._tar_xz)), ("tar", "xz"))
def test_tar_gz_extract(self):
self._source_loader.load_sources(self._tar_gz, self._tar_extracted)
self.assertTrue(path.isdir(path_to_str(self._tar_extracted)))
self.assertEqual(self.md5Tar(path_to_str(self._tar_gz)),
self.md5Dir(path_to_str(self._tar_extracted)))
self.assertExistInDir(["file"], self._tar_extracted)
def test_tar_xz_extract(self):
self._source_loader.load_sources(self._tar_xz, self._tar_extracted)
self.assertTrue(path.isdir(path_to_str(self._tar_extracted)))
self.assertEqual(self.md5Tar(path_to_str(self._tar_xz)),
self.md5Dir(path_to_str(self._tar_extracted)))
self.assertExistInDir(["file1", "file2"], self._tar_extracted)
def test_dir_source_loader(self):
self._source_loader.load_sources(self._tar_dir, self._tar_extracted)
self.assertEqual(self.md5Dir(self._tar_extracted),
self.md5Dir(self._tar_dir))
self.assertExistInDir(["sample.tar.gz", "sample.tar.xz"],
self._tar_extracted)
def test_extensions(self):
self.assertEqual(
("zip", None),
self._source_loader._get_compression_method("sample.zip"))
self.assertEqual(
("zip", None),
self._source_loader._get_compression_method("sample.sample.zip"))
self.assertEqual(
("zip", None),
self._source_loader._get_compression_method("sample.tar.zip"))
self.assertEqual(
("tar", "xz"),
self._source_loader._get_compression_method("sample.tar.xz"))
@expectedFailure
def test_extensions_fail(self):
self._source_loader._get_compression_method("sample.tar.zip")
self._source_loader._get_compression_method("sample.xz.tar")
self._source_loader._get_compression_method("sample.tar.")
self._source_loader._get_compression_method("sample.zip.xz")
@expectedFailure
def test_tar_xz_method_fail(self):
self._tar_xz = self._tar_gz
self.test_tar_xz_method()
@expectedFailure
def test_tar_gz_method_fail(self):
self._tar_gz = self._tar_xz
self.test_tar_gz_method()
@expectedFailure
def test_tar_gz_extract_fail(self):
self._tar_gz = "NotAnArchive"
self.test_tar_gz_extract()
@expectedFailure
def test_tar_xz_extract_fail(self):
self._tar_xz = "NotAnArchive"
self.test_tar_xz_extract()
class Base(object):
"""Base class that is controlled by RPM GUI"""
def __init__(self):
self._setup_logging()
self._project_builder = ProjectBuilder()
self.spec = Spec()
self.sack = None # TODO dnf sack
self._package_builder = PackageBuilder()
self._plugin_engine = PluginEngine(self.spec, self.sack)
self._source_loader = SourceLoader()
self._copr_uploader = CoprUploader()
self._plugin_engine.load_plugins(Path('rpg/plugins'))
def _setup_logging(self):
logging.basicConfig(level=logging.DEBUG,
format='[%(asctime)s] {%(pathname)s:%(lineno)d} '
'%(levelname)s - %(message)s',
handlers=[logging.FileHandler("rpg.log"),
logging.StreamHandler()],
datefmt='%H:%M:%S')
@property
def base_dir(self):
try:
return Path("/tmp/rpg-%s-%s" % (self._input_name, self._hash))
except AttributeError:
msg = "`process_archive_or_dir` method needs to be called first"
raise RuntimeError(msg)
@property
def extracted_dir(self):
return self.base_dir / "extracted"
@property
def compiled_dir(self):
return self.base_dir / "compiled"
@property
def installed_dir(self):
return self.base_dir / "installed"
@property
def project_name(self):
return self.spec.tags["name"]
@property
def spec_path(self):
return self.base_dir / (self.project_name + ".spec")
@property
def tarball_path(self):
return self.base_dir / (self.project_name + ".tar.gz")
@property
def rpm_path(self):
return next(self.base_dir.glob(self.project_name + "*.rpm"))
def process_archive_or_dir(self, path):
"""executed in background after dir/tarball/SRPM selection"""
p = Path(path)
self._hash = self.compute_checksum(p)
self._input_name = p.name
self.setup_workspace()
self._source_loader.load_sources(p, self.extracted_dir)
def run_raw_sources_analysis(self):
"""executed in background after dir/tarball/SRPM selection"""
self._plugin_engine.execute_phase(phases[0],
self.extracted_dir)
def apply_patches(self, ordered_patches):
"""executed in background after patch selection and reordering"""
self._project_builder.apply_patches(ordered_patches)
def run_patched_sources_analysis(self):
"""executed in background after patches are applied"""
self._plugin_engine.execute_phase(phases[1],
self.extracted_dir)
def build_project(self):
"""executed in background after filled requires screen"""
self._project_builder.build(self.extracted_dir,
self.compiled_dir,
self.spec.scripts["%build"])
def run_compiled_analysis(self):
"""executed in background after patches are applied"""
self._plugin_engine.execute_phase(phases[2],
self.extracted_dir)
def install_project(self):
"""executed in background after filled requires screen"""
self._project_builder.install(self.compiled_dir,
self.installed_dir,
self.spec.scripts["%install"])
def run_installed_analysis(self):
"""executed in background after successful project build"""
self._plugin_engine.execute_phase(phases[3],
self.compiled_dir)
def build_packages(self, *distros):
"""builds packages for desired distributions"""
for distro in distros:
self._package_builder.build(self.spec_path, self.tarball_path,
self.distro)
@staticmethod
def compute_checksum(sources):
if sources.is_dir():
cmd = "find %s -type f -print0 | sort -z | xargs " \
"-0 sha1sum | sha1sum" % sources.resolve()
else:
cmd = "sha1sum %s" % sources.resolve()
return cmd_output([cmd])[:7]
@property
def all_dirs(self):
return [
self.extracted_dir,
self.compiled_dir,
self.installed_dir
]
def setup_workspace(self):
"""make sure all directories used later will exist"""
try:
shutil.rmtree(str(self.base_dir))
except FileNotFoundError:
pass
for d in self.all_dirs:
d.mkdir(parents=True)
# predictor methods are used for autocompletion of the field,
# every guess_* method return list of strings matched ordered
# by their rank
def guess_name(self):
pass
def guess_provide(self):
pass
def guess_group(self):
pass
def guess_chagelog_data(self):
# returns list of tuples (author, email)
pass
def guess_build_dependency(self):
pass
def guess_dependency(self):
pass
def guess_license(self):
pass
class Base(object):
"""Base class that is controlled by RPM GUI
:Example:
>>> from rpg import Base
>>> base = Base()
>>> base.sack = base.load_dnf_sack()
>>> base.load_plugins()
>>> base.load_project_from_url("https://github.com/example/ex_repo")
>>> base.spec.Name = "Example"
>>> base.spec.Version = "0.6.11"
>>> base.spec.Release = "1%{?snapshot}%{?dist}"
>>> base.spec.License = "GPLv2"
>>> base.spec.Summary = "Example ..."
>>> base.spec.description = ("Example ...")
>>> base.spec.URL = "https://github.com/example/ex_repo"
>>> base.target_arch = "x86_64"
>>> base.target_distro = "fedora-22"
>>> base.fetch_repos(base.target_distro, base.target_arch)
>>> base.run_extracted_source_analysis()
>>> base.run_patched_source_analysis()
>>> base.build_project()
>>> base.run_compiled_source_analysis()
>>> base.install_project()
>>> base.run_installed_source_analysis()
>>> base.build_srpm()
>>> base.build_rpm_recover(self.base.target_distro, self.base.target_arch)
"""
def __init__(self):
self.conf = Conf()
self._setup_logging()
self._project_builder = ProjectBuilder()
self.spec = Spec()
self.sack = None
self._package_builder = PackageBuilder()
self._source_loader = SourceLoader()
def load_dnf_sack(self):
logging.info('DNF sack is loading')
import dnf
with dnf.Base() as self._dnf_base:
self._dnf_base.conf.releasever = dnf.rpm.detect_releasever(
self._dnf_base.conf.installroot)
self._dnf_base.read_all_repos()
self._dnf_base.fill_sack()
return self._dnf_base.sack
def _setup_logging(self):
if geteuid() == 0:
log_dir = "/var/log/rpg/"
else:
log_dir = "/var/tmp/rpg/"
if not isdir(log_dir):
makedirs(log_dir)
logging.basicConfig(level=logging.DEBUG,
format='[%(asctime)s] {%(pathname)s:%(lineno)d} '
'%(levelname)s - %(message)s',
handlers=[logging.FileHandler(log_dir + "rpg.log"),
logging.StreamHandler()],
datefmt='%H:%M:%S')
def load_plugins(self):
""" This method sets up plugin engine and loads them """
self._plugin_engine = PluginEngine(self.spec, self.sack)
self._plugin_engine.load_plugins(
Path('rpg/plugins'),
self.conf.exclude)
for directory in self.conf.directories:
self._plugin_engine.load_plugins(
Path(directory),
self.conf.exclude)
def create_archive(self):
""" Creates archive (archvie_path) from Source folder """
self.spec.Source = self.spec.Name + "-" + self.spec.Version + ".tar.gz"
_tar = Command("tar zcf " + path_to_str(self.archive_path) +
" -C " + path_to_str(self.extracted_dir) +
" . --transform='s/^\./" +
self.spec.Name + "-" + self.spec.Version + "/g'")
_tar.execute()
logging.debug(str(_tar))
@property
def base_dir(self):
""" Returns path where compiled, extracted, installed
directories are """
try:
return Path("/tmp/rpg-%s-%s" % (self._input_name, self._hash))
except AttributeError:
msg = "`load_project_from_url` method needs to be called first"
raise RuntimeError(msg)
@property
def extracted_dir(self):
return self.base_dir / "extracted"
@property
def compiled_dir(self):
return self.base_dir / "compiled"
@property
def installed_dir(self):
return self.base_dir / "installed"
@property
def project_name(self):
return self.spec.Name
@property
def spec_path(self):
return self.base_dir / (self.project_name + ".spec")
@property
def archive_path(self):
return self.base_dir / self.spec.Source
@property
def srpm_path(self):
""" Returns path to SRPM only, if it is created.
You have to build srpm first. """
try:
return next(self.base_dir.glob(self.project_name + "*.src.rpm"))
except StopIteration:
raise RuntimeError(
"Can't find '{}'! You need to call build_srpm first."
.format(str(self.base_dir /
(self.project_name + "*.src.rpm"))))
@property
def rpm_path(self):
""" This is the same as it is in srpm_path. But this returns
list of rpms - there may be severals rpm packages like
debuginfo, binary rpm and so on. """
try:
_ret = [
_path
for _path in self.base_dir.glob(self.project_name + "*.rpm")
if not str(_path).endswith(".src.rpm")
]
if not _ret:
raise StopIteration
return _ret
except StopIteration:
raise RuntimeError(
"Can't find '{}'! You need to call build_rpm first."
.format(str(self.base_dir / (self.project_name + "*.rpm"))))
def load_project_from_url(self, path):
"""executed in background after dir/tarball/SRPM selection"""
temp_arch = "downloaded_archive.tar.gz"
if search(r"github\.com/[^/]+/[^/]+/?$", str(path)):
self._source_loader.download_git_repo(path, temp_arch)
path = Path(temp_arch)
elif str(path).startswith("http"):
temp_arch = search(r"([^/]+\.[^/]+(?:\.[^/]+)?)$", str(path))\
.group(0)
self._source_loader.download_archive(path, temp_arch)
path = Path(temp_arch)
else:
temp_arch = None
path = Path(path)
self._hash = self._compute_checksum(path)
self._input_name = path.name
self._setup_workspace()
self._source_loader.load_sources(path, self.extracted_dir)
self.spec.prep = Command("%autosetup")
if temp_arch:
remove(temp_arch)
def run_extracted_source_analysis(self):
"""executed in background after dir/tarball/SRPM selection"""
self._plugin_engine.execute_phase(PluginEngine.phases[0],
self.extracted_dir)
def run_patched_source_analysis(self):
"""executed in background after patches are applied"""
self._plugin_engine.execute_phase(PluginEngine.phases[1],
self.extracted_dir)
def run_compiled_source_analysis(self):
"""executed in background after patches are applied"""
self._plugin_engine.execute_phase(PluginEngine.phases[2],
self.compiled_dir)
def install_project(self):
"""executed in background after filled requires screen"""
self._project_builder.install(self.compiled_dir,
self.installed_dir,
self.spec.install)
def run_installed_source_analysis(self):
"""executed in background after successful project build"""
self._plugin_engine.execute_phase(PluginEngine.phases[3],
self.installed_dir)
def write_spec(self):
""" Creates spec file or rewrites old one. """
with open(str(self.spec_path), 'w') as spec_file:
spec_file.write(str(self.spec))
def build_srpm(self):
""" Builds srpm into base directory. """
if not self.spec.Source or not self.archive_path.exists():
self.create_archive()
self.write_spec()
self._package_builder.build_srpm(
self.spec_path, self.archive_path, self.base_dir)
def build_rpm(self, target_distro, target_arch):
""" Build rpm from srpm. If srpm does not exists,
it will be created. """
try:
self.srpm_path
except RuntimeError:
self.build_srpm()
self._package_builder.build_rpm(
str(self.srpm_path), target_distro, target_arch, self.base_dir)
def build_rpm_recover(self, distro, arch):
""" Repeatedly build rpm with mock and finds all build errors.
May raise RuntimeError on failed recover. """
def build():
self.build_srpm()
self.build_rpm(distro, arch)
def analyse():
_files_to_pkgs.installed(self.base_dir, self.spec, self.sack)
self.write_spec()
_files_to_pkgs = FilesToPkgsPlugin()
analyse()
while True:
try:
build()
except BuildException as be:
if not self._plugin_engine.execute_mock_recover(be.errors):
if be.return_code:
raise RuntimeError(
"Build failed! See logs in '{}'"
.format(self._package_builder.mock_logs))
break
analyse()
def fetch_repos(self, dist, arch):
""" Initialize mock - should be called before build_rpm_recover """
self._package_builder.fetch_repos(dist, arch)
def build_project(self):
""" Executed in background after filled requires screen """
self._project_builder.build(self.extracted_dir,
self.compiled_dir,
self.spec.build)
def copr_set_config(self, username, login, token):
""" Logs into copr with username, login and token.
This has to be called before copr_create_project and copr_build
To sign up on copr go here: http://copr.fedoraproject.org """
self.cl = CoprClient(
username, login, token, copr_url="http://copr.fedoraproject.org")
def copr_create_project(self, name, chroots, desc, intro):
""" Creates metadata about project - won't build until
copr_build would be called """
self.cl.create_project(
name, chroots=chroots, description=desc, instructions=intro)
def copr_build(self, name, url):
""" Builds project on fedora copr server """
self.cl.create_new_build(name, pkgs=[url, ])
@staticmethod
def _compute_checksum(sources):
if sources.is_dir():
cmd = "find %s -type f -print0 | sort -z | xargs " \
"-0 sha1sum | sha1sum" % path_to_str(sources.resolve())
else:
cmd = "sha1sum %s" % path_to_str(sources.resolve())
return Command([cmd]).execute()[:7]
@property
def all_dirs(self):
""" Returns Extracted, Compiled and Installed direcotry paths. """
return [
self.extracted_dir,
self.compiled_dir,
self.installed_dir
]
def _setup_workspace(self):
"""make sure all directories used later will exist"""
shutil.rmtree(str(self.base_dir), True)
for d in self.all_dirs:
d.mkdir(parents=True)
# predictor methods are used for autocompletion of the field,
# every guess_* method return list of strings matched ordered
# by their rank
def guess_name(self):
""" Returns guessed name from source path """
suffixes = [".zip", ".tar", ".rar", ".tgz", ".lzma"]
name = str(self._input_name)
if isdir(name):
return basename(name)
else:
for suffix in suffixes:
if suffix in name:
return suffix.join(name.split(suffix)[:-1])
return ""
def guess_provide(self):
""" returns list of all known provides """
provides = set()
for pkg in self.sack.query():
provides.update(pkg.provides)
return sorted(provides)
def guess_changelog_data(self):
""" returns list of tuples (author, email) from git """
pass
def guess_dependency(self):
""" returns guess_provide() + all package names from repos """
names = map(lambda pkg: pkg.name, self.sack.query())
return sorted(set(names).union(set(self.guess_provide())))
def guess_license(self):
""" returns list of all known licenses """
licenses = set()
for pkg in self.sack.query():
licenses.update(pkg.license)
return sorted(licenses)
class Base(object):
"""Base class that is controlled by RPM GUI"""
def __init__(self):
self.conf = Conf()
self._setup_logging()
self._project_builder = ProjectBuilder()
self.spec = Spec()
self.sack = None
self._package_builder = PackageBuilder()
self._source_loader = SourceLoader()
def load_dnf_sack(self):
logging.info('DNF sack is loading')
import dnf
with dnf.Base() as self._dnf_base:
self._dnf_base.conf.releasever = dnf.rpm.detect_releasever(
self._dnf_base.conf.installroot)
self._dnf_base.read_all_repos()
self._dnf_base.fill_sack()
return self._dnf_base.sack
def _setup_logging(self):
if geteuid() == 0:
log_dir = "/var/log/rpg/"
else:
log_dir = "/var/tmp/rpg/"
if not isdir(log_dir):
makedirs(log_dir)
logging.basicConfig(level=logging.DEBUG,
format='[%(asctime)s] {%(pathname)s:%(lineno)d} '
'%(levelname)s - %(message)s',
handlers=[
logging.FileHandler(log_dir + "rpg.log"),
logging.StreamHandler()
],
datefmt='%H:%M:%S')
def load_plugins(self):
self._plugin_engine = PluginEngine(self.spec, self.sack)
self._plugin_engine.load_plugins(Path('rpg/plugins'),
self.conf.exclude)
for directory in self.conf.directories:
self._plugin_engine.load_plugins(Path(directory),
self.conf.exclude)
def create_archive(self):
""" Creates archive (archvie_path) from Source folder """
self.spec.Source = self.spec.Name + "-" + self.spec.Version + ".tar.gz"
_tar = Command("tar zcf " + path_to_str(self.archive_path) + " -C " +
path_to_str(self.extracted_dir) +
" . --transform='s/^\./" + self.spec.Name + "-" +
self.spec.Version + "/g'")
_tar.execute()
logging.debug(str(_tar))
@property
def base_dir(self):
try:
return Path("/tmp/rpg-%s-%s" % (self._input_name, self._hash))
except AttributeError:
msg = "`load_project_from_url` method needs to be called first"
raise RuntimeError(msg)
@property
def extracted_dir(self):
return self.base_dir / "extracted"
@property
def compiled_dir(self):
return self.base_dir / "compiled"
@property
def installed_dir(self):
return self.base_dir / "installed"
@property
def project_name(self):
return self.spec.Name
@property
def spec_path(self):
return self.base_dir / (self.project_name + ".spec")
@property
def archive_path(self):
return self.base_dir / self.spec.Source
@property
def srpm_path(self):
try:
return next(self.base_dir.glob(self.project_name + "*.src.rpm"))
except StopIteration:
raise RuntimeError(
"Can't find '{}'! You need to call build_srpm first.".format(
str(self.base_dir / (self.project_name + "*.src.rpm"))))
@property
def rpm_path(self):
try:
_ret = [
_path
for _path in self.base_dir.glob(self.project_name + "*.rpm")
if not str(_path).endswith(".src.rpm")
]
if not _ret:
raise StopIteration
return _ret
except StopIteration:
raise RuntimeError(
"Can't find '{}'! You need to call build_rpm first.".format(
str(self.base_dir / (self.project_name + "*.rpm"))))
def load_project_from_url(self, path):
"""executed in background after dir/tarball/SRPM selection"""
temp_arch = "downloaded_archive.tar.gz"
if search(r"github\.com/[^/]+/[^/]+/?$", str(path)):
self._source_loader.download_git_repo(path, temp_arch)
path = Path(temp_arch)
elif str(path).startswith("http"):
temp_arch = search(r"([^/]+\.[^/]+(?:\.[^/]+)?)$", str(path))\
.group(0)
self._source_loader.download_archive(path, temp_arch)
path = Path(temp_arch)
else:
temp_arch = None
path = Path(path)
self._hash = self._compute_checksum(path)
self._input_name = path.name
self._setup_workspace()
self._source_loader.load_sources(path, self.extracted_dir)
self.spec.prep = Command("%autosetup")
if temp_arch:
remove(temp_arch)
def run_extracted_source_analysis(self):
"""executed in background after dir/tarball/SRPM selection"""
self._plugin_engine.execute_phase(phases[0], self.extracted_dir)
def run_patched_source_analysis(self):
"""executed in background after patches are applied"""
self._plugin_engine.execute_phase(phases[1], self.extracted_dir)
def run_compiled_source_analysis(self):
"""executed in background after patches are applied"""
self._plugin_engine.execute_phase(phases[2], self.compiled_dir)
def install_project(self):
"""executed in background after filled requires screen"""
self._project_builder.install(self.compiled_dir, self.installed_dir,
self.spec.install)
def run_installed_source_analysis(self):
"""executed in background after successful project build"""
self._plugin_engine.execute_phase(phases[3], self.installed_dir)
def write_spec(self):
with open(str(self.spec_path), 'w') as spec_file:
spec_file.write(str(self.spec))
def build_srpm(self):
if not self.spec.Source or not self.archive_path.exists():
self.create_archive()
self.write_spec()
self._package_builder.build_srpm(self.spec_path, self.archive_path,
self.base_dir)
def build_rpm(self, target_distro, target_arch):
try:
self.srpm_path
except RuntimeError:
self.build_srpm()
return self._package_builder.build_rpm(str(self.srpm_path),
target_distro, target_arch,
self.base_dir)
def build_rpm_recover(self, distro, arch):
def build():
self.build_srpm()
self.build_rpm(distro, arch)
def analyse():
_files_to_pkgs.installed(self.base_dir, self.spec, self.sack)
self.write_spec()
_files_to_pkgs = FilesToPkgsPlugin()
analyse()
while True:
try:
build()
except BuildException as be:
if not self._plugin_engine.execute_mock_recover(be.errors):
if be.return_code:
raise RuntimeError(
"Build failed! See logs in '{}'".format(
self._package_builder.mock_logs))
break
analyse()
def fetch_repos(self, dist, arch):
self._package_builder.fetch_repos(dist, arch)
def build_project(self):
"""executed in background after filled requires screen"""
self._project_builder.build(self.extracted_dir, self.compiled_dir,
self.spec.build)
def copr_set_config(self, username, login, token):
self.cl = CoprClient(username,
login,
token,
copr_url="http://copr.fedoraproject.org")
def copr_create_project(self, name, chroots, desc, intro):
self.cl.create_project(name,
chroots=chroots,
description=desc,
instructions=intro)
def copr_build(self, name, url):
self.cl.create_new_build(name, pkgs=[
url,
])
@staticmethod
def _compute_checksum(sources):
if sources.is_dir():
cmd = "find %s -type f -print0 | sort -z | xargs " \
"-0 sha1sum | sha1sum" % path_to_str(sources.resolve())
else:
cmd = "sha1sum %s" % path_to_str(sources.resolve())
return cmd_output([cmd])[:7]
@property
def all_dirs(self):
return [self.extracted_dir, self.compiled_dir, self.installed_dir]
def _setup_workspace(self):
"""make sure all directories used later will exist"""
shutil.rmtree(str(self.base_dir), True)
for d in self.all_dirs:
d.mkdir(parents=True)
# predictor methods are used for autocompletion of the field,
# every guess_* method return list of strings matched ordered
# by their rank
def guess_name(self):
name = str(self._input_name)
if isdir(name):
return name
else:
if name[-4:] == ".zip":
return name[:-4]
else:
if "tar" in name:
return name.split(".tar")[0]
return ""
def guess_provide(self):
# returns list of all known provides
provides = set()
for pkg in self.sack.query():
provides.update(pkg.provides)
return sorted(provides)
def guess_changelog_data(self):
# returns list of tuples (author, email) from git
pass
def guess_dependency(self):
# returns guess_provide() + all package names from repos
names = map(lambda pkg: pkg.name, self.sack.query())
return sorted(set(names).union(set(self.guess_provide())))
def guess_license(self):
# returns list of all known licenses
licenses = set()
for pkg in self.sack.query():
licenses.update(pkg.license)
return sorted(licenses)
class SourceLoaderTest(RpgTestCase):
# FileNotFound hash
FNF_MD5 = "d41d8cd98f00b204e9800998ecf8427e"
def setUp(self):
self._source_loader = SourceLoader()
self._tar = None
self._tar_dir = self.test_project_dir / "archives"
self._tar_gz = self.test_project_dir / "archives" / "sample.tar.gz"
self._tar_xz = self.test_project_dir / "archives" / "sample.tar.xz"
self._tar_temp = Path("/var/tmp/rpg_test/")
self._tar_extracted = self._tar_temp / "extracted"
self._download = self._tar_temp / "download.tar.gz"
if path.isdir(str(self._tar_temp)):
rmtree(str(self._tar_temp))
makedirs(str(self._tar_temp))
makedirs(str(self._tar_extracted))
def tearDown(self):
if self._tar:
remove(str(self._tar))
if path.isdir(str(self._tar_temp)):
rmtree(str(self._tar_temp))
def md5Tar(self, t):
mdsum = Command(
"tar --list -f " + str(t) + " 2>/dev/null | "
"awk -F/ '{ if($NF != \"\") print $NF }' | "
r'sed -e "s/.*\///gm" | sort | md5sum'
).execute()
self.assertNotEqual(self.FNF_MD5, mdsum)
return mdsum
def md5Dir(self, d):
md5sum = Command(
"find " + str(d) +
r' -type f | sed -e "s/.*\///gm" | sort | md5sum'
).execute()
self.assertNotEqual(self.FNF_MD5, md5sum)
return md5sum
def test_tar_gz_method(self):
self.assertEqual(
self._source_loader._get_compression_method(
str(self._tar_gz)),
("tar", "gz"))
def test_tar_xz_method(self):
self.assertEqual(
self._source_loader._get_compression_method(
str(self._tar_xz)),
("tar", "xz"))
def test_tar_gz_extract(self):
self._source_loader.load_sources(
self._tar_gz,
self._tar_extracted)
self.assertTrue(
path.isdir(str(self._tar_extracted)))
self.assertEqual(
self.md5Tar(str(self._tar_gz)),
self.md5Dir(str(self._tar_extracted)))
self.assertExistInDir(["file"], self._tar_extracted)
def test_tar_xz_extract(self):
self._source_loader.load_sources(
self._tar_xz,
self._tar_extracted)
self.assertTrue(
path.isdir(str(self._tar_extracted)))
self.assertEqual(
self.md5Tar(str(self._tar_xz)),
self.md5Dir(str(self._tar_extracted)))
self.assertExistInDir(["file1", "file2"], self._tar_extracted)
def test_dir_source_loader(self):
self._source_loader.load_sources(
self._tar_dir,
self._tar_extracted)
self.assertEqual(
self.md5Dir(self._tar_extracted),
self.md5Dir(self._tar_dir))
self.assertExistInDir(["sample.tar.gz", "sample.tar.xz"],
self._tar_extracted)
@expectedFailure
def test_tar_xz_method_fail(self):
self._tar_xz = self._tar_gz
self.test_tar_xz_method()
@expectedFailure
def test_tar_gz_method_fail(self):
self._tar_gz = self._tar_xz
self.test_tar_gz_method()
@expectedFailure
def test_tar_gz_extract_fail(self):
self._tar_gz = "NotAnArchive"
self.test_tar_gz_extract()
@expectedFailure
def test_tar_xz_extract_fail(self):
self._tar_xz = "NotAnArchive"
self.test_tar_xz_extract()