def _find_debuginfo(cls, debug, pkg):
name = RpmHelper.split_nevra(os.path.basename(pkg))['name']
debuginfo = '{}-debuginfo'.format(name)
find = [x for x in debug if RpmHelper.split_nevra(os.path.basename(x))['name'] == debuginfo]
if find:
return find[0]
srpm = RpmHelper.get_info_from_rpm(pkg, rpm.RPMTAG_SOURCERPM)
debuginfo = '{}-debuginfo'.format(RpmHelper.split_nevra(srpm)['name'])
find = [x for x in debug if RpmHelper.split_nevra(os.path.basename(x))['name'] == debuginfo]
if find:
return find[0]
return None
def _find_debuginfo(cls, debug, pkg):
name = RpmHelper.split_nevra(os.path.basename(pkg))['name']
debuginfo = '{}-debuginfo'.format(name)
find = [x for x in debug if RpmHelper.split_nevra(os.path.basename(x))['name'] == debuginfo]
if find:
return find[0]
srpm = RpmHelper.get_header_from_rpm(pkg).sourcerpm
debuginfo = '{}-debuginfo'.format(RpmHelper.split_nevra(srpm)['name'])
find = [x for x in debug if RpmHelper.split_nevra(os.path.basename(x))['name'] == debuginfo]
if find:
return find[0]
return None
def run_check(cls, results_dir, **kwargs):
"""Compares old and new RPMs using abipkgdiff"""
# Check if ABI changes occured
cls.abi_changes = None
cls.results_dir = os.path.join(results_dir, cls.name)
os.makedirs(cls.results_dir)
debug_old, rest_pkgs_old = cls._get_packages_for_abipkgdiff(
results_store.get_build('old'))
debug_new, rest_pkgs_new = cls._get_packages_for_abipkgdiff(
results_store.get_build('new'))
cmd = [cls.CMD]
reports = {}
for pkg in rest_pkgs_old:
command = list(cmd)
debug = cls._find_debuginfo(debug_old, pkg)
if debug:
command.append('--d1')
command.append(debug)
old_name = RpmHelper.split_nevra(os.path.basename(pkg))['name']
find = [
x for x in rest_pkgs_new if RpmHelper.split_nevra(
os.path.basename(x))['name'] == old_name
]
if not find:
logger.warning('New version of package %s was not found!',
old_name)
continue
new_pkg = find[0]
debug = cls._find_debuginfo(debug_new, new_pkg)
if debug:
command.append('--d2')
command.append(debug)
command.append(pkg)
command.append(new_pkg)
logger.verbose('Package name for ABI comparison %s', old_name)
output = os.path.join(cls.results_dir, old_name + '.txt')
try:
ret_code = ProcessHelper.run_subprocess(command,
output_file=output)
except OSError:
raise CheckerNotFoundError(
"Checker '{}' was not found or installed.".format(
cls.name))
if int(ret_code) & cls.ABIDIFF_ERROR and int(
ret_code) & cls.ABIDIFF_USAGE_ERROR:
raise RebaseHelperError(
'Execution of {} failed.\nCommand line is: {}'.format(
cls.CMD, cmd))
reports[old_name] = int(ret_code)
return dict(packages=cls.parse_abi_logs(reports),
abi_changes=cls.abi_changes,
path=cls.get_checker_output_dir_short())
def prepare(cls, spec, conf):
"""
Checks if all build dependencies are installed. If not, asks user whether they should be installed.
If he agrees, installs build dependencies using PolicyKit.
:param spec: SpecFile object
"""
req_pkgs = spec.get_requires()
if not RpmHelper.all_packages_installed(req_pkgs):
question = '\nSome build dependencies are missing. Do you want to install them now'
if conf.non_interactive or InputHelper.get_message(question):
if RpmHelper.install_build_dependencies(spec.get_path(), assume_yes=conf.non_interactive) != 0:
raise RebaseHelperError('Failed to install build dependencies')
def prepare(cls, spec, conf):
"""
Checks if all build dependencies are installed. If not, asks user whether they should be installed.
If he agrees, installs build dependencies using PolicyKit.
:param spec: SpecFile object
"""
req_pkgs = spec.get_requires()
if not RpmHelper.all_packages_installed(req_pkgs):
question = '\nSome build dependencies are missing. Do you want to install them now'
if conf.non_interactive or InputHelper.get_message(question):
if RpmHelper.install_build_dependencies(spec.get_path(), assume_yes=conf.non_interactive) != 0:
raise RebaseHelperError('Failed to install build dependencies')
def run_check(cls, results_dir, **kwargs):
"""Compares old and new RPMs using abipkgdiff"""
# Check if ABI changes occured
cls.results_dir = os.path.join(results_dir, cls.name)
cls.prepare_results_dir()
debug_old, rest_pkgs_old = cls._get_packages_for_abipkgdiff(results_store.get_build('old'))
debug_new, rest_pkgs_new = cls._get_packages_for_abipkgdiff(results_store.get_build('new'))
ret_codes = {}
for pkg in rest_pkgs_old:
command = [cls.CMD]
command_fallback = [cls.CMD]
debug = cls._find_debuginfo(debug_old, pkg)
if debug:
command.append('--d1')
command.append(debug)
old_name = RpmHelper.split_nevra(os.path.basename(pkg))['name']
find = [x for x in rest_pkgs_new if RpmHelper.split_nevra(os.path.basename(x))['name'] == old_name]
if not find:
logger.warning('New version of package %s was not found!', old_name)
continue
new_pkg = find[0]
debug = cls._find_debuginfo(debug_new, new_pkg)
if debug:
command.append('--d2')
command.append(debug)
command.extend([pkg, new_pkg])
command_fallback.extend([pkg, new_pkg])
logger.verbose('Package name for ABI comparison %s', old_name)
output = os.path.join(cls.results_dir, old_name + '.txt')
for cmd in [command, command_fallback]:
try:
ret_code = ProcessHelper.run_subprocess(cmd, output_file=output)
except OSError as e:
raise CheckerNotFoundError("Checker '{}' was not found or installed.".format(cls.name)) from e
if int(ret_code) & cls.ABIDIFF_ERROR and int(ret_code) & cls.ABIDIFF_USAGE_ERROR:
raise RebaseHelperError(
'Execution of {} failed.\nCommand line is: {}'.format(cls.CMD, ' '.join(cmd)))
if int(ret_code) & cls.ABIDIFF_ERROR:
# abipkgdiff might not be able to read the debuginfo, try again without it
continue
break
ret_codes[old_name] = int(ret_code)
return dict(packages=cls.parse_abi_logs(ret_codes),
abi_changes=any(x & cls.ABIDIFF_ABI_CHANGE for x in ret_codes.values()),
abi_incompatible_changes=any(x & cls.ABIDIFF_ABI_INCOMPATIBLE_CHANGE for x in ret_codes.values()),
path=cls.get_checker_output_dir_short(),
ret_codes=ret_codes)
def download_task_results(cls, session, tasklist, destination):
"""Downloads packages and logs of finished Koji tasks.
Args:
session (koji.ClientSession): Active Koji session instance.
tasklist (list): List of task IDs.
destination (str): Path where to download files to.
Returns:
tuple: List of downloaded RPMs and list of downloaded logs.
Raises:
DownloadError: If download failed.
"""
rpms = []
logs = []
for task_id in tasklist:
logger.info('Downloading packages and logs for task %s', task_id)
task = session.getTaskInfo(task_id, request=True)
if task['state'] in [koji.TASK_STATES['FREE'], koji.TASK_STATES['OPEN']]:
logger.info('Task %s is still running!', task_id)
continue
elif task['state'] != koji.TASK_STATES['CLOSED']:
logger.info('Task %s did not complete successfully!', task_id)
if task['method'] == 'buildArch':
tasks = [task]
elif task['method'] == 'build':
opts = dict(parent=task_id, method='buildArch', decode=True,
state=[koji.TASK_STATES['CLOSED'], koji.TASK_STATES['FAILED']])
tasks = session.listTasks(opts=opts)
else:
logger.info('Task %s is not a build or buildArch task!', task_id)
continue
for task in tasks:
base_path = koji.pathinfo.taskrelpath(task['id'])
output = session.listTaskOutput(task['id'])
for filename in output:
local_path = os.path.join(destination, filename)
download = False
fn, ext = os.path.splitext(filename)
if ext == '.rpm':
if task['state'] != koji.TASK_STATES['CLOSED']:
continue
if local_path not in rpms:
nevra = RpmHelper.split_nevra(fn)
# FIXME: multiple arches
download = nevra['arch'] in ['noarch', 'x86_64']
if download:
rpms.append(local_path)
else:
if local_path not in logs:
download = True
logs.append(local_path)
if download:
logger.info('Downloading file %s', filename)
url = '/'.join([session.opts['topurl'], 'work', base_path, filename])
DownloadHelper.download_file(url, local_path)
return rpms, logs
def _find_debuginfo(cls, debug, pkg):
name = RpmHelper.split_nevra(os.path.basename(pkg))['name']
debuginfo = '{}-debuginfo'.format(name)
find = [
x for x in debug
if RpmHelper.split_nevra(os.path.basename(x))['name'] == debuginfo
]
if find:
return find[0]
srpm = RpmHelper.get_info_from_rpm(pkg, rpm.RPMTAG_SOURCERPM)
debuginfo = '{}-debuginfo'.format(RpmHelper.split_nevra(srpm)['name'])
find = [
x for x in debug
if RpmHelper.split_nevra(os.path.basename(x))['name'] == debuginfo
]
if find:
return find[0]
return None
def run_check(cls, results_dir, **kwargs):
old_headers = [
RpmHelper.get_header_from_rpm(x)
for x in results_store.get_old_build().get('rpm', [])
]
new_headers = [
RpmHelper.get_header_from_rpm(x)
for x in results_store.get_new_build().get('rpm', [])
]
soname_changes: SonameChanges = collections.defaultdict(
lambda: collections.defaultdict(list))
for old in old_headers:
new = [x for x in new_headers if x.name == old.name]
if not new:
logger.warning('New version of package %s was not found!',
old.name)
continue
else:
new = new[0]
old_sonames = cls._get_sonames(old.provides)
new_sonames = cls._get_sonames(new.provides)
for old_soname in old_sonames:
if old_soname in new_sonames:
new_sonames.remove(old_soname)
continue
soname = [
x for x in new_sonames if os.path.splitext(x)[0] ==
os.path.splitext(old_soname)[0]
]
if not soname:
soname_changes[old.name]['removed'].append(old_soname)
else:
soname_changes[old.name]['changed'].append({
'from': old_soname,
'to': soname[0]
})
new_sonames.remove(soname[0])
if new_sonames:
soname_changes[old.name]['added'] = list(new_sonames)
return dict(soname_changes=soname_changes)
def run_check(cls, results_dir, **kwargs):
"""Compares old and new RPMs using abipkgdiff"""
# Check if ABI changes occured
cls.abi_changes = False
cls.results_dir = os.path.join(results_dir, cls.name)
os.makedirs(cls.results_dir)
debug_old, rest_pkgs_old = cls._get_packages_for_abipkgdiff(results_store.get_build('old'))
debug_new, rest_pkgs_new = cls._get_packages_for_abipkgdiff(results_store.get_build('new'))
cmd = [cls.CMD]
reports = {}
for pkg in rest_pkgs_old:
command = list(cmd)
debug = cls._find_debuginfo(debug_old, pkg)
if debug:
command.append('--d1')
command.append(debug)
old_name = RpmHelper.split_nevra(os.path.basename(pkg))['name']
find = [x for x in rest_pkgs_new if RpmHelper.split_nevra(os.path.basename(x))['name'] == old_name]
if not find:
logger.warning('New version of package %s was not found!', old_name)
continue
new_pkg = find[0]
debug = cls._find_debuginfo(debug_new, new_pkg)
if debug:
command.append('--d2')
command.append(debug)
command.append(pkg)
command.append(new_pkg)
logger.verbose('Package name for ABI comparison %s', old_name)
output = os.path.join(cls.results_dir, old_name + '.txt')
try:
ret_code = ProcessHelper.run_subprocess(command, output_file=output)
except OSError:
raise CheckerNotFoundError("Checker '{}' was not found or installed.".format(cls.name))
if int(ret_code) & cls.ABIDIFF_ERROR and int(ret_code) & cls.ABIDIFF_USAGE_ERROR:
raise RebaseHelperError('Execution of {} failed.\nCommand line is: {}'.format(cls.CMD, cmd))
reports[old_name] = int(ret_code)
return dict(packages=cls.parse_abi_logs(reports),
abi_changes=cls.abi_changes,
path=cls.get_checker_output_dir_short())
def run_check(cls, results_dir, **kwargs):
cls.results_dir = os.path.join(results_dir, 'rpminspect-srpm')
cls.prepare_results_dir()
result = {
'path': cls.get_checker_output_dir_short(),
'files': [],
'checks': {}
}
old_pkg = results_store.get_old_build()['srpm']
new_pkg = results_store.get_new_build()['srpm']
name = RpmHelper.split_nevra(os.path.basename(old_pkg))['name']
outfile, pkg_data = cls.run_rpminspect(cls.results_dir, old_pkg,
new_pkg)
result['files'].append(os.path.basename(outfile))
result['checks'][name] = pkg_data
return result
def _get_rpm_info(cls, name, packages):
if packages is None:
return None
basic_package = sorted(packages)[0]
return RpmHelper.get_info_from_rpm(basic_package, name)
def _get_rpm_info(cls, name, packages):
if packages is None:
return None
basic_package = sorted(packages)[0]
return getattr(RpmHelper.get_header_from_rpm(basic_package), name)
def _get_rpm_info(cls, name, packages):
if packages is None:
return None
basic_package = sorted(packages)[0]
return RpmHelper.get_info_from_rpm(basic_package, name)
def download_task_results(cls, session, tasklist, destination):
"""Downloads packages and logs of finished Koji tasks.
Args:
session (koji.ClientSession): Active Koji session instance.
tasklist (list): List of task IDs.
destination (str): Path where to download files to.
Returns:
tuple: List of downloaded RPMs and list of downloaded logs.
Raises:
DownloadError: If download failed.
"""
rpms = []
logs = []
for task_id in tasklist:
logger.info('Downloading packages and logs for task %s', task_id)
task = session.getTaskInfo(task_id, request=True)
if task['state'] in [
koji.TASK_STATES['FREE'], koji.TASK_STATES['OPEN']
]:
logger.info('Task %s is still running!', task_id)
continue
elif task['state'] != koji.TASK_STATES['CLOSED']:
logger.info('Task %s did not complete successfully!', task_id)
if task['method'] == 'buildArch':
tasks = [task]
elif task['method'] == 'build':
opts = dict(parent=task_id,
method='buildArch',
decode=True,
state=[
koji.TASK_STATES['CLOSED'],
koji.TASK_STATES['FAILED']
])
tasks = session.listTasks(opts=opts)
else:
logger.info('Task %s is not a build or buildArch task!',
task_id)
continue
for task in tasks:
base_path = koji.pathinfo.taskrelpath(task['id'])
output = session.listTaskOutput(task['id'])
for filename in output:
local_path = os.path.join(destination, filename)
download = False
fn, ext = os.path.splitext(filename)
if ext == '.rpm':
if task['state'] != koji.TASK_STATES['CLOSED']:
continue
if local_path not in rpms:
nevra = RpmHelper.split_nevra(fn)
# FIXME: multiple arches
download = nevra['arch'] in ['noarch', 'x86_64']
if download:
rpms.append(local_path)
else:
if local_path not in logs:
download = True
logs.append(local_path)
if download:
logger.info('Downloading file %s', filename)
url = '/'.join([
session.opts['topurl'], 'work', base_path, filename
])
DownloadHelper.download_file(url, local_path)
return rpms, logs
def test_split_nevra(self, nevra, name, epoch, version, release, arch):
assert RpmHelper.split_nevra(nevra) == dict(name=name, epoch=epoch, version=version,
release=release, arch=arch)
def test_split_nevra(self, nevra, name, epoch, version, release, arch):
assert RpmHelper.split_nevra(nevra) == dict(name=name,
epoch=epoch,
version=version,
release=release,
arch=arch)
def test_all_packages_installed_existing(self):
assert RpmHelper.all_packages_installed(['glibc', 'coreutils']) is True
def test_is_package_installed_non_existing(self):
assert RpmHelper.is_package_installed('non-existing-package') is False
assert RpmHelper.is_package_installed('another-non-existing-package') is False
def test_is_package_installed_existing(self):
assert RpmHelper.is_package_installed('glibc') is True
assert RpmHelper.is_package_installed('coreutils') is True
def test_is_package_installed_existing(self):
assert RpmHelper.is_package_installed('glibc') is True
assert RpmHelper.is_package_installed('coreutils') is True
def test_is_package_installed_non_existing(self):
assert RpmHelper.is_package_installed('non-existing-package') is False
assert RpmHelper.is_package_installed(
'another-non-existing-package') is False
def test_all_packages_installed_existing(self):
assert RpmHelper.all_packages_installed(['glibc', 'coreutils']) is True
def test_all_packages_installed_one_non_existing(self):
assert RpmHelper.all_packages_installed(['glibc', 'coreutils', 'non-existing-package']) is False
def _get_rpms(cls, rpm_list):
rpm_dict = {}
for rpm_name in rpm_list:
rpm_dict[RpmHelper.get_info_from_rpm(rpm_name, 'name')] = rpm_name
return rpm_dict
def test_all_packages_installed_one_non_existing(self):
assert RpmHelper.all_packages_installed(
['glibc', 'coreutils', 'non-existing-package']) is False