def run_check(cls, results_dir, **kwargs):
cls.results_dir = os.path.join(results_dir, 'rpminspect-rpm')
cls.prepare_results_dir()
result = {
'path': cls.get_checker_output_dir_short(),
'files': [],
'checks': {}
}
old_pkgs = results_store.get_old_build()['rpm']
new_pkgs = results_store.get_new_build()['rpm']
for old_pkg in old_pkgs:
if 'debuginfo' in old_pkg:
continue
name = RpmHelper.split_nevra(os.path.basename(old_pkg))['name']
found = [
x for x in new_pkgs
if RpmHelper.split_nevra(os.path.basename(x))['name'] == name
]
if not found:
logger.warning('New version of package %s was not found!',
name)
continue
new_pkg = found[0]
outfile, pkg_data = cls.run_rpminspect(cls.results_dir, old_pkg,
new_pkg)
result['files'].append(os.path.basename(outfile))
result['checks'][name] = pkg_data
return result
def _find_debuginfo(cls, debug, pkg):
name = RpmHelper.split_nevra(os.path.basename(pkg))['name']
debuginfo = '{}-debuginfo'.format(name)
find = [x for x in debug if RpmHelper.split_nevra(os.path.basename(x))['name'] == debuginfo]
if find:
return find[0]
srpm = RpmHelper.get_header_from_rpm(pkg).sourcerpm
debuginfo = '{}-debuginfo'.format(RpmHelper.split_nevra(srpm)['name'])
find = [x for x in debug if RpmHelper.split_nevra(os.path.basename(x))['name'] == debuginfo]
if find:
return find[0]
return None
def _find_debuginfo(cls, debug, pkg):
name = RpmHelper.split_nevra(os.path.basename(pkg))['name']
debuginfo = '{}-debuginfo'.format(name)
find = [x for x in debug if RpmHelper.split_nevra(os.path.basename(x))['name'] == debuginfo]
if find:
return find[0]
srpm = RpmHelper.get_info_from_rpm(pkg, rpm.RPMTAG_SOURCERPM)
debuginfo = '{}-debuginfo'.format(RpmHelper.split_nevra(srpm)['name'])
find = [x for x in debug if RpmHelper.split_nevra(os.path.basename(x))['name'] == debuginfo]
if find:
return find[0]
return None
def run_check(cls, results_dir, **kwargs):
"""Compares old and new RPMs using abipkgdiff"""
# Check if ABI changes occured
cls.abi_changes = None
cls.results_dir = os.path.join(results_dir, cls.name)
os.makedirs(cls.results_dir)
debug_old, rest_pkgs_old = cls._get_packages_for_abipkgdiff(
results_store.get_build('old'))
debug_new, rest_pkgs_new = cls._get_packages_for_abipkgdiff(
results_store.get_build('new'))
cmd = [cls.CMD]
reports = {}
for pkg in rest_pkgs_old:
command = list(cmd)
debug = cls._find_debuginfo(debug_old, pkg)
if debug:
command.append('--d1')
command.append(debug)
old_name = RpmHelper.split_nevra(os.path.basename(pkg))['name']
find = [
x for x in rest_pkgs_new if RpmHelper.split_nevra(
os.path.basename(x))['name'] == old_name
]
if not find:
logger.warning('New version of package %s was not found!',
old_name)
continue
new_pkg = find[0]
debug = cls._find_debuginfo(debug_new, new_pkg)
if debug:
command.append('--d2')
command.append(debug)
command.append(pkg)
command.append(new_pkg)
logger.verbose('Package name for ABI comparison %s', old_name)
output = os.path.join(cls.results_dir, old_name + '.txt')
try:
ret_code = ProcessHelper.run_subprocess(command,
output_file=output)
except OSError:
raise CheckerNotFoundError(
"Checker '{}' was not found or installed.".format(
cls.name))
if int(ret_code) & cls.ABIDIFF_ERROR and int(
ret_code) & cls.ABIDIFF_USAGE_ERROR:
raise RebaseHelperError(
'Execution of {} failed.\nCommand line is: {}'.format(
cls.CMD, cmd))
reports[old_name] = int(ret_code)
return dict(packages=cls.parse_abi_logs(reports),
abi_changes=cls.abi_changes,
path=cls.get_checker_output_dir_short())
def run_check(cls, results_dir, **kwargs):
"""Compares old and new RPMs using abipkgdiff"""
# Check if ABI changes occured
cls.results_dir = os.path.join(results_dir, cls.name)
cls.prepare_results_dir()
debug_old, rest_pkgs_old = cls._get_packages_for_abipkgdiff(results_store.get_build('old'))
debug_new, rest_pkgs_new = cls._get_packages_for_abipkgdiff(results_store.get_build('new'))
ret_codes = {}
for pkg in rest_pkgs_old:
command = [cls.CMD]
command_fallback = [cls.CMD]
debug = cls._find_debuginfo(debug_old, pkg)
if debug:
command.append('--d1')
command.append(debug)
old_name = RpmHelper.split_nevra(os.path.basename(pkg))['name']
find = [x for x in rest_pkgs_new if RpmHelper.split_nevra(os.path.basename(x))['name'] == old_name]
if not find:
logger.warning('New version of package %s was not found!', old_name)
continue
new_pkg = find[0]
debug = cls._find_debuginfo(debug_new, new_pkg)
if debug:
command.append('--d2')
command.append(debug)
command.extend([pkg, new_pkg])
command_fallback.extend([pkg, new_pkg])
logger.verbose('Package name for ABI comparison %s', old_name)
output = os.path.join(cls.results_dir, old_name + '.txt')
for cmd in [command, command_fallback]:
try:
ret_code = ProcessHelper.run_subprocess(cmd, output_file=output)
except OSError as e:
raise CheckerNotFoundError("Checker '{}' was not found or installed.".format(cls.name)) from e
if int(ret_code) & cls.ABIDIFF_ERROR and int(ret_code) & cls.ABIDIFF_USAGE_ERROR:
raise RebaseHelperError(
'Execution of {} failed.\nCommand line is: {}'.format(cls.CMD, ' '.join(cmd)))
if int(ret_code) & cls.ABIDIFF_ERROR:
# abipkgdiff might not be able to read the debuginfo, try again without it
continue
break
ret_codes[old_name] = int(ret_code)
return dict(packages=cls.parse_abi_logs(ret_codes),
abi_changes=any(x & cls.ABIDIFF_ABI_CHANGE for x in ret_codes.values()),
abi_incompatible_changes=any(x & cls.ABIDIFF_ABI_INCOMPATIBLE_CHANGE for x in ret_codes.values()),
path=cls.get_checker_output_dir_short(),
ret_codes=ret_codes)
def download_task_results(cls, session, tasklist, destination):
"""Downloads packages and logs of finished Koji tasks.
Args:
session (koji.ClientSession): Active Koji session instance.
tasklist (list): List of task IDs.
destination (str): Path where to download files to.
Returns:
tuple: List of downloaded RPMs and list of downloaded logs.
Raises:
DownloadError: If download failed.
"""
rpms = []
logs = []
for task_id in tasklist:
logger.info('Downloading packages and logs for task %s', task_id)
task = session.getTaskInfo(task_id, request=True)
if task['state'] in [koji.TASK_STATES['FREE'], koji.TASK_STATES['OPEN']]:
logger.info('Task %s is still running!', task_id)
continue
elif task['state'] != koji.TASK_STATES['CLOSED']:
logger.info('Task %s did not complete successfully!', task_id)
if task['method'] == 'buildArch':
tasks = [task]
elif task['method'] == 'build':
opts = dict(parent=task_id, method='buildArch', decode=True,
state=[koji.TASK_STATES['CLOSED'], koji.TASK_STATES['FAILED']])
tasks = session.listTasks(opts=opts)
else:
logger.info('Task %s is not a build or buildArch task!', task_id)
continue
for task in tasks:
base_path = koji.pathinfo.taskrelpath(task['id'])
output = session.listTaskOutput(task['id'])
for filename in output:
local_path = os.path.join(destination, filename)
download = False
fn, ext = os.path.splitext(filename)
if ext == '.rpm':
if task['state'] != koji.TASK_STATES['CLOSED']:
continue
if local_path not in rpms:
nevra = RpmHelper.split_nevra(fn)
# FIXME: multiple arches
download = nevra['arch'] in ['noarch', 'x86_64']
if download:
rpms.append(local_path)
else:
if local_path not in logs:
download = True
logs.append(local_path)
if download:
logger.info('Downloading file %s', filename)
url = '/'.join([session.opts['topurl'], 'work', base_path, filename])
DownloadHelper.download_file(url, local_path)
return rpms, logs
def _find_debuginfo(cls, debug, pkg):
name = RpmHelper.split_nevra(os.path.basename(pkg))['name']
debuginfo = '{}-debuginfo'.format(name)
find = [
x for x in debug
if RpmHelper.split_nevra(os.path.basename(x))['name'] == debuginfo
]
if find:
return find[0]
srpm = RpmHelper.get_info_from_rpm(pkg, rpm.RPMTAG_SOURCERPM)
debuginfo = '{}-debuginfo'.format(RpmHelper.split_nevra(srpm)['name'])
find = [
x for x in debug
if RpmHelper.split_nevra(os.path.basename(x))['name'] == debuginfo
]
if find:
return find[0]
return None
def run_check(cls, results_dir, **kwargs):
"""Compares old and new RPMs using abipkgdiff"""
# Check if ABI changes occured
cls.abi_changes = False
cls.results_dir = os.path.join(results_dir, cls.name)
os.makedirs(cls.results_dir)
debug_old, rest_pkgs_old = cls._get_packages_for_abipkgdiff(results_store.get_build('old'))
debug_new, rest_pkgs_new = cls._get_packages_for_abipkgdiff(results_store.get_build('new'))
cmd = [cls.CMD]
reports = {}
for pkg in rest_pkgs_old:
command = list(cmd)
debug = cls._find_debuginfo(debug_old, pkg)
if debug:
command.append('--d1')
command.append(debug)
old_name = RpmHelper.split_nevra(os.path.basename(pkg))['name']
find = [x for x in rest_pkgs_new if RpmHelper.split_nevra(os.path.basename(x))['name'] == old_name]
if not find:
logger.warning('New version of package %s was not found!', old_name)
continue
new_pkg = find[0]
debug = cls._find_debuginfo(debug_new, new_pkg)
if debug:
command.append('--d2')
command.append(debug)
command.append(pkg)
command.append(new_pkg)
logger.verbose('Package name for ABI comparison %s', old_name)
output = os.path.join(cls.results_dir, old_name + '.txt')
try:
ret_code = ProcessHelper.run_subprocess(command, output_file=output)
except OSError:
raise CheckerNotFoundError("Checker '{}' was not found or installed.".format(cls.name))
if int(ret_code) & cls.ABIDIFF_ERROR and int(ret_code) & cls.ABIDIFF_USAGE_ERROR:
raise RebaseHelperError('Execution of {} failed.\nCommand line is: {}'.format(cls.CMD, cmd))
reports[old_name] = int(ret_code)
return dict(packages=cls.parse_abi_logs(reports),
abi_changes=cls.abi_changes,
path=cls.get_checker_output_dir_short())
def run_check(cls, results_dir, **kwargs):
cls.results_dir = os.path.join(results_dir, 'rpminspect-srpm')
cls.prepare_results_dir()
result = {
'path': cls.get_checker_output_dir_short(),
'files': [],
'checks': {}
}
old_pkg = results_store.get_old_build()['srpm']
new_pkg = results_store.get_new_build()['srpm']
name = RpmHelper.split_nevra(os.path.basename(old_pkg))['name']
outfile, pkg_data = cls.run_rpminspect(cls.results_dir, old_pkg,
new_pkg)
result['files'].append(os.path.basename(outfile))
result['checks'][name] = pkg_data
return result
def run_rpminspect(cls, checker_dir: str, old_pkg: str,
new_pkg: str) -> Tuple[str, Dict[str, int]]:
"""Runs rpminspect on the given package.
Args:
checker_dir: Path to the results directory of the checker
old_pkg: Path to the old package.
new_pkg: Path to the new package.
Returns:
Tuple of (path, data), where path is the path to rpminspect output
and data is the dict of count of OK/BAD/VERIFY checks.
"""
cmd = [cls.CMD, '-F', 'json']
pkg_name = RpmHelper.split_nevra(os.path.basename(new_pkg))['name']
cmd.append(old_pkg)
cmd.append(new_pkg)
outfile = os.path.join(checker_dir, '{}.json'.format(pkg_name))
try:
ret = ProcessHelper.run_subprocess(cmd,
output_file=outfile,
ignore_stderr=True)
except OSError as e:
raise CheckerNotFoundError(
'Checker \'{}\' was not found or installed.'.format(
cls.name)) from e
# Exit code 1 is used when bad check is found, 0 when everything is OK. Others on error
if ret not in (0, 1):
raise RebaseHelperError(
'An error occurred when running checker \'{}\''.format(
cls.name))
with open(outfile, 'r', encoding=ENCODING) as json_file:
data = json.load(json_file)
return outfile, cls.process_data(data)
def download_task_results(cls, session, tasklist, destination):
"""Downloads packages and logs of finished Koji tasks.
Args:
session (koji.ClientSession): Active Koji session instance.
tasklist (list): List of task IDs.
destination (str): Path where to download files to.
Returns:
tuple: List of downloaded RPMs and list of downloaded logs.
Raises:
DownloadError: If download failed.
"""
rpms = []
logs = []
for task_id in tasklist:
logger.info('Downloading packages and logs for task %s', task_id)
task = session.getTaskInfo(task_id, request=True)
if task['state'] in [
koji.TASK_STATES['FREE'], koji.TASK_STATES['OPEN']
]:
logger.info('Task %s is still running!', task_id)
continue
elif task['state'] != koji.TASK_STATES['CLOSED']:
logger.info('Task %s did not complete successfully!', task_id)
if task['method'] == 'buildArch':
tasks = [task]
elif task['method'] == 'build':
opts = dict(parent=task_id,
method='buildArch',
decode=True,
state=[
koji.TASK_STATES['CLOSED'],
koji.TASK_STATES['FAILED']
])
tasks = session.listTasks(opts=opts)
else:
logger.info('Task %s is not a build or buildArch task!',
task_id)
continue
for task in tasks:
base_path = koji.pathinfo.taskrelpath(task['id'])
output = session.listTaskOutput(task['id'])
for filename in output:
local_path = os.path.join(destination, filename)
download = False
fn, ext = os.path.splitext(filename)
if ext == '.rpm':
if task['state'] != koji.TASK_STATES['CLOSED']:
continue
if local_path not in rpms:
nevra = RpmHelper.split_nevra(fn)
# FIXME: multiple arches
download = nevra['arch'] in ['noarch', 'x86_64']
if download:
rpms.append(local_path)
else:
if local_path not in logs:
download = True
logs.append(local_path)
if download:
logger.info('Downloading file %s', filename)
url = '/'.join([
session.opts['topurl'], 'work', base_path, filename
])
DownloadHelper.download_file(url, local_path)
return rpms, logs
def test_split_nevra(self, nevra, name, epoch, version, release, arch):
assert RpmHelper.split_nevra(nevra) == dict(name=name, epoch=epoch, version=version,
release=release, arch=arch)
def test_split_nevra(self, nevra, name, epoch, version, release, arch):
assert RpmHelper.split_nevra(nevra) == dict(name=name,
epoch=epoch,
version=version,
release=release,
arch=arch)