def run_check(cls, results_dir, **kwargs):
"""Compares old and new RPMs using pkgdiff"""
csmock_report = {}
old_pkgs = results_store.get_old_build().get('srpm', None)
new_pkgs = results_store.get_new_build().get('srpm', None)
results_dir = os.path.join(results_dir, cls.CMD)
os.makedirs(results_dir)
arguments = ['--force', '-a', '-r', 'fedora-rawhide-x86_64', '--base-srpm']
if old_pkgs and new_pkgs:
cmd = [cls.CMD]
cmd.extend(arguments)
cmd.append(old_pkgs)
cmd.append(new_pkgs)
cmd.extend(['-o', results_dir])
output = io.StringIO()
try:
ProcessHelper.run_subprocess(cmd, output_file=output)
except OSError:
raise CheckerNotFoundError("Checker '{}' was not found or installed.".format(cls.name))
csmock_report['error'] = PathHelper.find_all_files_current_dir(results_dir, '*.err')
csmock_report['txt'] = PathHelper.find_all_files_current_dir(results_dir, '*.txt')
csmock_report['log'] = PathHelper.find_all_files_current_dir(results_dir, '*.log')
csmock_report['path'] = cls.get_checker_output_dir_short()
return csmock_report
def run_check(cls, results_dir, **kwargs):
"""Compares old and new RPMs using pkgdiff"""
csmock_report = {}
old_pkgs = results_store.get_old_build().get('srpm', None)
new_pkgs = results_store.get_new_build().get('srpm', None)
cls.results_dir = os.path.join(results_dir, cls.CMD)
cls.prepare_results_dir()
arguments = [
'--force', '-a', '-r', 'fedora-rawhide-x86_64', '--base-srpm'
]
if old_pkgs and new_pkgs:
cmd = [cls.CMD]
cmd.extend(arguments)
cmd.append(old_pkgs)
cmd.append(new_pkgs)
cmd.extend(['-o', results_dir])
output = io.StringIO()
try:
ProcessHelper.run_subprocess(cmd, output_file=output)
except OSError as e:
raise CheckerNotFoundError(
"Checker '{}' was not found or installed.".format(
cls.name)) from e
csmock_report['error'] = PathHelper.find_all_files_current_dir(
results_dir, '*.err')
csmock_report['txt'] = PathHelper.find_all_files_current_dir(
results_dir, '*.txt')
csmock_report['log'] = PathHelper.find_all_files_current_dir(
results_dir, '*.log')
csmock_report['path'] = cls.get_checker_output_dir_short()
return csmock_report
def run_mergetool(cls, repo):
# we can't use GitPython here, as it doesn't allow
# for the command to attach to stdout directly
cwd = os.getcwd()
try:
os.chdir(repo.working_tree_dir)
ProcessHelper.run_subprocess(['git', 'mergetool'])
finally:
os.chdir(cwd)
def run_mergetool(cls, repo):
# we can't use GitPython here, as it doesn't allow
# for the command to attach to stdout directly
cwd = os.getcwd()
try:
os.chdir(repo.working_tree_dir)
ProcessHelper.run_subprocess(['git', 'mergetool'])
finally:
os.chdir(cwd)
def run_check(cls, results_dir, **kwargs):
"""Compares old and new RPMs using rpmdiff"""
results_dict = {}
for tag in cls.CHECKER_TAGS:
results_dict[tag] = []
cls.results_dir = os.path.join(results_dir, cls.name)
os.makedirs(cls.results_dir)
# Only S (size), M(mode) and 5 (checksum) are now important
not_catched_flags = ['T', 'F', 'G', 'U', 'V', 'L', 'D', 'N']
old_pkgs = cls._get_rpms(results_store.get_old_build().get('rpm', None))
new_pkgs = cls._get_rpms(results_store.get_new_build().get('rpm', None))
for key, value in old_pkgs.items():
if 'debuginfo' in key or 'debugsource' in key:
# skip debug{info,source} packages
continue
cmd = [cls.CMD]
# TODO modify to online command
for x in not_catched_flags:
cmd.extend(['-i', x])
cmd.append(value)
# We would like to build correct old package against correct new packages
try:
cmd.append(new_pkgs[key])
except KeyError:
logger.warning('New version of package %s was not found!', key)
continue
output = io.StringIO()
try:
ProcessHelper.run_subprocess(cmd, output_file=output)
except OSError:
raise CheckerNotFoundError("Checker '{}' was not found or installed.".format(cls.name))
results_dict = cls._analyze_logs(output, results_dict)
results_dict = cls.update_added_removed(results_dict)
cls.results_dict = {k: v for k, v in results_dict.items() if v}
lines = []
for key, val in results_dict.items():
if val:
if lines:
lines.append('')
lines.append('Following files were {}:'.format(key))
lines.extend(val)
rpmdiff_report = os.path.join(cls.results_dir, 'report.txt')
counts = {k: len(v) for k, v in results_dict.items()}
try:
with open(rpmdiff_report, "w") as f:
f.write('\n'.join(lines))
except IOError:
raise RebaseHelperError("Unable to write result from {} to '{}'".format(cls.name, rpmdiff_report))
return {'path': cls.get_checker_output_dir_short(), 'files_changes': counts}
def run_check(cls, results_dir, **kwargs):
"""Compares old and new RPMs using rpmdiff"""
results_dict: Dict[str, List[str]] = {}
for tag in cls.CHECKER_TAGS:
results_dict[tag] = []
cls.results_dir = os.path.join(results_dir, cls.name)
cls.prepare_results_dir()
# Only S (size), M(mode) and 5 (checksum) are now important
not_catched_flags = ['T', 'F', 'G', 'U', 'V', 'L', 'D', 'N']
old_pkgs = cls._get_rpms(results_store.get_old_build().get('rpm', None))
new_pkgs = cls._get_rpms(results_store.get_new_build().get('rpm', None))
for key, value in old_pkgs.items():
if 'debuginfo' in key or 'debugsource' in key:
# skip debug{info,source} packages
continue
cmd = [cls.CMD]
# TODO modify to online command
for x in not_catched_flags:
cmd.extend(['--ignore={0}'.format(x)])
cmd.append(value)
# We would like to build correct old package against correct new packages
try:
cmd.append(new_pkgs[key])
except KeyError:
logger.warning('New version of package %s was not found!', key)
continue
output = io.StringIO()
try:
ProcessHelper.run_subprocess(cmd, output_file=output)
except OSError as e:
raise CheckerNotFoundError("Checker '{}' was not found or installed.".format(cls.name)) from e
results_dict = cls._analyze_logs(output, results_dict)
results_dict = cls.update_added_removed(results_dict)
cls.results_dict = {k: v for k, v in results_dict.items() if v}
lines: List[str] = []
for key, val in results_dict.items():
if val:
if lines:
lines.append('')
lines.append('Following files were {}:'.format(key))
lines.extend(val)
rpmdiff_report = os.path.join(cls.results_dir, 'report.txt')
counts = {k: len(v) for k, v in results_dict.items()}
try:
with open(rpmdiff_report, "w", encoding=ENCODING) as f:
f.write('\n'.join(lines))
except IOError as e:
raise RebaseHelperError("Unable to write result from {} to '{}'".format(cls.name, rpmdiff_report)) from e
return {'path': cls.get_checker_output_dir_short(), 'files_changes': counts}
def _build_rpm(cls, srpm, workdir, results_dir, rpm_results_dir, builder_options=None):
"""
Build RPM using rpmbuild.
:param srpm: abs path to SRPM
:param workdir: abs path to working directory with rpmbuild directory
structure, which will be used as HOME dir.
:param results_dir: abs path to dir where the log should be placed.
:param rpm_results_dir: path directory to where RPMs will be placed.
:return: abs paths to built RPMs.
"""
logger.info("Building RPMs")
output = os.path.join(results_dir, "build.log")
cmd = [cls.CMD, '--rebuild', srpm]
if builder_options is not None:
cmd.extend(builder_options)
ret = ProcessHelper.run_subprocess_cwd_env(cmd,
env={'HOME': workdir},
output_file=output)
build_log_path = os.path.join(rpm_results_dir, 'build.log')
if ret == 0:
return [f for f in PathHelper.find_all_files(workdir, '*.rpm') if not f.endswith('.src.rpm')]
# An error occurred, raise an exception
logfile = build_log_path
logs = [l for l in PathHelper.find_all_files(results_dir, '*.log')]
cls.logs.extend([os.path.join(rpm_results_dir, os.path.basename(l)) for l in logs])
raise BinaryPackageBuildError("Building RPMs failed!", results_dir, logfile=logfile)
def is_available(cls):
try:
return ProcessHelper.run_subprocess(
[cls.CMD, '--version'],
output_file=ProcessHelper.DEV_NULL) == 0
except (IOError, OSError):
return False
def _build_rpm(cls, srpm, results_dir, rpm_results_dir, root=None, arch=None, builder_options=None):
"""
Build RPM using mock.
:param srpm: full path to the srpm.
:param results_dir: abs path to dir where the log should be placed.
:param rpm_results_dir: directory where rpms will be placed.
:param root: path to where chroot should be built.
:param arch: target architectures for the build.
:param builder_options: builder_options for mock.
:return abs paths to RPMs.
"""
logger.info("Building RPMs")
output = os.path.join(results_dir, "mock_output.log")
cmd = [cls.CMD, '--old-chroot', '--rebuild', srpm, '--resultdir', results_dir]
if root is not None:
cmd.extend(['--root', root])
if arch is not None:
cmd.extend(['--arch', arch])
if builder_options is not None:
cmd.extend(builder_options)
ret = ProcessHelper.run_subprocess(cmd, output_file=output)
if ret == 0:
return [f for f in PathHelper.find_all_files(results_dir, '*.rpm') if not f.endswith('.src.rpm')]
else:
logfile = Mock.get_mock_logfile_path(ret, rpm_results_dir, tmp_path=results_dir)
logs = [l for l in PathHelper.find_all_files(results_dir, '*.log')]
cls.logs.extend(os.path.join(rpm_results_dir, os.path.basename(l)) for l in logs)
raise BinaryPackageBuildError("Building RPMs failed!", rpm_results_dir, logfile=logfile)
def _build_srpm(cls, spec, workdir, results_dir, srpm_results_dir, srpm_builder_options):
"""
Build SRPM using rpmbuild.
:param spec: abs path to SPEC file inside the rpmbuild/SPECS in workdir.
:param workdir: abs path to working directory with rpmbuild directory
structure, which will be used as HOME dir.
:param results_dir: abs path to dir where the log should be placed.
:param srpm_results_dir: path to directory where SRPM will be placed.
:param srpm_builder_options: list of additional options to rpmbuild.
:return: abs path to built SRPM.
"""
logger.info("Building SRPM")
spec_loc, spec_name = os.path.split(spec)
output = os.path.join(results_dir, "build.log")
cmd = ['rpmbuild', '-bs', spec_name]
if srpm_builder_options is not None:
cmd.extend(srpm_builder_options)
ret = ProcessHelper.run_subprocess_cwd_env(cmd,
cwd=spec_loc,
env={'HOME': workdir},
output_file=output)
build_log_path = os.path.join(srpm_results_dir, 'build.log')
if ret == 0:
return PathHelper.find_first_file(workdir, '*.src.rpm')
# An error occurred, raise an exception
logfile = build_log_path
logs = [l for l in PathHelper.find_all_files(results_dir, '*.log')]
cls.logs = [os.path.join(srpm_results_dir, os.path.basename(l)) for l in logs]
raise SourcePackageBuildError("Building SRPM failed!", logfile=logfile)
def _build_rpm(cls, srpm, workdir, results_dir, rpm_results_dir, builder_options=None):
"""
Build RPM using rpmbuild.
:param srpm: abs path to SRPM
:param workdir: abs path to working directory with rpmbuild directory
structure, which will be used as HOME dir.
:param results_dir: abs path to dir where the log should be placed.
:param rpm_results_dir: path directory to where RPMs will be placed.
:return: abs paths to built RPMs.
"""
logger.info("Building RPMs")
output = os.path.join(results_dir, "build.log")
cmd = [cls.CMD, '--rebuild', srpm]
if builder_options is not None:
cmd.extend(builder_options)
ret = ProcessHelper.run_subprocess_cwd_env(cmd,
env={'HOME': workdir},
output_file=output)
build_log_path = os.path.join(rpm_results_dir, 'build.log')
if ret == 0:
return [f for f in PathHelper.find_all_files(workdir, '*.rpm') if not f.endswith('.src.rpm')]
# An error occurred, raise an exception
logfile = build_log_path
logs = [l for l in PathHelper.find_all_files(results_dir, '*.log')]
cls.logs.extend(os.path.join(rpm_results_dir, os.path.basename(l)) for l in logs)
raise BinaryPackageBuildError("Building RPMs failed!", results_dir, logfile=logfile)
def test_simple_cmd_with_redirected_output_fileobject(self):
buff = io.StringIO()
ret = ProcessHelper.run_subprocess(self.ECHO_COMMAND,
output_file=buff)
assert ret == 0
assert not os.path.exists(self.OUT_FILE)
assert buff.readline().strip('\n') == self.PHRASE
buff.close()
def test_simple_cmd_with_redirected_output_path(self):
ret = ProcessHelper.run_subprocess(self.ECHO_COMMAND,
output_file=self.OUT_FILE)
assert ret == 0
assert os.path.exists(self.OUT_FILE)
assert open(
self.OUT_FILE,
encoding=ENCODING).readline().strip('\n') == self.PHRASE
def test_simple_cmd_with_redirected_output_fileobject(self):
buff = io.StringIO()
ret = ProcessHelper.run_subprocess(self.ECHO_COMMAND,
output_file=buff)
assert ret == 0
assert not os.path.exists(self.OUT_FILE)
assert buff.readline().strip('\n') == self.PHRASE
buff.close()
def run_check(cls, results_dir, **kwargs):
"""
Compares old and new RPMs using pkgdiff
:param results_dir result dir where are stored results
"""
cls.results_dir = os.path.join(results_dir, cls.name)
cls.prepare_results_dir()
cls.pkgdiff_results_full_path_html = os.path.join(
cls.results_dir, cls.pkgdiff_results_filename + '.html')
cmd = [cls.CMD]
cmd.append('-hide-unchanged')
for version in ['old', 'new']:
old = results_store.get_build(version)
if old:
file_name = cls._create_xml(version, input_structure=old)
cmd.append(file_name)
cmd.append('-extra-info')
cmd.append(cls.results_dir)
cmd.append('-report-path')
cmd.append(cls.pkgdiff_results_full_path_html)
try:
ret_code = ProcessHelper.run_subprocess(
cmd, output_file=ProcessHelper.DEV_NULL)
except OSError as e:
raise CheckerNotFoundError(
"Checker '{}' was not found or installed.".format(
cls.name)) from e
# From pkgdiff source code:
# ret_code 0 means unchanged
# ret_code 1 means Changed
# other return codes means error
if int(ret_code) != 0 and int(ret_code) != 1:
raise RebaseHelperError(
'Execution of {} failed.\nCommand line is: {}'.format(
cls.CMD, cmd))
results_dict = cls.process_xml_results(cls.results_dir)
lines: List[str] = []
for key, val in results_dict.items():
if val:
if lines:
lines.append('')
lines.append('Following files were {}:'.format(key))
lines.extend(val)
pkgdiff_report = os.path.join(cls.results_dir,
cls.pkgdiff_results_filename + '.txt')
try:
with open(pkgdiff_report, "w", encoding=ENCODING) as f:
f.write('\n'.join(lines))
except IOError as e:
raise RebaseHelperError(
"Unable to write result from {} to '{}'".format(
cls.name, pkgdiff_report)) from e
return dict(path=cls.get_checker_output_dir_short())
def _build_rpm(cls,
srpm,
results_dir,
rpm_results_dir,
root=None,
arch=None,
builder_options=None):
"""Builds RPMs using mock.
Args:
srpm: Path to SRPM.
results_dir: Path to directory where logs will be placed.
rpm_results_dir: Path to directory where rpms will be placed.
root: Path to where chroot will be built
arch: Target architectures for the build.
builder_options: Additional options for mock.
Returns:
Tuple where first element is list of paths to built RPMs and
the second element is list of paths to logs.
"""
logger.info("Building RPMs")
output = os.path.join(results_dir, "mock_output.log")
cmd = [
cls.CMD, '--old-chroot', '--rebuild', srpm, '--resultdir',
results_dir
]
if root is not None:
cmd.extend(['--root', root])
if arch is not None:
cmd.extend(['--arch', arch])
if builder_options is not None:
cmd.extend(builder_options)
if not check_mock_privileges():
cmd = ['pkexec'] + cmd
ret = ProcessHelper.run_subprocess(cmd, output_file=output)
logs = []
for log in PathHelper.find_all_files(results_dir, '*.log'):
logs.append(os.path.join(rpm_results_dir, os.path.basename(log)))
if ret == 0:
return [
f for f in PathHelper.find_all_files(results_dir, '*.rpm')
if not f.endswith('.src.rpm')
], logs
else:
logfile = get_mock_logfile_path(ret,
rpm_results_dir,
tmp_path=results_dir)
raise BinaryPackageBuildError("Building RPMs failed!",
rpm_results_dir,
logfile=logfile,
logs=logs)
def _build_srpm(cls, spec, workdir, results_dir, srpm_results_dir,
srpm_builder_options):
"""Builds SRPM using mock.
Args:
spec: Path to SPEC file to build the SRPM from.
workdir: Path to working directory with rpmbuild directory structure.
results_dir: Path to directory where logs will be placed.
srpm_results_dir: Path to directory where SRPM will be placed.
srpm_builder_options: Additional mock options.
Returns:
Tuple, the first element is path to SRPM, the second is a list of paths
to logs.
"""
logger.info("Building SRPM")
spec_loc = os.path.dirname(spec)
output = os.path.join(results_dir, "build.log")
path_to_sources = os.path.join(workdir, 'SOURCES')
cmd = [cls.CMD, '--old-chroot', '--buildsrpm']
if srpm_builder_options is not None:
cmd.extend(srpm_builder_options)
cmd.extend(['--spec', spec])
cmd.extend(['--sources', path_to_sources])
cmd.extend(['--resultdir', results_dir])
if not check_mock_privileges():
cmd = ['pkexec'] + cmd
ret = ProcessHelper.run_subprocess_cwd_env(cmd,
cwd=spec_loc,
env={'HOME': workdir},
output_file=output)
build_log_path = os.path.join(srpm_results_dir, 'build.log')
mock_log_path = os.path.join(srpm_results_dir, 'mock_output.log')
root_log_path = os.path.join(srpm_results_dir, 'root.log')
logs = []
for log in PathHelper.find_all_files(results_dir, '*.log'):
logs.append(os.path.join(srpm_results_dir, os.path.basename(log)))
if ret == 0:
return PathHelper.find_first_file(workdir, '*.src.rpm'), logs
if ret == 1:
if not os.path.exists(build_log_path) and os.path.exists(
mock_log_path):
logfile = mock_log_path
else:
logfile = build_log_path
else:
logfile = root_log_path
raise SourcePackageBuildError("Building SRPM failed!",
logfile=logfile,
logs=logs)
def test_simple_cmd_changed_work_dir_with_redirected_output(self):
# create temp_file in temp_dir
self.test_simple_cmd_changed_work_dir()
ret = ProcessHelper.run_subprocess_cwd(self.LS_COMMAND,
self.TEMP_DIR,
output_file=self.OUT_FILE)
assert ret == 0
assert os.path.exists(os.path.join(self.TEMP_DIR, self.TEMP_FILE))
assert os.path.exists(self.OUT_FILE)
assert open(self.OUT_FILE).readline().strip("\n") == self.TEMP_FILE
def test_simple_cmd_changed_work_dir_with_redirected_output(self):
# create temp_file in temp_dir
self.test_simple_cmd_changed_work_dir()
ret = ProcessHelper.run_subprocess_cwd(self.LS_COMMAND,
self.TEMP_DIR,
output_file=self.OUT_FILE)
assert ret == 0
assert os.path.exists(os.path.join(self.TEMP_DIR, self.TEMP_FILE))
assert os.path.exists(self.OUT_FILE)
assert open(self.OUT_FILE).readline().strip("\n") == self.TEMP_FILE
def run_check(cls, results_dir, **kwargs):
"""Compares old and new RPMs using abipkgdiff"""
# Check if ABI changes occured
cls.abi_changes = None
cls.results_dir = os.path.join(results_dir, cls.name)
os.makedirs(cls.results_dir)
debug_old, rest_pkgs_old = cls._get_packages_for_abipkgdiff(
results_store.get_build('old'))
debug_new, rest_pkgs_new = cls._get_packages_for_abipkgdiff(
results_store.get_build('new'))
cmd = [cls.CMD]
reports = {}
for pkg in rest_pkgs_old:
command = list(cmd)
debug = cls._find_debuginfo(debug_old, pkg)
if debug:
command.append('--d1')
command.append(debug)
old_name = RpmHelper.split_nevra(os.path.basename(pkg))['name']
find = [
x for x in rest_pkgs_new if RpmHelper.split_nevra(
os.path.basename(x))['name'] == old_name
]
if not find:
logger.warning('New version of package %s was not found!',
old_name)
continue
new_pkg = find[0]
debug = cls._find_debuginfo(debug_new, new_pkg)
if debug:
command.append('--d2')
command.append(debug)
command.append(pkg)
command.append(new_pkg)
logger.verbose('Package name for ABI comparison %s', old_name)
output = os.path.join(cls.results_dir, old_name + '.txt')
try:
ret_code = ProcessHelper.run_subprocess(command,
output_file=output)
except OSError:
raise CheckerNotFoundError(
"Checker '{}' was not found or installed.".format(
cls.name))
if int(ret_code) & cls.ABIDIFF_ERROR and int(
ret_code) & cls.ABIDIFF_USAGE_ERROR:
raise RebaseHelperError(
'Execution of {} failed.\nCommand line is: {}'.format(
cls.CMD, cmd))
reports[old_name] = int(ret_code)
return dict(packages=cls.parse_abi_logs(reports),
abi_changes=cls.abi_changes,
path=cls.get_checker_output_dir_short())
def test_simple_cmd_with_input_path_and_redirected_output_path(self):
with open(self.IN_FILE, 'w') as f:
f.write(self.PHRASE)
assert os.path.exists(self.IN_FILE)
assert open(self.IN_FILE).readline().strip('\n') == self.PHRASE
ret = ProcessHelper.run_subprocess(self.CAT_COMMAND,
input_file=self.IN_FILE,
output_file=self.OUT_FILE)
assert ret == 0
assert os.path.exists(self.OUT_FILE)
assert open(self.OUT_FILE).readline().strip('\n') == self.PHRASE
def test_simple_cmd_with_input_path_and_redirected_output_path(self):
with open(self.IN_FILE, 'w') as f:
f.write(self.PHRASE)
assert os.path.exists(self.IN_FILE)
assert open(self.IN_FILE).readline().strip('\n') == self.PHRASE
ret = ProcessHelper.run_subprocess(self.CAT_COMMAND,
input_file=self.IN_FILE,
output_file=self.OUT_FILE)
assert ret == 0
assert os.path.exists(self.OUT_FILE)
assert open(self.OUT_FILE).readline().strip('\n') == self.PHRASE
def _build_rpm(cls,
srpm,
results_dir,
rpm_results_dir,
root=None,
arch=None,
builder_options=None):
"""
Build RPM using mock.
:param srpm: full path to the srpm.
:param results_dir: abs path to dir where the log should be placed.
:param rpm_results_dir: directory where rpms will be placed.
:param root: path to where chroot should be built.
:param arch: target architectures for the build.
:param builder_options: builder_options for mock.
:return abs paths to RPMs.
"""
logger.info("Building RPMs")
output = os.path.join(results_dir, "mock_output.log")
cmd = [
cls.CMD, '--old-chroot', '--rebuild', srpm, '--resultdir',
results_dir
]
if root is not None:
cmd.extend(['--root', root])
if arch is not None:
cmd.extend(['--arch', arch])
if builder_options is not None:
cmd.extend(builder_options)
ret = ProcessHelper.run_subprocess(cmd, output_file=output)
if ret == 0:
return [
f for f in PathHelper.find_all_files(results_dir, '*.rpm')
if not f.endswith('.src.rpm')
]
else:
logfile = MockBuildTool.get_mock_logfile_path(ret,
rpm_results_dir,
tmp_path=results_dir)
logs = [l for l in PathHelper.find_all_files(results_dir, '*.log')]
cls.logs.extend(
[os.path.join(rpm_results_dir, os.path.basename(l)) for l in logs])
raise BinaryPackageBuildError("Building RPMs failed!",
rpm_results_dir,
logfile=logfile)
def run_check(cls, results_dir, **kwargs):
"""
Compares old and new RPMs using pkgdiff
:param results_dir result dir where are stored results
"""
cls.results_dir = os.path.join(results_dir, cls.name)
os.makedirs(cls.results_dir)
cls.pkgdiff_results_full_path_html = os.path.join(cls.results_dir, cls.pkgdiff_results_filename + '.html')
cmd = [cls.CMD]
cmd.append('-hide-unchanged')
for version in ['old', 'new']:
old = results_store.get_build(version)
if old:
file_name = cls._create_xml(version, input_structure=old)
cmd.append(file_name)
cmd.append('-extra-info')
cmd.append(cls.results_dir)
cmd.append('-report-path')
cmd.append(cls.pkgdiff_results_full_path_html)
try:
ret_code = ProcessHelper.run_subprocess(cmd, output_file=ProcessHelper.DEV_NULL)
except OSError:
raise CheckerNotFoundError("Checker '{}' was not found or installed.".format(cls.name))
# From pkgdiff source code:
# ret_code 0 means unchanged
# ret_code 1 means Changed
# other return codes means error
if int(ret_code) != 0 and int(ret_code) != 1:
raise RebaseHelperError('Execution of {} failed.\nCommand line is: {}'.format(cls.CMD, cmd))
results_dict = cls.process_xml_results(cls.results_dir)
lines = []
for key, val in results_dict.items():
if val:
if lines:
lines.append('')
lines.append('Following files were {}:'.format(key))
lines.extend(val)
pkgdiff_report = os.path.join(cls.results_dir, cls.pkgdiff_results_filename + '.txt')
try:
with open(pkgdiff_report, "w") as f:
f.write('\n'.join(lines))
except IOError:
raise RebaseHelperError("Unable to write result from {} to '{}'".format(cls.name, pkgdiff_report))
return dict(path=cls.get_checker_output_dir_short())
def _build_source_from_instructions(cls, instructions, source, logfile):
"""Run instructions to create source archive"""
logger.info("Attempting to create source '%s' using instructions in comments", source)
with TemporaryEnvironment() as tmp:
script = os.path.join(tmp.path(), 'script.sh')
with open(script, 'w') as f:
f.write('#!/bin/sh -x\n')
f.write('{}\n'.format('\n'.join(instructions)))
f.write('cp "{}" "{}"\n'.format(source, os.getcwd()))
os.chmod(script, 0o755)
result = ProcessHelper.run_subprocess_cwd(script, tmp.path(), output_file=logfile, shell=True)
if result == 0 and os.path.isfile(source):
logger.info('Source creation succeeded.')
else:
logger.info('Source creation failed.')
def _build_source_from_instructions(cls, instructions, source, logfile):
"""Run instructions to create source archive"""
logger.info("Attempting to create source '%s' using instructions in comments", source)
with TemporaryEnvironment() as tmp:
script = os.path.join(tmp.path(), 'script.sh')
with open(script, 'w') as f:
f.write('#!/bin/sh -x\n')
f.write('{}\n'.format('\n'.join(instructions)))
f.write('cp "{}" "{}"\n'.format(source, os.getcwd()))
os.chmod(script, 0o755)
result = ProcessHelper.run_subprocess_cwd(script, tmp.path(), output_file=logfile, shell=True)
if result == 0 and os.path.isfile(source):
logger.info('Source creation succeeded.')
else:
logger.info('Source creation failed.')
def test_simple_cmd_with_input_fileobject_and_redirected_output_path(self):
in_buff = io.StringIO()
in_buff.write(self.PHRASE)
assert not os.path.exists(self.IN_FILE)
in_buff.seek(0)
assert in_buff.readline().strip('\n') == self.PHRASE
ret = ProcessHelper.run_subprocess(self.CAT_COMMAND,
input_file=in_buff,
output_file=self.OUT_FILE)
in_buff.close()
assert ret == 0
assert os.path.exists(self.OUT_FILE)
assert open(self.OUT_FILE).readline().strip('\n') == self.PHRASE
def test_simple_cmd_with_input_fileobject_and_redirected_output_path(
self):
in_buff = io.StringIO()
in_buff.write(self.PHRASE)
assert not os.path.exists(self.IN_FILE)
in_buff.seek(0)
assert in_buff.readline().strip('\n') == self.PHRASE
ret = ProcessHelper.run_subprocess(self.CAT_COMMAND,
input_file=in_buff,
output_file=self.OUT_FILE)
in_buff.close()
assert ret == 0
assert os.path.exists(self.OUT_FILE)
assert open(self.OUT_FILE).readline().strip('\n') == self.PHRASE
def run_check(cls, results_dir, **kwargs):
"""Compares old and new RPMs using abipkgdiff"""
# Check if ABI changes occured
cls.results_dir = os.path.join(results_dir, cls.name)
cls.prepare_results_dir()
debug_old, rest_pkgs_old = cls._get_packages_for_abipkgdiff(results_store.get_build('old'))
debug_new, rest_pkgs_new = cls._get_packages_for_abipkgdiff(results_store.get_build('new'))
ret_codes = {}
for pkg in rest_pkgs_old:
command = [cls.CMD]
command_fallback = [cls.CMD]
debug = cls._find_debuginfo(debug_old, pkg)
if debug:
command.append('--d1')
command.append(debug)
old_name = RpmHelper.split_nevra(os.path.basename(pkg))['name']
find = [x for x in rest_pkgs_new if RpmHelper.split_nevra(os.path.basename(x))['name'] == old_name]
if not find:
logger.warning('New version of package %s was not found!', old_name)
continue
new_pkg = find[0]
debug = cls._find_debuginfo(debug_new, new_pkg)
if debug:
command.append('--d2')
command.append(debug)
command.extend([pkg, new_pkg])
command_fallback.extend([pkg, new_pkg])
logger.verbose('Package name for ABI comparison %s', old_name)
output = os.path.join(cls.results_dir, old_name + '.txt')
for cmd in [command, command_fallback]:
try:
ret_code = ProcessHelper.run_subprocess(cmd, output_file=output)
except OSError as e:
raise CheckerNotFoundError("Checker '{}' was not found or installed.".format(cls.name)) from e
if int(ret_code) & cls.ABIDIFF_ERROR and int(ret_code) & cls.ABIDIFF_USAGE_ERROR:
raise RebaseHelperError(
'Execution of {} failed.\nCommand line is: {}'.format(cls.CMD, ' '.join(cmd)))
if int(ret_code) & cls.ABIDIFF_ERROR:
# abipkgdiff might not be able to read the debuginfo, try again without it
continue
break
ret_codes[old_name] = int(ret_code)
return dict(packages=cls.parse_abi_logs(ret_codes),
abi_changes=any(x & cls.ABIDIFF_ABI_CHANGE for x in ret_codes.values()),
abi_incompatible_changes=any(x & cls.ABIDIFF_ABI_INCOMPATIBLE_CHANGE for x in ret_codes.values()),
path=cls.get_checker_output_dir_short(),
ret_codes=ret_codes)
def _build_srpm(cls, spec, workdir, results_dir, srpm_results_dir, srpm_builder_options):
"""
Build SRPM using mock.
:param spec: abs path to SPEC file inside the rpmbuild/SPECS in workdir.
:param workdir: abs path to working directory with rpmbuild directory
structure, which will be used as HOME dir.
:param results_dir: abs path to dir where the log should be placed.
:param srpm_results_dir: path to directory where SRPM will be placed.
:param srpm_builder_options: list of additional options for mock build tool(eg. '-r fedora-XX-x86_64').
:return: abs path to built SRPM.
"""
logger.info("Building SRPM")
spec_loc = os.path.dirname(spec)
output = os.path.join(results_dir, "build.log")
path_to_sources = os.path.join(workdir, 'SOURCES')
cmd = ['mock', '--old-chroot', '--buildsrpm']
if srpm_builder_options is not None:
cmd.extend(srpm_builder_options)
cmd.extend(['--spec', spec])
cmd.extend(['--sources', path_to_sources])
cmd.extend(['--resultdir', results_dir])
ret = ProcessHelper.run_subprocess_cwd_env(cmd,
cwd=spec_loc,
env={'HOME': workdir},
output_file=output)
build_log_path = os.path.join(srpm_results_dir, 'build.log')
mock_log_path = os.path.join(srpm_results_dir, 'mock_output.log')
root_log_path = os.path.join(srpm_results_dir, 'root.log')
if ret == 0:
return PathHelper.find_first_file(workdir, '*.src.rpm')
if ret == 1:
if not os.path.exists(build_log_path) and os.path.exists(mock_log_path):
logfile = mock_log_path
else:
logfile = build_log_path
else:
logfile = root_log_path
logs = [l for l in PathHelper.find_all_files(results_dir, '*.log')]
cls.logs = [os.path.join(srpm_results_dir, os.path.basename(l)) for l in logs]
raise SourcePackageBuildError("Building SRPM failed!", logfile=logfile)
def _build_srpm(cls, spec, workdir, results_dir, srpm_results_dir, srpm_builder_options):
"""
Build SRPM using mock.
:param spec: abs path to SPEC file inside the rpmbuild/SPECS in workdir.
:param workdir: abs path to working directory with rpmbuild directory
structure, which will be used as HOME dir.
:param results_dir: abs path to dir where the log should be placed.
:param srpm_results_dir: path to directory where SRPM will be placed.
:param srpm_builder_options: list of additional options for mock build tool(eg. '-r fedora-XX-x86_64').
:return: abs path to built SRPM.
"""
logger.info("Building SRPM")
spec_loc = os.path.dirname(spec)
output = os.path.join(results_dir, "build.log")
path_to_sources = os.path.join(workdir, 'SOURCES')
cmd = ['mock', '--old-chroot', '--buildsrpm']
if srpm_builder_options is not None:
cmd.extend(srpm_builder_options)
cmd.extend(['--spec', spec])
cmd.extend(['--sources', path_to_sources])
cmd.extend(['--resultdir', results_dir])
ret = ProcessHelper.run_subprocess_cwd_env(cmd,
cwd=spec_loc,
env={'HOME': workdir},
output_file=output)
build_log_path = os.path.join(srpm_results_dir, 'build.log')
mock_log_path = os.path.join(srpm_results_dir, 'mock_output.log')
root_log_path = os.path.join(srpm_results_dir, 'root.log')
if ret == 0:
return PathHelper.find_first_file(workdir, '*.src.rpm')
if ret == 1:
if not os.path.exists(build_log_path) and os.path.exists(mock_log_path):
logfile = mock_log_path
else:
logfile = build_log_path
else:
logfile = root_log_path
logs = [l for l in PathHelper.find_all_files(results_dir, '*.log')]
cls.logs = [os.path.join(srpm_results_dir, os.path.basename(l)) for l in logs]
raise SourcePackageBuildError("Building SRPM failed!", logfile=logfile)
def _build_rpm(cls,
srpm,
workdir,
results_dir,
rpm_results_dir,
builder_options=None):
"""Builds RPMs using rpmbuild
Args:
srpm: Path to SRPM.
workdir: Path to working directory with rpmbuild directory structure.
results_dir: Path to directory where logs will be placed.
rpm_results_dir: Path to directory where RPMs will be placed.
builder_options: Additional options for rpmbuild.
Returns:
Tuple, the first element is a list of paths to built RPMs,
the second is a list of paths to logs.
"""
logger.info("Building RPMs")
output = os.path.join(results_dir, "build.log")
cmd = [cls.CMD, '--rebuild', srpm]
if builder_options is not None:
cmd.extend(builder_options)
ret = ProcessHelper.run_subprocess_cwd_env(cmd,
env={'HOME': workdir},
output_file=output)
build_log_path = os.path.join(rpm_results_dir, 'build.log')
logs = []
for log in PathHelper.find_all_files(results_dir, '*.log'):
logs.append(os.path.join(rpm_results_dir, os.path.basename(log)))
if ret == 0:
return [
f for f in PathHelper.find_all_files(workdir, '*.rpm')
if not f.endswith('.src.rpm')
], logs
# An error occurred, raise an exception
raise BinaryPackageBuildError("Building RPMs failed!",
results_dir,
logfile=build_log_path,
logs=logs)
def install_build_dependencies(spec_path=None, assume_yes=False):
"""Installs build dependencies of a package using dnf.
Args:
spec_path (str): Absolute path to the SPEC file.
assume_yes (bool): Whether to automatically answer 'yes' to all questions.
Returns:
int: Exit code of the subprocess run.
"""
cmd = ['dnf', 'builddep', spec_path]
if os.geteuid() != 0:
logger.warning("Authentication required to install build dependencies using '%s'", ' '.join(cmd))
cmd = ['pkexec'] + cmd
if assume_yes:
cmd.append('-y')
return ProcessHelper.run_subprocess(cmd)
def test_setting_new_env(self):
# make copy of existing environment
en_variables = os.environ.copy().keys()
# pick up non-existing name
while True:
rand_name = ''.join(random.choice(string.ascii_letters) for _ in range(6)).upper()
if rand_name not in en_variables:
break
cmd = 'echo "$' + rand_name + '"'
ret = ProcessHelper.run_subprocess_cwd_env(cmd,
env={rand_name: self.PHRASE},
output_file=self.OUT_FILE,
shell=True)
assert ret == 0
assert os.path.exists(self.OUT_FILE)
assert open(self.OUT_FILE).readline().strip("\n") == self.PHRASE
def test_setting_existing_env(self):
# make copy of existing environment
en_variables = list(os.environ.copy().keys())
# there are no variables set on the system -> nothing to test
if not en_variables:
pass
assert os.environ.get(en_variables[0]) != self.PHRASE
cmd = 'echo "$' + en_variables[0] + '"'
ret = ProcessHelper.run_subprocess_cwd_env(cmd,
env={en_variables[0]: self.PHRASE},
output_file=self.OUT_FILE,
shell=True)
assert ret == 0
assert os.path.exists(self.OUT_FILE)
assert open(self.OUT_FILE).readline().strip("\n") == self.PHRASE
def test_setting_existing_env(self):
# make copy of existing environment
en_variables = list(os.environ.copy().keys())
# there are no variables set on the system -> nothing to test
if not en_variables:
pass
assert os.environ.get(en_variables[0]) != self.PHRASE
cmd = 'echo "$' + en_variables[0] + '"'
ret = ProcessHelper.run_subprocess_cwd_env(
cmd,
env={en_variables[0]: self.PHRASE},
output_file=self.OUT_FILE,
shell=True)
assert ret == 0
assert os.path.exists(self.OUT_FILE)
assert open(self.OUT_FILE).readline().strip("\n") == self.PHRASE
def test_simple_cmd_with_input_path_and_redirected_output_fileobject(
self):
out_buff = io.StringIO()
with open(self.IN_FILE, 'w', encoding=ENCODING) as f:
f.write(self.PHRASE)
assert os.path.exists(self.IN_FILE)
assert open(
self.IN_FILE,
encoding=ENCODING).readline().strip('\n') == self.PHRASE
ret = ProcessHelper.run_subprocess(self.CAT_COMMAND,
input_file=self.IN_FILE,
output_file=out_buff)
assert ret == 0
assert not os.path.exists(self.OUT_FILE)
out_buff.seek(0)
assert out_buff.readline().strip('\n') == self.PHRASE
out_buff.close()
def run_check(cls, results_dir, **kwargs):
"""Compares old and new RPMs using abipkgdiff"""
# Check if ABI changes occured
cls.abi_changes = False
cls.results_dir = os.path.join(results_dir, cls.name)
os.makedirs(cls.results_dir)
debug_old, rest_pkgs_old = cls._get_packages_for_abipkgdiff(results_store.get_build('old'))
debug_new, rest_pkgs_new = cls._get_packages_for_abipkgdiff(results_store.get_build('new'))
cmd = [cls.CMD]
reports = {}
for pkg in rest_pkgs_old:
command = list(cmd)
debug = cls._find_debuginfo(debug_old, pkg)
if debug:
command.append('--d1')
command.append(debug)
old_name = RpmHelper.split_nevra(os.path.basename(pkg))['name']
find = [x for x in rest_pkgs_new if RpmHelper.split_nevra(os.path.basename(x))['name'] == old_name]
if not find:
logger.warning('New version of package %s was not found!', old_name)
continue
new_pkg = find[0]
debug = cls._find_debuginfo(debug_new, new_pkg)
if debug:
command.append('--d2')
command.append(debug)
command.append(pkg)
command.append(new_pkg)
logger.verbose('Package name for ABI comparison %s', old_name)
output = os.path.join(cls.results_dir, old_name + '.txt')
try:
ret_code = ProcessHelper.run_subprocess(command, output_file=output)
except OSError:
raise CheckerNotFoundError("Checker '{}' was not found or installed.".format(cls.name))
if int(ret_code) & cls.ABIDIFF_ERROR and int(ret_code) & cls.ABIDIFF_USAGE_ERROR:
raise RebaseHelperError('Execution of {} failed.\nCommand line is: {}'.format(cls.CMD, cmd))
reports[old_name] = int(ret_code)
return dict(packages=cls.parse_abi_logs(reports),
abi_changes=cls.abi_changes,
path=cls.get_checker_output_dir_short())
def test_setting_new_env(self):
# make copy of existing environment
en_variables = os.environ.copy().keys()
# pick up non-existing name
while True:
rand_name = ''.join(
random.choice(string.ascii_letters)
for _ in range(6)).upper()
if rand_name not in en_variables:
break
cmd = 'echo "$' + rand_name + '"'
ret = ProcessHelper.run_subprocess_cwd_env(
cmd,
env={rand_name: self.PHRASE},
output_file=self.OUT_FILE,
shell=True)
assert ret == 0
assert os.path.exists(self.OUT_FILE)
assert open(self.OUT_FILE).readline().strip("\n") == self.PHRASE
def run_rpminspect(cls, checker_dir: str, old_pkg: str,
new_pkg: str) -> Tuple[str, Dict[str, int]]:
"""Runs rpminspect on the given package.
Args:
checker_dir: Path to the results directory of the checker
old_pkg: Path to the old package.
new_pkg: Path to the new package.
Returns:
Tuple of (path, data), where path is the path to rpminspect output
and data is the dict of count of OK/BAD/VERIFY checks.
"""
cmd = [cls.CMD, '-F', 'json']
pkg_name = RpmHelper.split_nevra(os.path.basename(new_pkg))['name']
cmd.append(old_pkg)
cmd.append(new_pkg)
outfile = os.path.join(checker_dir, '{}.json'.format(pkg_name))
try:
ret = ProcessHelper.run_subprocess(cmd,
output_file=outfile,
ignore_stderr=True)
except OSError as e:
raise CheckerNotFoundError(
'Checker \'{}\' was not found or installed.'.format(
cls.name)) from e
# Exit code 1 is used when bad check is found, 0 when everything is OK. Others on error
if ret not in (0, 1):
raise RebaseHelperError(
'An error occurred when running checker \'{}\''.format(
cls.name))
with open(outfile, 'r', encoding=ENCODING) as json_file:
data = json.load(json_file)
return outfile, cls.process_data(data)
def _build_srpm(cls, spec, workdir, results_dir, srpm_results_dir,
srpm_builder_options):
"""
Build SRPM using rpmbuild.
:param spec: abs path to SPEC file inside the rpmbuild/SPECS in workdir.
:param workdir: abs path to working directory with rpmbuild directory
structure, which will be used as HOME dir.
:param results_dir: abs path to dir where the log should be placed.
:param srpm_results_dir: path to directory where SRPM will be placed.
:param srpm_builder_options: list of additional options to rpmbuild.
:return: abs path to built SRPM.
"""
logger.info("Building SRPM")
spec_loc, spec_name = os.path.split(spec)
output = os.path.join(results_dir, "build.log")
cmd = ['rpmbuild', '-bs', spec_name]
if srpm_builder_options is not None:
cmd.extend(srpm_builder_options)
ret = ProcessHelper.run_subprocess_cwd_env(cmd,
cwd=spec_loc,
env={'HOME': workdir},
output_file=output)
build_log_path = os.path.join(srpm_results_dir, 'build.log')
if ret == 0:
return PathHelper.find_first_file(workdir, '*.src.rpm')
# An error occurred, raise an exception
logfile = build_log_path
logs = [l for l in PathHelper.find_all_files(results_dir, '*.log')]
cls.logs = [
os.path.join(srpm_results_dir, os.path.basename(l)) for l in logs
]
raise SourcePackageBuildError("Building SRPM failed!", logfile=logfile)
def _build_srpm(cls, spec, workdir, results_dir, srpm_results_dir, srpm_builder_options):
"""Builds SRPM using rpmbuild.
Args:
spec: Path to SPEC file to build the SRPM from.
workdir: Path to working directory with rpmbuild directory structure.
results_dir: Path to directory where logs will be placed.
srpm_results_dir: Path to directory where SRPM will be placed.
srpm_builder_options: Additional rpmbuild options.
Returns:
Tuple, the first element is path to SRPM, the second is a list of paths
to logs.
"""
logger.info("Building SRPM")
spec_loc, spec_name = os.path.split(spec)
output = os.path.join(results_dir, "build.log")
cmd = [cls.CMD, '-bs', spec_name]
if srpm_builder_options is not None:
cmd.extend(srpm_builder_options)
ret = ProcessHelper.run_subprocess_cwd_env(cmd,
cwd=spec_loc,
env={'HOME': workdir},
output_file=output)
build_log_path = os.path.join(srpm_results_dir, 'build.log')
logs = []
for l in PathHelper.find_all_files(results_dir, '*.log'):
logs.append(os.path.join(srpm_results_dir, os.path.basename(l)))
if ret == 0:
return PathHelper.find_first_file(workdir, '*.src.rpm'), logs
# An error occurred, raise an exception
raise SourcePackageBuildError("Building SRPM failed!", logfile=build_log_path, logs=logs)
def get_license_changes(cls, old_dir, new_dir):
"""
Finds differences in licenses between old and new source files.
Args:
old_dir(str): Path to the old sources directory.
new_dir(str): Path to the new sources directory.
Returns:
tuple: Changes dictionary, new_licenses set, disappeared_licenses set.
"""
diffs = []
possible_license_names = r'COPYING|LICENSE|LICENCE|LICENSING|BSD|(L)?GPL(v[23][+]?)?'
for source_dir in [old_dir, new_dir]:
out = io.StringIO()
ProcessHelper.run_subprocess(["/usr/bin/licensecheck", source_dir, "--machine", "--recursive"],
output_file=out, ignore_stderr=True)
diff = {}
for l in out:
# licensecheck output format: 'Filepath\tlicense'
file_path, dlicense = l.split('\t')
if 'GENERATED FILE' in dlicense:
continue
file_path = os.path.relpath(file_path, source_dir)
diff[file_path] = dlicense.strip()
diffs.append(diff)
old_lics, new_lics = set(), set()
changes = {'appeared': {}, 'transitioned': {}, 'disappeared': {}}
# Get changed licenses in existing files
for new_file, new_license in diffs[1].items():
new_lics.add(new_license)
for old_file, old_license in diffs[0].items():
old_lics.add(old_license)
if (new_file == old_file and
(new_license != old_license)):
new_key = '{} => {}'.format(old_license, new_license)
if new_license == 'UNKNOWN':
# Conversion `known license` => `None/Unknown`
if re.search(possible_license_names, new_file, re.IGNORECASE):
cls.license_files_changes.update({old_license: 'disappeared in {}'.format(new_file)})
if old_license not in changes['disappeared']:
changes['disappeared'][old_license] = []
changes['disappeared'][old_license].append(new_file)
elif old_license == 'UNKNOWN':
# Conversion `None/Unknown` => `known license`
if re.search(possible_license_names, new_file, re.IGNORECASE):
cls.license_files_changes.update({new_license: 'appeared in {}'.format(new_file)})
if new_license not in changes['appeared']:
changes['appeared'][new_license] = []
changes['appeared'][new_license].append(new_file)
else:
# Conversion `known license` => `known license`
if re.search(possible_license_names, new_file, re.IGNORECASE):
cls.license_files_changes.update({new_key: 'in {}'.format(new_file)})
if new_key not in changes['transitioned']:
changes['transitioned'][new_key] = []
if new_file not in changes['transitioned'][new_key]:
changes['transitioned'][new_key].append(new_file)
# Get newly appeared files
for new_file, new_license in diffs[1].items():
if new_file not in diffs[0]:
if new_license == 'UNKNOWN':
continue
if re.search(possible_license_names, new_file, re.IGNORECASE):
cls.license_files_changes.update({new_license: 'appeared in {}'.format(new_file)})
if new_license not in changes['appeared']:
changes['appeared'][new_license] = []
changes['appeared'][new_license].append(new_file)
# Get removed files
for old_file, old_license in diffs[0].items():
if old_file not in diffs[1]:
if old_license == 'UNKNOWN':
continue
if re.search(possible_license_names, old_file, re.IGNORECASE):
cls.license_files_changes.update({old_license: 'disappeared in {}'.format(old_file)})
if old_license not in changes['disappeared']:
changes['disappeared'][old_license] = []
changes['disappeared'][old_license].append(old_file)
new_licenses = new_lics - old_lics
disappeared_licenses = old_lics - new_lics
if new_licenses or disappeared_licenses:
cls.license_changes = True
return changes, list(new_licenses), list(disappeared_licenses)
def get_license_changes(cls, old_dir, new_dir):
"""
Finds differences in licenses between old and new source files.
Args:
old_dir(str): Path to the old sources directory.
new_dir(str): Path to the new sources directory.
Returns:
tuple: Changes dictionary, new_licenses set, disappeared_licenses set.
"""
diffs = []
possible_license_names = r'COPYING|LICENSE|LICENCE|LICENSING|BSD|(L)?GPL(v[23][+]?)?'
for source_dir in [old_dir, new_dir]:
out = six.StringIO()
ProcessHelper.run_subprocess([
"/usr/bin/licensecheck", source_dir, "--machine", "--recursive"
],
output_file=out,
ignore_stderr=True)
diff = {}
for l in out:
# licensecheck output format: 'Filepath\tlicense'
file_path, dlicense = l.split('\t')
if 'GENERATED FILE' in dlicense:
continue
file_path = os.path.relpath(file_path, source_dir)
diff[file_path] = dlicense.strip()
diffs.append(diff)
old_lics, new_lics = set(), set()
changes = {'appeared': {}, 'transitioned': {}, 'disappeared': {}}
# Get changed licenses in existing files
for new_file, new_license in six.iteritems(diffs[1]):
new_lics.add(new_license)
for old_file, old_license in six.iteritems(diffs[0]):
old_lics.add(old_license)
if (new_file == old_file and (new_license != old_license)):
new_key = '{} => {}'.format(old_license, new_license)
if new_license == 'UNKNOWN':
# Conversion `known license` => `None/Unknown`
if re.search(possible_license_names, new_file,
re.IGNORECASE):
cls.license_files_changes.update({
old_license:
'disappeared in {}'.format(new_file)
})
if old_license not in changes['disappeared']:
changes['disappeared'][old_license] = []
changes['disappeared'][old_license].append(new_file)
elif old_license == 'UNKNOWN':
# Conversion `None/Unknown` => `known license`
if re.search(possible_license_names, new_file,
re.IGNORECASE):
cls.license_files_changes.update({
new_license:
'appeared in {}'.format(new_file)
})
if new_license not in changes['appeared']:
changes['appeared'][new_license] = []
changes['appeared'][new_license].append(new_file)
else:
# Conversion `known license` => `known license`
if re.search(possible_license_names, new_file,
re.IGNORECASE):
cls.license_files_changes.update(
{new_key: 'in {}'.format(new_file)})
if new_key not in changes['transitioned']:
changes['transitioned'][new_key] = []
if new_file not in changes['transitioned'][new_key]:
changes['transitioned'][new_key].append(new_file)
# Get newly appeared files
for new_file, new_license in six.iteritems(diffs[1]):
if new_file not in diffs[0]:
if new_license == 'UNKNOWN':
continue
if re.search(possible_license_names, new_file, re.IGNORECASE):
cls.license_files_changes.update(
{new_license: 'appeared in {}'.format(new_file)})
if new_license not in changes['appeared']:
changes['appeared'][new_license] = []
changes['appeared'][new_license].append(new_file)
# Get removed files
for old_file, old_license in six.iteritems(diffs[0]):
if old_file not in diffs[1]:
if old_license == 'UNKNOWN':
continue
if re.search(possible_license_names, old_file, re.IGNORECASE):
cls.license_files_changes.update(
{old_license: 'disappeared in {}'.format(old_file)})
if old_license not in changes['disappeared']:
changes['disappeared'][old_license] = []
changes['disappeared'][old_license].append(old_file)
new_licenses = new_lics - old_lics
disappeared_licenses = old_lics - new_lics
if new_licenses or disappeared_licenses:
cls.license_changes = True
return changes, list(new_licenses), list(disappeared_licenses)
def is_available(cls):
try:
return ProcessHelper.run_subprocess([cls.CMD, '--help'], output_file=ProcessHelper.DEV_NULL) == 0
except (IOError, OSError):
return False
def test_simple_cmd_changed_work_dir(self):
os.mkdir(self.TEMP_DIR)
ret = ProcessHelper.run_subprocess_cwd(self.TOUCH_COMMAND,
self.TEMP_DIR)
assert ret == 0
assert os.path.exists(os.path.join(self.TEMP_DIR, self.TEMP_FILE))
def test_simple_cmd_with_redirected_output_path(self):
ret = ProcessHelper.run_subprocess(self.ECHO_COMMAND,
output_file=self.OUT_FILE)
assert ret == 0
assert os.path.exists(self.OUT_FILE)
assert open(self.OUT_FILE).readline().strip('\n') == self.PHRASE
def test_simple_cmd_changed_work_dir(self):
os.mkdir(self.TEMP_DIR)
ret = ProcessHelper.run_subprocess_cwd(self.TOUCH_COMMAND,
self.TEMP_DIR)
assert ret == 0
assert os.path.exists(os.path.join(self.TEMP_DIR, self.TEMP_FILE))
def test_simple_cmd(self):
ret = ProcessHelper.run_subprocess(self.TOUCH_COMMAND)
assert ret == 0
assert os.path.exists(self.TEMP_FILE)
def test_simple_cmd(self):
ret = ProcessHelper.run_subprocess(self.TOUCH_COMMAND)
assert ret == 0
assert os.path.exists(self.TEMP_FILE)
