def run_check(cls, results_dir, **kwargs): """Compares old and new RPMs using pkgdiff""" csmock_report = {} old_pkgs = results_store.get_old_build().get('srpm', None) new_pkgs = results_store.get_new_build().get('srpm', None) results_dir = os.path.join(results_dir, cls.CMD) os.makedirs(results_dir) arguments = ['--force', '-a', '-r', 'fedora-rawhide-x86_64', '--base-srpm'] if old_pkgs and new_pkgs: cmd = [cls.CMD] cmd.extend(arguments) cmd.append(old_pkgs) cmd.append(new_pkgs) cmd.extend(['-o', results_dir]) output = io.StringIO() try: ProcessHelper.run_subprocess(cmd, output_file=output) except OSError: raise CheckerNotFoundError("Checker '{}' was not found or installed.".format(cls.name)) csmock_report['error'] = PathHelper.find_all_files_current_dir(results_dir, '*.err') csmock_report['txt'] = PathHelper.find_all_files_current_dir(results_dir, '*.txt') csmock_report['log'] = PathHelper.find_all_files_current_dir(results_dir, '*.log') csmock_report['path'] = cls.get_checker_output_dir_short() return csmock_report
def run_check(cls, results_dir, **kwargs): """Compares old and new RPMs using pkgdiff""" csmock_report = {} old_pkgs = results_store.get_old_build().get('srpm', None) new_pkgs = results_store.get_new_build().get('srpm', None) cls.results_dir = os.path.join(results_dir, cls.CMD) cls.prepare_results_dir() arguments = [ '--force', '-a', '-r', 'fedora-rawhide-x86_64', '--base-srpm' ] if old_pkgs and new_pkgs: cmd = [cls.CMD] cmd.extend(arguments) cmd.append(old_pkgs) cmd.append(new_pkgs) cmd.extend(['-o', results_dir]) output = io.StringIO() try: ProcessHelper.run_subprocess(cmd, output_file=output) except OSError as e: raise CheckerNotFoundError( "Checker '{}' was not found or installed.".format( cls.name)) from e csmock_report['error'] = PathHelper.find_all_files_current_dir( results_dir, '*.err') csmock_report['txt'] = PathHelper.find_all_files_current_dir( results_dir, '*.txt') csmock_report['log'] = PathHelper.find_all_files_current_dir( results_dir, '*.log') csmock_report['path'] = cls.get_checker_output_dir_short() return csmock_report
def run_mergetool(cls, repo): # we can't use GitPython here, as it doesn't allow # for the command to attach to stdout directly cwd = os.getcwd() try: os.chdir(repo.working_tree_dir) ProcessHelper.run_subprocess(['git', 'mergetool']) finally: os.chdir(cwd)
def run_check(cls, results_dir, **kwargs): """Compares old and new RPMs using rpmdiff""" results_dict = {} for tag in cls.CHECKER_TAGS: results_dict[tag] = [] cls.results_dir = os.path.join(results_dir, cls.name) os.makedirs(cls.results_dir) # Only S (size), M(mode) and 5 (checksum) are now important not_catched_flags = ['T', 'F', 'G', 'U', 'V', 'L', 'D', 'N'] old_pkgs = cls._get_rpms(results_store.get_old_build().get('rpm', None)) new_pkgs = cls._get_rpms(results_store.get_new_build().get('rpm', None)) for key, value in old_pkgs.items(): if 'debuginfo' in key or 'debugsource' in key: # skip debug{info,source} packages continue cmd = [cls.CMD] # TODO modify to online command for x in not_catched_flags: cmd.extend(['-i', x]) cmd.append(value) # We would like to build correct old package against correct new packages try: cmd.append(new_pkgs[key]) except KeyError: logger.warning('New version of package %s was not found!', key) continue output = io.StringIO() try: ProcessHelper.run_subprocess(cmd, output_file=output) except OSError: raise CheckerNotFoundError("Checker '{}' was not found or installed.".format(cls.name)) results_dict = cls._analyze_logs(output, results_dict) results_dict = cls.update_added_removed(results_dict) cls.results_dict = {k: v for k, v in results_dict.items() if v} lines = [] for key, val in results_dict.items(): if val: if lines: lines.append('') lines.append('Following files were {}:'.format(key)) lines.extend(val) rpmdiff_report = os.path.join(cls.results_dir, 'report.txt') counts = {k: len(v) for k, v in results_dict.items()} try: with open(rpmdiff_report, "w") as f: f.write('\n'.join(lines)) except IOError: raise RebaseHelperError("Unable to write result from {} to '{}'".format(cls.name, rpmdiff_report)) return {'path': cls.get_checker_output_dir_short(), 'files_changes': counts}
def run_check(cls, results_dir, **kwargs): """Compares old and new RPMs using rpmdiff""" results_dict: Dict[str, List[str]] = {} for tag in cls.CHECKER_TAGS: results_dict[tag] = [] cls.results_dir = os.path.join(results_dir, cls.name) cls.prepare_results_dir() # Only S (size), M(mode) and 5 (checksum) are now important not_catched_flags = ['T', 'F', 'G', 'U', 'V', 'L', 'D', 'N'] old_pkgs = cls._get_rpms(results_store.get_old_build().get('rpm', None)) new_pkgs = cls._get_rpms(results_store.get_new_build().get('rpm', None)) for key, value in old_pkgs.items(): if 'debuginfo' in key or 'debugsource' in key: # skip debug{info,source} packages continue cmd = [cls.CMD] # TODO modify to online command for x in not_catched_flags: cmd.extend(['--ignore={0}'.format(x)]) cmd.append(value) # We would like to build correct old package against correct new packages try: cmd.append(new_pkgs[key]) except KeyError: logger.warning('New version of package %s was not found!', key) continue output = io.StringIO() try: ProcessHelper.run_subprocess(cmd, output_file=output) except OSError as e: raise CheckerNotFoundError("Checker '{}' was not found or installed.".format(cls.name)) from e results_dict = cls._analyze_logs(output, results_dict) results_dict = cls.update_added_removed(results_dict) cls.results_dict = {k: v for k, v in results_dict.items() if v} lines: List[str] = [] for key, val in results_dict.items(): if val: if lines: lines.append('') lines.append('Following files were {}:'.format(key)) lines.extend(val) rpmdiff_report = os.path.join(cls.results_dir, 'report.txt') counts = {k: len(v) for k, v in results_dict.items()} try: with open(rpmdiff_report, "w", encoding=ENCODING) as f: f.write('\n'.join(lines)) except IOError as e: raise RebaseHelperError("Unable to write result from {} to '{}'".format(cls.name, rpmdiff_report)) from e return {'path': cls.get_checker_output_dir_short(), 'files_changes': counts}
def _build_rpm(cls, srpm, workdir, results_dir, rpm_results_dir, builder_options=None): """ Build RPM using rpmbuild. :param srpm: abs path to SRPM :param workdir: abs path to working directory with rpmbuild directory structure, which will be used as HOME dir. :param results_dir: abs path to dir where the log should be placed. :param rpm_results_dir: path directory to where RPMs will be placed. :return: abs paths to built RPMs. """ logger.info("Building RPMs") output = os.path.join(results_dir, "build.log") cmd = [cls.CMD, '--rebuild', srpm] if builder_options is not None: cmd.extend(builder_options) ret = ProcessHelper.run_subprocess_cwd_env(cmd, env={'HOME': workdir}, output_file=output) build_log_path = os.path.join(rpm_results_dir, 'build.log') if ret == 0: return [f for f in PathHelper.find_all_files(workdir, '*.rpm') if not f.endswith('.src.rpm')] # An error occurred, raise an exception logfile = build_log_path logs = [l for l in PathHelper.find_all_files(results_dir, '*.log')] cls.logs.extend([os.path.join(rpm_results_dir, os.path.basename(l)) for l in logs]) raise BinaryPackageBuildError("Building RPMs failed!", results_dir, logfile=logfile)
def is_available(cls): try: return ProcessHelper.run_subprocess( [cls.CMD, '--version'], output_file=ProcessHelper.DEV_NULL) == 0 except (IOError, OSError): return False
def _build_rpm(cls, srpm, results_dir, rpm_results_dir, root=None, arch=None, builder_options=None): """ Build RPM using mock. :param srpm: full path to the srpm. :param results_dir: abs path to dir where the log should be placed. :param rpm_results_dir: directory where rpms will be placed. :param root: path to where chroot should be built. :param arch: target architectures for the build. :param builder_options: builder_options for mock. :return abs paths to RPMs. """ logger.info("Building RPMs") output = os.path.join(results_dir, "mock_output.log") cmd = [cls.CMD, '--old-chroot', '--rebuild', srpm, '--resultdir', results_dir] if root is not None: cmd.extend(['--root', root]) if arch is not None: cmd.extend(['--arch', arch]) if builder_options is not None: cmd.extend(builder_options) ret = ProcessHelper.run_subprocess(cmd, output_file=output) if ret == 0: return [f for f in PathHelper.find_all_files(results_dir, '*.rpm') if not f.endswith('.src.rpm')] else: logfile = Mock.get_mock_logfile_path(ret, rpm_results_dir, tmp_path=results_dir) logs = [l for l in PathHelper.find_all_files(results_dir, '*.log')] cls.logs.extend(os.path.join(rpm_results_dir, os.path.basename(l)) for l in logs) raise BinaryPackageBuildError("Building RPMs failed!", rpm_results_dir, logfile=logfile)
def _build_srpm(cls, spec, workdir, results_dir, srpm_results_dir, srpm_builder_options): """ Build SRPM using rpmbuild. :param spec: abs path to SPEC file inside the rpmbuild/SPECS in workdir. :param workdir: abs path to working directory with rpmbuild directory structure, which will be used as HOME dir. :param results_dir: abs path to dir where the log should be placed. :param srpm_results_dir: path to directory where SRPM will be placed. :param srpm_builder_options: list of additional options to rpmbuild. :return: abs path to built SRPM. """ logger.info("Building SRPM") spec_loc, spec_name = os.path.split(spec) output = os.path.join(results_dir, "build.log") cmd = ['rpmbuild', '-bs', spec_name] if srpm_builder_options is not None: cmd.extend(srpm_builder_options) ret = ProcessHelper.run_subprocess_cwd_env(cmd, cwd=spec_loc, env={'HOME': workdir}, output_file=output) build_log_path = os.path.join(srpm_results_dir, 'build.log') if ret == 0: return PathHelper.find_first_file(workdir, '*.src.rpm') # An error occurred, raise an exception logfile = build_log_path logs = [l for l in PathHelper.find_all_files(results_dir, '*.log')] cls.logs = [os.path.join(srpm_results_dir, os.path.basename(l)) for l in logs] raise SourcePackageBuildError("Building SRPM failed!", logfile=logfile)
def _build_rpm(cls, srpm, workdir, results_dir, rpm_results_dir, builder_options=None): """ Build RPM using rpmbuild. :param srpm: abs path to SRPM :param workdir: abs path to working directory with rpmbuild directory structure, which will be used as HOME dir. :param results_dir: abs path to dir where the log should be placed. :param rpm_results_dir: path directory to where RPMs will be placed. :return: abs paths to built RPMs. """ logger.info("Building RPMs") output = os.path.join(results_dir, "build.log") cmd = [cls.CMD, '--rebuild', srpm] if builder_options is not None: cmd.extend(builder_options) ret = ProcessHelper.run_subprocess_cwd_env(cmd, env={'HOME': workdir}, output_file=output) build_log_path = os.path.join(rpm_results_dir, 'build.log') if ret == 0: return [f for f in PathHelper.find_all_files(workdir, '*.rpm') if not f.endswith('.src.rpm')] # An error occurred, raise an exception logfile = build_log_path logs = [l for l in PathHelper.find_all_files(results_dir, '*.log')] cls.logs.extend(os.path.join(rpm_results_dir, os.path.basename(l)) for l in logs) raise BinaryPackageBuildError("Building RPMs failed!", results_dir, logfile=logfile)
def test_simple_cmd_with_redirected_output_fileobject(self): buff = io.StringIO() ret = ProcessHelper.run_subprocess(self.ECHO_COMMAND, output_file=buff) assert ret == 0 assert not os.path.exists(self.OUT_FILE) assert buff.readline().strip('\n') == self.PHRASE buff.close()
def test_simple_cmd_with_redirected_output_path(self): ret = ProcessHelper.run_subprocess(self.ECHO_COMMAND, output_file=self.OUT_FILE) assert ret == 0 assert os.path.exists(self.OUT_FILE) assert open( self.OUT_FILE, encoding=ENCODING).readline().strip('\n') == self.PHRASE
def run_check(cls, results_dir, **kwargs): """ Compares old and new RPMs using pkgdiff :param results_dir result dir where are stored results """ cls.results_dir = os.path.join(results_dir, cls.name) cls.prepare_results_dir() cls.pkgdiff_results_full_path_html = os.path.join( cls.results_dir, cls.pkgdiff_results_filename + '.html') cmd = [cls.CMD] cmd.append('-hide-unchanged') for version in ['old', 'new']: old = results_store.get_build(version) if old: file_name = cls._create_xml(version, input_structure=old) cmd.append(file_name) cmd.append('-extra-info') cmd.append(cls.results_dir) cmd.append('-report-path') cmd.append(cls.pkgdiff_results_full_path_html) try: ret_code = ProcessHelper.run_subprocess( cmd, output_file=ProcessHelper.DEV_NULL) except OSError as e: raise CheckerNotFoundError( "Checker '{}' was not found or installed.".format( cls.name)) from e # From pkgdiff source code: # ret_code 0 means unchanged # ret_code 1 means Changed # other return codes means error if int(ret_code) != 0 and int(ret_code) != 1: raise RebaseHelperError( 'Execution of {} failed.\nCommand line is: {}'.format( cls.CMD, cmd)) results_dict = cls.process_xml_results(cls.results_dir) lines: List[str] = [] for key, val in results_dict.items(): if val: if lines: lines.append('') lines.append('Following files were {}:'.format(key)) lines.extend(val) pkgdiff_report = os.path.join(cls.results_dir, cls.pkgdiff_results_filename + '.txt') try: with open(pkgdiff_report, "w", encoding=ENCODING) as f: f.write('\n'.join(lines)) except IOError as e: raise RebaseHelperError( "Unable to write result from {} to '{}'".format( cls.name, pkgdiff_report)) from e return dict(path=cls.get_checker_output_dir_short())
def _build_rpm(cls, srpm, results_dir, rpm_results_dir, root=None, arch=None, builder_options=None): """Builds RPMs using mock. Args: srpm: Path to SRPM. results_dir: Path to directory where logs will be placed. rpm_results_dir: Path to directory where rpms will be placed. root: Path to where chroot will be built arch: Target architectures for the build. builder_options: Additional options for mock. Returns: Tuple where first element is list of paths to built RPMs and the second element is list of paths to logs. """ logger.info("Building RPMs") output = os.path.join(results_dir, "mock_output.log") cmd = [ cls.CMD, '--old-chroot', '--rebuild', srpm, '--resultdir', results_dir ] if root is not None: cmd.extend(['--root', root]) if arch is not None: cmd.extend(['--arch', arch]) if builder_options is not None: cmd.extend(builder_options) if not check_mock_privileges(): cmd = ['pkexec'] + cmd ret = ProcessHelper.run_subprocess(cmd, output_file=output) logs = [] for log in PathHelper.find_all_files(results_dir, '*.log'): logs.append(os.path.join(rpm_results_dir, os.path.basename(log))) if ret == 0: return [ f for f in PathHelper.find_all_files(results_dir, '*.rpm') if not f.endswith('.src.rpm') ], logs else: logfile = get_mock_logfile_path(ret, rpm_results_dir, tmp_path=results_dir) raise BinaryPackageBuildError("Building RPMs failed!", rpm_results_dir, logfile=logfile, logs=logs)
def _build_srpm(cls, spec, workdir, results_dir, srpm_results_dir, srpm_builder_options): """Builds SRPM using mock. Args: spec: Path to SPEC file to build the SRPM from. workdir: Path to working directory with rpmbuild directory structure. results_dir: Path to directory where logs will be placed. srpm_results_dir: Path to directory where SRPM will be placed. srpm_builder_options: Additional mock options. Returns: Tuple, the first element is path to SRPM, the second is a list of paths to logs. """ logger.info("Building SRPM") spec_loc = os.path.dirname(spec) output = os.path.join(results_dir, "build.log") path_to_sources = os.path.join(workdir, 'SOURCES') cmd = [cls.CMD, '--old-chroot', '--buildsrpm'] if srpm_builder_options is not None: cmd.extend(srpm_builder_options) cmd.extend(['--spec', spec]) cmd.extend(['--sources', path_to_sources]) cmd.extend(['--resultdir', results_dir]) if not check_mock_privileges(): cmd = ['pkexec'] + cmd ret = ProcessHelper.run_subprocess_cwd_env(cmd, cwd=spec_loc, env={'HOME': workdir}, output_file=output) build_log_path = os.path.join(srpm_results_dir, 'build.log') mock_log_path = os.path.join(srpm_results_dir, 'mock_output.log') root_log_path = os.path.join(srpm_results_dir, 'root.log') logs = [] for log in PathHelper.find_all_files(results_dir, '*.log'): logs.append(os.path.join(srpm_results_dir, os.path.basename(log))) if ret == 0: return PathHelper.find_first_file(workdir, '*.src.rpm'), logs if ret == 1: if not os.path.exists(build_log_path) and os.path.exists( mock_log_path): logfile = mock_log_path else: logfile = build_log_path else: logfile = root_log_path raise SourcePackageBuildError("Building SRPM failed!", logfile=logfile, logs=logs)
def test_simple_cmd_changed_work_dir_with_redirected_output(self): # create temp_file in temp_dir self.test_simple_cmd_changed_work_dir() ret = ProcessHelper.run_subprocess_cwd(self.LS_COMMAND, self.TEMP_DIR, output_file=self.OUT_FILE) assert ret == 0 assert os.path.exists(os.path.join(self.TEMP_DIR, self.TEMP_FILE)) assert os.path.exists(self.OUT_FILE) assert open(self.OUT_FILE).readline().strip("\n") == self.TEMP_FILE
def run_check(cls, results_dir, **kwargs): """Compares old and new RPMs using abipkgdiff""" # Check if ABI changes occured cls.abi_changes = None cls.results_dir = os.path.join(results_dir, cls.name) os.makedirs(cls.results_dir) debug_old, rest_pkgs_old = cls._get_packages_for_abipkgdiff( results_store.get_build('old')) debug_new, rest_pkgs_new = cls._get_packages_for_abipkgdiff( results_store.get_build('new')) cmd = [cls.CMD] reports = {} for pkg in rest_pkgs_old: command = list(cmd) debug = cls._find_debuginfo(debug_old, pkg) if debug: command.append('--d1') command.append(debug) old_name = RpmHelper.split_nevra(os.path.basename(pkg))['name'] find = [ x for x in rest_pkgs_new if RpmHelper.split_nevra( os.path.basename(x))['name'] == old_name ] if not find: logger.warning('New version of package %s was not found!', old_name) continue new_pkg = find[0] debug = cls._find_debuginfo(debug_new, new_pkg) if debug: command.append('--d2') command.append(debug) command.append(pkg) command.append(new_pkg) logger.verbose('Package name for ABI comparison %s', old_name) output = os.path.join(cls.results_dir, old_name + '.txt') try: ret_code = ProcessHelper.run_subprocess(command, output_file=output) except OSError: raise CheckerNotFoundError( "Checker '{}' was not found or installed.".format( cls.name)) if int(ret_code) & cls.ABIDIFF_ERROR and int( ret_code) & cls.ABIDIFF_USAGE_ERROR: raise RebaseHelperError( 'Execution of {} failed.\nCommand line is: {}'.format( cls.CMD, cmd)) reports[old_name] = int(ret_code) return dict(packages=cls.parse_abi_logs(reports), abi_changes=cls.abi_changes, path=cls.get_checker_output_dir_short())
def test_simple_cmd_with_input_path_and_redirected_output_path(self): with open(self.IN_FILE, 'w') as f: f.write(self.PHRASE) assert os.path.exists(self.IN_FILE) assert open(self.IN_FILE).readline().strip('\n') == self.PHRASE ret = ProcessHelper.run_subprocess(self.CAT_COMMAND, input_file=self.IN_FILE, output_file=self.OUT_FILE) assert ret == 0 assert os.path.exists(self.OUT_FILE) assert open(self.OUT_FILE).readline().strip('\n') == self.PHRASE
def _build_rpm(cls, srpm, results_dir, rpm_results_dir, root=None, arch=None, builder_options=None): """ Build RPM using mock. :param srpm: full path to the srpm. :param results_dir: abs path to dir where the log should be placed. :param rpm_results_dir: directory where rpms will be placed. :param root: path to where chroot should be built. :param arch: target architectures for the build. :param builder_options: builder_options for mock. :return abs paths to RPMs. """ logger.info("Building RPMs") output = os.path.join(results_dir, "mock_output.log") cmd = [ cls.CMD, '--old-chroot', '--rebuild', srpm, '--resultdir', results_dir ] if root is not None: cmd.extend(['--root', root]) if arch is not None: cmd.extend(['--arch', arch]) if builder_options is not None: cmd.extend(builder_options) ret = ProcessHelper.run_subprocess(cmd, output_file=output) if ret == 0: return [ f for f in PathHelper.find_all_files(results_dir, '*.rpm') if not f.endswith('.src.rpm') ] else: logfile = MockBuildTool.get_mock_logfile_path(ret, rpm_results_dir, tmp_path=results_dir) logs = [l for l in PathHelper.find_all_files(results_dir, '*.log')] cls.logs.extend( [os.path.join(rpm_results_dir, os.path.basename(l)) for l in logs]) raise BinaryPackageBuildError("Building RPMs failed!", rpm_results_dir, logfile=logfile)
def run_check(cls, results_dir, **kwargs): """ Compares old and new RPMs using pkgdiff :param results_dir result dir where are stored results """ cls.results_dir = os.path.join(results_dir, cls.name) os.makedirs(cls.results_dir) cls.pkgdiff_results_full_path_html = os.path.join(cls.results_dir, cls.pkgdiff_results_filename + '.html') cmd = [cls.CMD] cmd.append('-hide-unchanged') for version in ['old', 'new']: old = results_store.get_build(version) if old: file_name = cls._create_xml(version, input_structure=old) cmd.append(file_name) cmd.append('-extra-info') cmd.append(cls.results_dir) cmd.append('-report-path') cmd.append(cls.pkgdiff_results_full_path_html) try: ret_code = ProcessHelper.run_subprocess(cmd, output_file=ProcessHelper.DEV_NULL) except OSError: raise CheckerNotFoundError("Checker '{}' was not found or installed.".format(cls.name)) # From pkgdiff source code: # ret_code 0 means unchanged # ret_code 1 means Changed # other return codes means error if int(ret_code) != 0 and int(ret_code) != 1: raise RebaseHelperError('Execution of {} failed.\nCommand line is: {}'.format(cls.CMD, cmd)) results_dict = cls.process_xml_results(cls.results_dir) lines = [] for key, val in results_dict.items(): if val: if lines: lines.append('') lines.append('Following files were {}:'.format(key)) lines.extend(val) pkgdiff_report = os.path.join(cls.results_dir, cls.pkgdiff_results_filename + '.txt') try: with open(pkgdiff_report, "w") as f: f.write('\n'.join(lines)) except IOError: raise RebaseHelperError("Unable to write result from {} to '{}'".format(cls.name, pkgdiff_report)) return dict(path=cls.get_checker_output_dir_short())
def _build_source_from_instructions(cls, instructions, source, logfile): """Run instructions to create source archive""" logger.info("Attempting to create source '%s' using instructions in comments", source) with TemporaryEnvironment() as tmp: script = os.path.join(tmp.path(), 'script.sh') with open(script, 'w') as f: f.write('#!/bin/sh -x\n') f.write('{}\n'.format('\n'.join(instructions))) f.write('cp "{}" "{}"\n'.format(source, os.getcwd())) os.chmod(script, 0o755) result = ProcessHelper.run_subprocess_cwd(script, tmp.path(), output_file=logfile, shell=True) if result == 0 and os.path.isfile(source): logger.info('Source creation succeeded.') else: logger.info('Source creation failed.')
def test_simple_cmd_with_input_fileobject_and_redirected_output_path(self): in_buff = io.StringIO() in_buff.write(self.PHRASE) assert not os.path.exists(self.IN_FILE) in_buff.seek(0) assert in_buff.readline().strip('\n') == self.PHRASE ret = ProcessHelper.run_subprocess(self.CAT_COMMAND, input_file=in_buff, output_file=self.OUT_FILE) in_buff.close() assert ret == 0 assert os.path.exists(self.OUT_FILE) assert open(self.OUT_FILE).readline().strip('\n') == self.PHRASE
def test_simple_cmd_with_input_fileobject_and_redirected_output_path( self): in_buff = io.StringIO() in_buff.write(self.PHRASE) assert not os.path.exists(self.IN_FILE) in_buff.seek(0) assert in_buff.readline().strip('\n') == self.PHRASE ret = ProcessHelper.run_subprocess(self.CAT_COMMAND, input_file=in_buff, output_file=self.OUT_FILE) in_buff.close() assert ret == 0 assert os.path.exists(self.OUT_FILE) assert open(self.OUT_FILE).readline().strip('\n') == self.PHRASE
def run_check(cls, results_dir, **kwargs): """Compares old and new RPMs using abipkgdiff""" # Check if ABI changes occured cls.results_dir = os.path.join(results_dir, cls.name) cls.prepare_results_dir() debug_old, rest_pkgs_old = cls._get_packages_for_abipkgdiff(results_store.get_build('old')) debug_new, rest_pkgs_new = cls._get_packages_for_abipkgdiff(results_store.get_build('new')) ret_codes = {} for pkg in rest_pkgs_old: command = [cls.CMD] command_fallback = [cls.CMD] debug = cls._find_debuginfo(debug_old, pkg) if debug: command.append('--d1') command.append(debug) old_name = RpmHelper.split_nevra(os.path.basename(pkg))['name'] find = [x for x in rest_pkgs_new if RpmHelper.split_nevra(os.path.basename(x))['name'] == old_name] if not find: logger.warning('New version of package %s was not found!', old_name) continue new_pkg = find[0] debug = cls._find_debuginfo(debug_new, new_pkg) if debug: command.append('--d2') command.append(debug) command.extend([pkg, new_pkg]) command_fallback.extend([pkg, new_pkg]) logger.verbose('Package name for ABI comparison %s', old_name) output = os.path.join(cls.results_dir, old_name + '.txt') for cmd in [command, command_fallback]: try: ret_code = ProcessHelper.run_subprocess(cmd, output_file=output) except OSError as e: raise CheckerNotFoundError("Checker '{}' was not found or installed.".format(cls.name)) from e if int(ret_code) & cls.ABIDIFF_ERROR and int(ret_code) & cls.ABIDIFF_USAGE_ERROR: raise RebaseHelperError( 'Execution of {} failed.\nCommand line is: {}'.format(cls.CMD, ' '.join(cmd))) if int(ret_code) & cls.ABIDIFF_ERROR: # abipkgdiff might not be able to read the debuginfo, try again without it continue break ret_codes[old_name] = int(ret_code) return dict(packages=cls.parse_abi_logs(ret_codes), abi_changes=any(x & cls.ABIDIFF_ABI_CHANGE for x in ret_codes.values()), abi_incompatible_changes=any(x & cls.ABIDIFF_ABI_INCOMPATIBLE_CHANGE for x in ret_codes.values()), path=cls.get_checker_output_dir_short(), ret_codes=ret_codes)
def _build_srpm(cls, spec, workdir, results_dir, srpm_results_dir, srpm_builder_options): """ Build SRPM using mock. :param spec: abs path to SPEC file inside the rpmbuild/SPECS in workdir. :param workdir: abs path to working directory with rpmbuild directory structure, which will be used as HOME dir. :param results_dir: abs path to dir where the log should be placed. :param srpm_results_dir: path to directory where SRPM will be placed. :param srpm_builder_options: list of additional options for mock build tool(eg. '-r fedora-XX-x86_64'). :return: abs path to built SRPM. """ logger.info("Building SRPM") spec_loc = os.path.dirname(spec) output = os.path.join(results_dir, "build.log") path_to_sources = os.path.join(workdir, 'SOURCES') cmd = ['mock', '--old-chroot', '--buildsrpm'] if srpm_builder_options is not None: cmd.extend(srpm_builder_options) cmd.extend(['--spec', spec]) cmd.extend(['--sources', path_to_sources]) cmd.extend(['--resultdir', results_dir]) ret = ProcessHelper.run_subprocess_cwd_env(cmd, cwd=spec_loc, env={'HOME': workdir}, output_file=output) build_log_path = os.path.join(srpm_results_dir, 'build.log') mock_log_path = os.path.join(srpm_results_dir, 'mock_output.log') root_log_path = os.path.join(srpm_results_dir, 'root.log') if ret == 0: return PathHelper.find_first_file(workdir, '*.src.rpm') if ret == 1: if not os.path.exists(build_log_path) and os.path.exists(mock_log_path): logfile = mock_log_path else: logfile = build_log_path else: logfile = root_log_path logs = [l for l in PathHelper.find_all_files(results_dir, '*.log')] cls.logs = [os.path.join(srpm_results_dir, os.path.basename(l)) for l in logs] raise SourcePackageBuildError("Building SRPM failed!", logfile=logfile)
def _build_rpm(cls, srpm, workdir, results_dir, rpm_results_dir, builder_options=None): """Builds RPMs using rpmbuild Args: srpm: Path to SRPM. workdir: Path to working directory with rpmbuild directory structure. results_dir: Path to directory where logs will be placed. rpm_results_dir: Path to directory where RPMs will be placed. builder_options: Additional options for rpmbuild. Returns: Tuple, the first element is a list of paths to built RPMs, the second is a list of paths to logs. """ logger.info("Building RPMs") output = os.path.join(results_dir, "build.log") cmd = [cls.CMD, '--rebuild', srpm] if builder_options is not None: cmd.extend(builder_options) ret = ProcessHelper.run_subprocess_cwd_env(cmd, env={'HOME': workdir}, output_file=output) build_log_path = os.path.join(rpm_results_dir, 'build.log') logs = [] for log in PathHelper.find_all_files(results_dir, '*.log'): logs.append(os.path.join(rpm_results_dir, os.path.basename(log))) if ret == 0: return [ f for f in PathHelper.find_all_files(workdir, '*.rpm') if not f.endswith('.src.rpm') ], logs # An error occurred, raise an exception raise BinaryPackageBuildError("Building RPMs failed!", results_dir, logfile=build_log_path, logs=logs)
def install_build_dependencies(spec_path=None, assume_yes=False): """Installs build dependencies of a package using dnf. Args: spec_path (str): Absolute path to the SPEC file. assume_yes (bool): Whether to automatically answer 'yes' to all questions. Returns: int: Exit code of the subprocess run. """ cmd = ['dnf', 'builddep', spec_path] if os.geteuid() != 0: logger.warning("Authentication required to install build dependencies using '%s'", ' '.join(cmd)) cmd = ['pkexec'] + cmd if assume_yes: cmd.append('-y') return ProcessHelper.run_subprocess(cmd)
def test_setting_new_env(self): # make copy of existing environment en_variables = os.environ.copy().keys() # pick up non-existing name while True: rand_name = ''.join(random.choice(string.ascii_letters) for _ in range(6)).upper() if rand_name not in en_variables: break cmd = 'echo "$' + rand_name + '"' ret = ProcessHelper.run_subprocess_cwd_env(cmd, env={rand_name: self.PHRASE}, output_file=self.OUT_FILE, shell=True) assert ret == 0 assert os.path.exists(self.OUT_FILE) assert open(self.OUT_FILE).readline().strip("\n") == self.PHRASE
def test_setting_existing_env(self): # make copy of existing environment en_variables = list(os.environ.copy().keys()) # there are no variables set on the system -> nothing to test if not en_variables: pass assert os.environ.get(en_variables[0]) != self.PHRASE cmd = 'echo "$' + en_variables[0] + '"' ret = ProcessHelper.run_subprocess_cwd_env(cmd, env={en_variables[0]: self.PHRASE}, output_file=self.OUT_FILE, shell=True) assert ret == 0 assert os.path.exists(self.OUT_FILE) assert open(self.OUT_FILE).readline().strip("\n") == self.PHRASE
def test_setting_existing_env(self): # make copy of existing environment en_variables = list(os.environ.copy().keys()) # there are no variables set on the system -> nothing to test if not en_variables: pass assert os.environ.get(en_variables[0]) != self.PHRASE cmd = 'echo "$' + en_variables[0] + '"' ret = ProcessHelper.run_subprocess_cwd_env( cmd, env={en_variables[0]: self.PHRASE}, output_file=self.OUT_FILE, shell=True) assert ret == 0 assert os.path.exists(self.OUT_FILE) assert open(self.OUT_FILE).readline().strip("\n") == self.PHRASE
def test_simple_cmd_with_input_path_and_redirected_output_fileobject( self): out_buff = io.StringIO() with open(self.IN_FILE, 'w', encoding=ENCODING) as f: f.write(self.PHRASE) assert os.path.exists(self.IN_FILE) assert open( self.IN_FILE, encoding=ENCODING).readline().strip('\n') == self.PHRASE ret = ProcessHelper.run_subprocess(self.CAT_COMMAND, input_file=self.IN_FILE, output_file=out_buff) assert ret == 0 assert not os.path.exists(self.OUT_FILE) out_buff.seek(0) assert out_buff.readline().strip('\n') == self.PHRASE out_buff.close()
def run_check(cls, results_dir, **kwargs): """Compares old and new RPMs using abipkgdiff""" # Check if ABI changes occured cls.abi_changes = False cls.results_dir = os.path.join(results_dir, cls.name) os.makedirs(cls.results_dir) debug_old, rest_pkgs_old = cls._get_packages_for_abipkgdiff(results_store.get_build('old')) debug_new, rest_pkgs_new = cls._get_packages_for_abipkgdiff(results_store.get_build('new')) cmd = [cls.CMD] reports = {} for pkg in rest_pkgs_old: command = list(cmd) debug = cls._find_debuginfo(debug_old, pkg) if debug: command.append('--d1') command.append(debug) old_name = RpmHelper.split_nevra(os.path.basename(pkg))['name'] find = [x for x in rest_pkgs_new if RpmHelper.split_nevra(os.path.basename(x))['name'] == old_name] if not find: logger.warning('New version of package %s was not found!', old_name) continue new_pkg = find[0] debug = cls._find_debuginfo(debug_new, new_pkg) if debug: command.append('--d2') command.append(debug) command.append(pkg) command.append(new_pkg) logger.verbose('Package name for ABI comparison %s', old_name) output = os.path.join(cls.results_dir, old_name + '.txt') try: ret_code = ProcessHelper.run_subprocess(command, output_file=output) except OSError: raise CheckerNotFoundError("Checker '{}' was not found or installed.".format(cls.name)) if int(ret_code) & cls.ABIDIFF_ERROR and int(ret_code) & cls.ABIDIFF_USAGE_ERROR: raise RebaseHelperError('Execution of {} failed.\nCommand line is: {}'.format(cls.CMD, cmd)) reports[old_name] = int(ret_code) return dict(packages=cls.parse_abi_logs(reports), abi_changes=cls.abi_changes, path=cls.get_checker_output_dir_short())
def test_setting_new_env(self): # make copy of existing environment en_variables = os.environ.copy().keys() # pick up non-existing name while True: rand_name = ''.join( random.choice(string.ascii_letters) for _ in range(6)).upper() if rand_name not in en_variables: break cmd = 'echo "$' + rand_name + '"' ret = ProcessHelper.run_subprocess_cwd_env( cmd, env={rand_name: self.PHRASE}, output_file=self.OUT_FILE, shell=True) assert ret == 0 assert os.path.exists(self.OUT_FILE) assert open(self.OUT_FILE).readline().strip("\n") == self.PHRASE
def run_rpminspect(cls, checker_dir: str, old_pkg: str, new_pkg: str) -> Tuple[str, Dict[str, int]]: """Runs rpminspect on the given package. Args: checker_dir: Path to the results directory of the checker old_pkg: Path to the old package. new_pkg: Path to the new package. Returns: Tuple of (path, data), where path is the path to rpminspect output and data is the dict of count of OK/BAD/VERIFY checks. """ cmd = [cls.CMD, '-F', 'json'] pkg_name = RpmHelper.split_nevra(os.path.basename(new_pkg))['name'] cmd.append(old_pkg) cmd.append(new_pkg) outfile = os.path.join(checker_dir, '{}.json'.format(pkg_name)) try: ret = ProcessHelper.run_subprocess(cmd, output_file=outfile, ignore_stderr=True) except OSError as e: raise CheckerNotFoundError( 'Checker \'{}\' was not found or installed.'.format( cls.name)) from e # Exit code 1 is used when bad check is found, 0 when everything is OK. Others on error if ret not in (0, 1): raise RebaseHelperError( 'An error occurred when running checker \'{}\''.format( cls.name)) with open(outfile, 'r', encoding=ENCODING) as json_file: data = json.load(json_file) return outfile, cls.process_data(data)
def _build_srpm(cls, spec, workdir, results_dir, srpm_results_dir, srpm_builder_options): """ Build SRPM using rpmbuild. :param spec: abs path to SPEC file inside the rpmbuild/SPECS in workdir. :param workdir: abs path to working directory with rpmbuild directory structure, which will be used as HOME dir. :param results_dir: abs path to dir where the log should be placed. :param srpm_results_dir: path to directory where SRPM will be placed. :param srpm_builder_options: list of additional options to rpmbuild. :return: abs path to built SRPM. """ logger.info("Building SRPM") spec_loc, spec_name = os.path.split(spec) output = os.path.join(results_dir, "build.log") cmd = ['rpmbuild', '-bs', spec_name] if srpm_builder_options is not None: cmd.extend(srpm_builder_options) ret = ProcessHelper.run_subprocess_cwd_env(cmd, cwd=spec_loc, env={'HOME': workdir}, output_file=output) build_log_path = os.path.join(srpm_results_dir, 'build.log') if ret == 0: return PathHelper.find_first_file(workdir, '*.src.rpm') # An error occurred, raise an exception logfile = build_log_path logs = [l for l in PathHelper.find_all_files(results_dir, '*.log')] cls.logs = [ os.path.join(srpm_results_dir, os.path.basename(l)) for l in logs ] raise SourcePackageBuildError("Building SRPM failed!", logfile=logfile)
def _build_srpm(cls, spec, workdir, results_dir, srpm_results_dir, srpm_builder_options): """Builds SRPM using rpmbuild. Args: spec: Path to SPEC file to build the SRPM from. workdir: Path to working directory with rpmbuild directory structure. results_dir: Path to directory where logs will be placed. srpm_results_dir: Path to directory where SRPM will be placed. srpm_builder_options: Additional rpmbuild options. Returns: Tuple, the first element is path to SRPM, the second is a list of paths to logs. """ logger.info("Building SRPM") spec_loc, spec_name = os.path.split(spec) output = os.path.join(results_dir, "build.log") cmd = [cls.CMD, '-bs', spec_name] if srpm_builder_options is not None: cmd.extend(srpm_builder_options) ret = ProcessHelper.run_subprocess_cwd_env(cmd, cwd=spec_loc, env={'HOME': workdir}, output_file=output) build_log_path = os.path.join(srpm_results_dir, 'build.log') logs = [] for l in PathHelper.find_all_files(results_dir, '*.log'): logs.append(os.path.join(srpm_results_dir, os.path.basename(l))) if ret == 0: return PathHelper.find_first_file(workdir, '*.src.rpm'), logs # An error occurred, raise an exception raise SourcePackageBuildError("Building SRPM failed!", logfile=build_log_path, logs=logs)
def get_license_changes(cls, old_dir, new_dir): """ Finds differences in licenses between old and new source files. Args: old_dir(str): Path to the old sources directory. new_dir(str): Path to the new sources directory. Returns: tuple: Changes dictionary, new_licenses set, disappeared_licenses set. """ diffs = [] possible_license_names = r'COPYING|LICENSE|LICENCE|LICENSING|BSD|(L)?GPL(v[23][+]?)?' for source_dir in [old_dir, new_dir]: out = io.StringIO() ProcessHelper.run_subprocess(["/usr/bin/licensecheck", source_dir, "--machine", "--recursive"], output_file=out, ignore_stderr=True) diff = {} for l in out: # licensecheck output format: 'Filepath\tlicense' file_path, dlicense = l.split('\t') if 'GENERATED FILE' in dlicense: continue file_path = os.path.relpath(file_path, source_dir) diff[file_path] = dlicense.strip() diffs.append(diff) old_lics, new_lics = set(), set() changes = {'appeared': {}, 'transitioned': {}, 'disappeared': {}} # Get changed licenses in existing files for new_file, new_license in diffs[1].items(): new_lics.add(new_license) for old_file, old_license in diffs[0].items(): old_lics.add(old_license) if (new_file == old_file and (new_license != old_license)): new_key = '{} => {}'.format(old_license, new_license) if new_license == 'UNKNOWN': # Conversion `known license` => `None/Unknown` if re.search(possible_license_names, new_file, re.IGNORECASE): cls.license_files_changes.update({old_license: 'disappeared in {}'.format(new_file)}) if old_license not in changes['disappeared']: changes['disappeared'][old_license] = [] changes['disappeared'][old_license].append(new_file) elif old_license == 'UNKNOWN': # Conversion `None/Unknown` => `known license` if re.search(possible_license_names, new_file, re.IGNORECASE): cls.license_files_changes.update({new_license: 'appeared in {}'.format(new_file)}) if new_license not in changes['appeared']: changes['appeared'][new_license] = [] changes['appeared'][new_license].append(new_file) else: # Conversion `known license` => `known license` if re.search(possible_license_names, new_file, re.IGNORECASE): cls.license_files_changes.update({new_key: 'in {}'.format(new_file)}) if new_key not in changes['transitioned']: changes['transitioned'][new_key] = [] if new_file not in changes['transitioned'][new_key]: changes['transitioned'][new_key].append(new_file) # Get newly appeared files for new_file, new_license in diffs[1].items(): if new_file not in diffs[0]: if new_license == 'UNKNOWN': continue if re.search(possible_license_names, new_file, re.IGNORECASE): cls.license_files_changes.update({new_license: 'appeared in {}'.format(new_file)}) if new_license not in changes['appeared']: changes['appeared'][new_license] = [] changes['appeared'][new_license].append(new_file) # Get removed files for old_file, old_license in diffs[0].items(): if old_file not in diffs[1]: if old_license == 'UNKNOWN': continue if re.search(possible_license_names, old_file, re.IGNORECASE): cls.license_files_changes.update({old_license: 'disappeared in {}'.format(old_file)}) if old_license not in changes['disappeared']: changes['disappeared'][old_license] = [] changes['disappeared'][old_license].append(old_file) new_licenses = new_lics - old_lics disappeared_licenses = old_lics - new_lics if new_licenses or disappeared_licenses: cls.license_changes = True return changes, list(new_licenses), list(disappeared_licenses)
def get_license_changes(cls, old_dir, new_dir): """ Finds differences in licenses between old and new source files. Args: old_dir(str): Path to the old sources directory. new_dir(str): Path to the new sources directory. Returns: tuple: Changes dictionary, new_licenses set, disappeared_licenses set. """ diffs = [] possible_license_names = r'COPYING|LICENSE|LICENCE|LICENSING|BSD|(L)?GPL(v[23][+]?)?' for source_dir in [old_dir, new_dir]: out = six.StringIO() ProcessHelper.run_subprocess([ "/usr/bin/licensecheck", source_dir, "--machine", "--recursive" ], output_file=out, ignore_stderr=True) diff = {} for l in out: # licensecheck output format: 'Filepath\tlicense' file_path, dlicense = l.split('\t') if 'GENERATED FILE' in dlicense: continue file_path = os.path.relpath(file_path, source_dir) diff[file_path] = dlicense.strip() diffs.append(diff) old_lics, new_lics = set(), set() changes = {'appeared': {}, 'transitioned': {}, 'disappeared': {}} # Get changed licenses in existing files for new_file, new_license in six.iteritems(diffs[1]): new_lics.add(new_license) for old_file, old_license in six.iteritems(diffs[0]): old_lics.add(old_license) if (new_file == old_file and (new_license != old_license)): new_key = '{} => {}'.format(old_license, new_license) if new_license == 'UNKNOWN': # Conversion `known license` => `None/Unknown` if re.search(possible_license_names, new_file, re.IGNORECASE): cls.license_files_changes.update({ old_license: 'disappeared in {}'.format(new_file) }) if old_license not in changes['disappeared']: changes['disappeared'][old_license] = [] changes['disappeared'][old_license].append(new_file) elif old_license == 'UNKNOWN': # Conversion `None/Unknown` => `known license` if re.search(possible_license_names, new_file, re.IGNORECASE): cls.license_files_changes.update({ new_license: 'appeared in {}'.format(new_file) }) if new_license not in changes['appeared']: changes['appeared'][new_license] = [] changes['appeared'][new_license].append(new_file) else: # Conversion `known license` => `known license` if re.search(possible_license_names, new_file, re.IGNORECASE): cls.license_files_changes.update( {new_key: 'in {}'.format(new_file)}) if new_key not in changes['transitioned']: changes['transitioned'][new_key] = [] if new_file not in changes['transitioned'][new_key]: changes['transitioned'][new_key].append(new_file) # Get newly appeared files for new_file, new_license in six.iteritems(diffs[1]): if new_file not in diffs[0]: if new_license == 'UNKNOWN': continue if re.search(possible_license_names, new_file, re.IGNORECASE): cls.license_files_changes.update( {new_license: 'appeared in {}'.format(new_file)}) if new_license not in changes['appeared']: changes['appeared'][new_license] = [] changes['appeared'][new_license].append(new_file) # Get removed files for old_file, old_license in six.iteritems(diffs[0]): if old_file not in diffs[1]: if old_license == 'UNKNOWN': continue if re.search(possible_license_names, old_file, re.IGNORECASE): cls.license_files_changes.update( {old_license: 'disappeared in {}'.format(old_file)}) if old_license not in changes['disappeared']: changes['disappeared'][old_license] = [] changes['disappeared'][old_license].append(old_file) new_licenses = new_lics - old_lics disappeared_licenses = old_lics - new_lics if new_licenses or disappeared_licenses: cls.license_changes = True return changes, list(new_licenses), list(disappeared_licenses)
def is_available(cls): try: return ProcessHelper.run_subprocess([cls.CMD, '--help'], output_file=ProcessHelper.DEV_NULL) == 0 except (IOError, OSError): return False
def test_simple_cmd_changed_work_dir(self): os.mkdir(self.TEMP_DIR) ret = ProcessHelper.run_subprocess_cwd(self.TOUCH_COMMAND, self.TEMP_DIR) assert ret == 0 assert os.path.exists(os.path.join(self.TEMP_DIR, self.TEMP_FILE))
def test_simple_cmd_with_redirected_output_path(self): ret = ProcessHelper.run_subprocess(self.ECHO_COMMAND, output_file=self.OUT_FILE) assert ret == 0 assert os.path.exists(self.OUT_FILE) assert open(self.OUT_FILE).readline().strip('\n') == self.PHRASE
def test_simple_cmd(self): ret = ProcessHelper.run_subprocess(self.TOUCH_COMMAND) assert ret == 0 assert os.path.exists(self.TEMP_FILE)