def run_check(cls, results_dir): """ Compares old and new RPMs using pkgdiff """ results_dict = {} for tag in settings.CHECKER_TAGS: results_dict[tag] = [] cls.results_dir = results_dir # Only S (size), M(mode) and 5 (checksum) are now important not_catched_flags = ['T', 'F', 'G', 'U', 'V', 'L', 'D', 'N'] old_pkgs = cls._get_rpms(OutputLogger.get_build('old').get('rpm', None)) new_pkgs = cls._get_rpms(OutputLogger.get_build('new').get('rpm', None)) for key, value in six.iteritems(old_pkgs): cmd = [cls.CMD] # TODO modify to online command for x in not_catched_flags: cmd.extend(['-i', x]) cmd.append(value) # We would like to build correct old package against correct new packages cmd.append(new_pkgs[key]) output = StringIO() ProcessHelper.run_subprocess(cmd, output=output) results_dict = cls._analyze_logs(output, results_dict) results_dict = cls.update_added_removed(results_dict) results_dict = dict((k, v) for k, v in six.iteritems(results_dict) if v) text = [] for key, val in six.iteritems(results_dict): text.append('Following files were %s:\n%s' % (key, '\n'.join(val))) return text
def run_check(cls, results_dir): """Compares old and new RPMs using pkgdiff""" csmock_report = {} old_pkgs = results_store.get_old_build().get('srpm', None) new_pkgs = results_store.get_new_build().get('srpm', None) csmock_dir = os.path.join(results_dir, cls.CMD) os.makedirs(csmock_dir) arguments = [ '--force', '-a', '-r', 'fedora-rawhide-x86_64', '--base-srpm' ] if old_pkgs and new_pkgs: cmd = [cls.CMD] cmd.extend(arguments) cmd.append(old_pkgs) cmd.append(new_pkgs) cmd.extend(['-o', csmock_dir]) output = StringIO() try: ProcessHelper.run_subprocess(cmd, output=output) except OSError: raise CheckerNotFoundError( "Checker '%s' was not found or installed." % cls.CMD) csmock_report['error'] = PathHelper.find_all_files_current_dir( csmock_dir, '*.err') csmock_report['txt'] = PathHelper.find_all_files_current_dir( csmock_dir, '*.txt') csmock_report['log'] = PathHelper.find_all_files_current_dir( csmock_dir, '*.log') return csmock_report
def call_prep_script(cls, prep_script_path): cwd = os.getcwd() os.chdir(cls.old_sources) ProcessHelper.run_subprocess(prep_script_path, output=os.path.join(cls.kwargs['results_dir'], 'prep_script.log')) if not cls.patch_sources_by_prep_script: cls.git_helper.command_add_files(parameters=["--all"]) cls.git_helper.command_commit(message="prep_script prep_corrections") os.chdir(cwd)
def run_check(cls, results_dir): """Compares old and new RPMs using rpmdiff""" results_dict = {} for tag in settings.CHECKER_TAGS: results_dict[tag] = [] cls.results_dir = results_dir # Only S (size), M(mode) and 5 (checksum) are now important not_catched_flags = ['T', 'F', 'G', 'U', 'V', 'L', 'D', 'N'] old_pkgs = cls._get_rpms(results_store.get_old_build().get( 'rpm', None)) new_pkgs = cls._get_rpms(results_store.get_new_build().get( 'rpm', None)) for key, value in six.iteritems(old_pkgs): if 'debuginfo' in key or 'debugsource' in key: # skip debug{info,source} packages continue cmd = [cls.CMD] # TODO modify to online command for x in not_catched_flags: cmd.extend(['-i', x]) cmd.append(value) # We would like to build correct old package against correct new packages try: cmd.append(new_pkgs[key]) except KeyError: logger.warning('New version of package %s was not found!', key) continue output = StringIO() try: ProcessHelper.run_subprocess(cmd, output=output) except OSError: raise CheckerNotFoundError( "Checker '%s' was not found or installed." % cls.CMD) results_dict = cls._analyze_logs(output, results_dict) results_dict = cls.update_added_removed(results_dict) results_dict = dict( (k, v) for k, v in six.iteritems(results_dict) if v) lines = [] for key, val in six.iteritems(results_dict): if val: if lines: lines.append('') lines.append('Following files were %s:' % key) lines.extend(val) rpmdiff_report = os.path.join(cls.results_dir, 'report-' + cls.CMD + '.log') try: with open(rpmdiff_report, "w") as f: f.write('\n'.join(lines)) except IOError: raise RebaseHelperError("Unable to write result from %s to '%s'" % (cls.CMD, rpmdiff_report)) return {rpmdiff_report: None}
def call_prep_script(cls, prep_script_path): cwd = os.getcwd() os.chdir(cls.old_sources) ProcessHelper.run_subprocess(prep_script_path, output=os.path.join(cls.kwargs['workspace_dir'], 'prep_script.log')) if not cls.patch_sources_by_prep_script: cls.old_repo.git.add(all=True) cls.old_repo.index.commit('prep_script prep_corrections', skip_hooks=True) os.chdir(cwd)
def run_check(cls, results_dir, **kwargs): """Compares old and new RPMs using rpmdiff""" results_dict = {} for tag in cls.CHECKER_TAGS: results_dict[tag] = [] cls.results_dir = os.path.join(results_dir, cls.NAME) os.makedirs(cls.results_dir) # Only S (size), M(mode) and 5 (checksum) are now important not_catched_flags = ['T', 'F', 'G', 'U', 'V', 'L', 'D', 'N'] old_pkgs = cls._get_rpms(results_store.get_old_build().get('rpm', None)) new_pkgs = cls._get_rpms(results_store.get_new_build().get('rpm', None)) for key, value in six.iteritems(old_pkgs): if 'debuginfo' in key or 'debugsource' in key: # skip debug{info,source} packages continue cmd = [cls.NAME] # TODO modify to online command for x in not_catched_flags: cmd.extend(['-i', x]) cmd.append(value) # We would like to build correct old package against correct new packages try: cmd.append(new_pkgs[key]) except KeyError: logger.warning('New version of package %s was not found!', key) continue output = StringIO() try: ProcessHelper.run_subprocess(cmd, output_file=output) except OSError: raise CheckerNotFoundError("Checker '{}' was not found or installed.".format(cls.NAME)) results_dict = cls._analyze_logs(output, results_dict) results_dict = cls.update_added_removed(results_dict) cls.results_dict = {k: v for k, v in six.iteritems(results_dict) if v} lines = [] for key, val in six.iteritems(results_dict): if val: if lines: lines.append('') lines.append('Following files were {}:'.format(key)) lines.extend(val) rpmdiff_report = os.path.join(cls.results_dir, 'report.txt') counts = {k: len(v) for k, v in six.iteritems(results_dict)} try: with open(rpmdiff_report, "w") as f: f.write('\n'.join(lines)) except IOError: raise RebaseHelperError("Unable to write result from {} to '{}'".format(cls.NAME, rpmdiff_report)) return {'path': cls.get_checker_output_dir_short(), 'files_changes': counts}
def call_prep_script(cls, prep_script_path): cwd = os.getcwd() os.chdir(cls.old_sources) ProcessHelper.run_subprocess(prep_script_path, output=os.path.join( cls.kwargs['workspace_dir'], 'prep_script.log')) if not cls.patch_sources_by_prep_script: cls.git_helper.command_add_files(parameters=["--all"]) cls.git_helper.command_commit( message="prep_script prep_corrections") os.chdir(cwd)
def test_simple_cmd_with_redirected_output_fileobject(self): buff = StringIO() ret = ProcessHelper.run_subprocess(self.ECHO_COMMAND, output=buff) assert ret == 0 assert not os.path.exists(self.OUT_FILE) assert buff.readline().strip("\n") == self.PHRASE buff.close()
def _build_rpm(cls, srpm, workdir, results_dir, rpm_results_dir, builder_options=None): """ Build RPM using rpmbuild. :param srpm: abs path to SRPM :param workdir: abs path to working directory with rpmbuild directory structure, which will be used as HOME dir. :param results_dir: abs path to dir where the log should be placed. :param rpm_results_dir: path directory to where RPMs will be placed. :return: abs paths to built RPMs. """ logger.info("Building RPMs") output = os.path.join(results_dir, "build.log") cmd = [cls.CMD, '--rebuild', srpm] if builder_options is not None: cmd.extend(builder_options) ret = ProcessHelper.run_subprocess_cwd_env(cmd, env={'HOME': workdir}, output_file=output) build_log_path = os.path.join(rpm_results_dir, 'build.log') if ret == 0: return [f for f in PathHelper.find_all_files(workdir, '*.rpm') if not f.endswith('.src.rpm')] # An error occurred, raise an exception logfile = build_log_path cls.logs.extend([l for l in PathHelper.find_all_files(rpm_results_dir, '*.log')]) raise BinaryPackageBuildError("Building RPMs failed!", results_dir, logfile=logfile)
def _do_build_srpm(cls, spec, workdir, results_dir): """ Build SRPM using rpmbuild. :param spec: abs path to SPEC file inside the rpmbuild/SPECS in workdir. :param workdir: abs path to working directory with rpmbuild directory structure, which will be used as HOME dir. :param results_dir: abs path to dir where the log should be placed. :return: If build process ends successfully returns abs path to built SRPM, otherwise 'None'. """ logger.info("Building SRPM") spec_loc, spec_name = os.path.split(spec) output = os.path.join(results_dir, "build.log") cmd = ['rpmbuild', '-bs', spec_name] ret = ProcessHelper.run_subprocess_cwd_env(cmd, cwd=spec_loc, env={'HOME': workdir}, output=output) if ret != 0: return None else: return PathHelper.find_first_file(workdir, '*.src.rpm')
def _build_rpm(cls, srpm, results_dir, root=None, arch=None, builder_options=None): """Build RPM using mock.""" logger.info("Building RPMs") output = os.path.join(results_dir, "mock_output.log") cmd = [cls.CMD, '--rebuild', srpm, '--resultdir', results_dir] if root is not None: cmd.extend(['--root', root]) if arch is not None: cmd.extend(['--arch', arch]) if builder_options is not None: cmd.extend(builder_options) ret = ProcessHelper.run_subprocess(cmd, output=output) if ret != 0: return None else: return [ f for f in PathHelper.find_all_files(results_dir, '*.rpm') if not f.endswith('.src.rpm') ]
def _build_rpm(cls, srpm, workdir, results_dir, builder_options=None): """ Build RPM using rpmbuild. :param srpm: abs path to SRPM :param workdir: abs path to working directory with rpmbuild directory structure, which will be used as HOME dir. :param results_dir: abs path to dir where the log should be placed. :return: If build process ends successfully returns list of abs paths to built RPMs, otherwise 'None'. """ logger.info("Building RPMs") output = os.path.join(results_dir, "build.log") cmd = [cls.CMD, '--rebuild', srpm] if builder_options is not None: cmd.extend(builder_options) ret = ProcessHelper.run_subprocess_cwd_env(cmd, env={'HOME': workdir}, output=output) if ret != 0: return None else: return [ f for f in PathHelper.find_all_files(workdir, '*.rpm') if not f.endswith('.src.rpm') ]
def test_simple_cmd_with_redirected_output_fileobject(self): buff = StringIO() ret = ProcessHelper.run_subprocess(self.ECHO_COMMAND, output=buff) assert ret == 0 assert not os.path.exists(self.OUT_FILE) assert buff.readline().strip('\n') == self.PHRASE buff.close()
def run_check(cls, results_dir): """ Compares old and new RPMs using pkgdiff """ cls.results_dir = results_dir cls.pkgdiff_results_full_path = os.path.join(cls.results_dir, cls.pkgdiff_results_filename) versions = ['old', 'new'] cmd = [cls.CMD] for version in versions: old = OutputLogger.get_build(version) if old: file_name = cls._create_xml(version, input_structure=old) cmd.append(file_name) cmd.append('-extra-info') cmd.append(cls.results_dir) cmd.append('-report-path') cmd.append(cls.pkgdiff_results_full_path) ret_code = ProcessHelper.run_subprocess(cmd, output=ProcessHelper.DEV_NULL) """ From pkgdiff source code: ret_code 0 means unchanged ret_code 1 means Changed other return codes means error """ if int(ret_code) != 0 and int(ret_code) != 1: raise RebaseHelperError('Execution of %s failed.\nCommand line is: %s', cls.CMD, cmd) OutputLogger.set_info_text('Result HTML page from pkgdiff is store in: ', cls.pkgdiff_results_full_path) results_dict = cls.process_xml_results(cls.results_dir) text = [] for key, val in six.iteritems(results_dict): text.append('Following files were %s:\n%s' % (key, '\n'.join(val))) return text
def _build_rpm(cls, srpm, workdir, results_dir, builder_options=None): """ Build RPM using rpmbuild. :param srpm: abs path to SRPM :param workdir: abs path to working directory with rpmbuild directory structure, which will be used as HOME dir. :param results_dir: abs path to dir where the log should be placed. :return: If build process ends successfully returns list of abs paths to built RPMs, otherwise 'None'. """ logger.info("Building RPMs") output = os.path.join(results_dir, "build.log") cmd = [cls.CMD, '--rebuild', srpm] if builder_options is not None: cmd.extend(builder_options) ret = ProcessHelper.run_subprocess_cwd_env(cmd, env={'HOME': workdir}, output=output) if ret != 0: return None else: return [f for f in PathHelper.find_all_files(workdir, '*.rpm') if not f.endswith('.src.rpm')]
def run_check(cls, results_dir): """ Compares old and new RPMs using pkgdiff """ results_dict = {} for tag in settings.CHECKER_TAGS: results_dict[tag] = [] cls.results_dir = results_dir # Only S (size), M(mode) and 5 (checksum) are now important not_catched_flags = ['T', 'F', 'G', 'U', 'V', 'L', 'D', 'N'] old_pkgs = cls._get_rpms(OutputLogger.get_build('old').get('rpm', None)) new_pkgs = cls._get_rpms(OutputLogger.get_build('new').get('rpm', None)) for key, value in six.iteritems(old_pkgs): cmd = [cls.CMD] # TODO modify to online command for x in not_catched_flags: cmd.extend(['-i', x]) cmd.append(value) # We would like to build correct old package against correct new packages cmd.append(new_pkgs[key]) output = StringIO() ret_code = ProcessHelper.run_subprocess(cmd, output=output) results_dict = cls._analyze_logs(output, results_dict) results_dict = cls.update_added_removed(results_dict) # TODO Check for changed files and # remove them from 'removed' and 'added' #cls._unpack_rpm(old_pkgs) #cls._unpack_rpm(new_pkgs) #cls._find_file_diffs(old_pkgs, new_pkgs) return results_dict
def test_simple_cmd_changed_work_dir_with_redirected_output(self): # create temp_file in temp_dir self.test_simple_cmd_changed_work_dir() ret = ProcessHelper.run_subprocess_cwd(self.LS_COMMAND, self.TEMP_DIR, output=self.OUT_FILE) assert ret == 0 assert os.path.exists(os.path.join(self.TEMP_DIR, self.TEMP_FILE)) assert os.path.exists(self.OUT_FILE) assert open(self.OUT_FILE).readline().strip("\n") == self.TEMP_FILE
def run_check(cls, result_dir): """Compares old and new RPMs using pkgdiff""" debug_old, rest_pkgs_old = cls._get_packages_for_abipkgdiff(results_store.get_build('old')) debug_new, rest_pkgs_new = cls._get_packages_for_abipkgdiff(results_store.get_build('new')) cmd = [cls.CMD] if debug_old is None: logger.warning("Package doesn't contain any debug package") return None try: cmd.append('--d1') cmd.append(debug_old[0]) except IndexError: logger.error('Debuginfo package not found for old package.') return None try: cmd.append('--d2') cmd.append(debug_new[0]) except IndexError: logger.error('Debuginfo package not found for new package.') return None reports = {} for pkg in rest_pkgs_old: command = list(cmd) # Package can be <letters><numbers>-<letters>-<and_whatever> regexp = r'^(\w*)(-\D+)?.*$' reg = re.compile(regexp) matched = reg.search(os.path.basename(pkg)) if matched: file_name = matched.group(1) command.append(pkg) find = [x for x in rest_pkgs_new if os.path.basename(x).startswith(file_name)] command.append(find[0]) package_name = os.path.basename(os.path.basename(pkg)) logger.debug('Package name for ABI comparision %s', package_name) regexp_name = r'(\w-)*(\D+)*' reg_name = re.compile(regexp_name) matched = reg_name.search(os.path.basename(pkg)) logger.debug('Found matches %s', matched.groups()) if matched: package_name = matched.group(0) + cls.log_name else: package_name = package_name + '-' + cls.log_name output = os.path.join(cls.results_dir, result_dir, package_name) try: ret_code = ProcessHelper.run_subprocess(command, output=output) except OSError: raise CheckerNotFoundError("Checker '%s' was not found or installed." % cls.CMD) if int(ret_code) & settings.ABIDIFF_ERROR and int(ret_code) & settings.ABIDIFF_USAGE_ERROR: raise RebaseHelperError('Execution of %s failed.\nCommand line is: %s' % (cls.CMD, cmd)) if int(ret_code) == 0: text = 'ABI of the compared binaries in package %s are equal.' % package_name else: text = 'ABI of the compared binaries in package %s are not equal.' % package_name reports[output] = text else: logger.debug("Rebase-helper did not find a package name in '%s'", package_name) return reports
def run_check(cls, result_dir): """Compares old and new RPMs using pkgdiff""" debug_old, rest_pkgs_old = cls._get_packages_for_abipkgdiff(OutputLogger.get_build('old')) debug_new, rest_pkgs_new = cls._get_packages_for_abipkgdiff(OutputLogger.get_build('new')) cmd = [cls.CMD] if debug_old is None: logger.warning("Package doesn't contain any debug package") return None try: cmd.append('--d1') cmd.append(debug_old[0]) except IndexError: logger.error('Debuginfo package not found for old package.') return None try: cmd.append('--d2') cmd.append(debug_new[0]) except IndexError: logger.error('Debuginfo package not found for new package.') return None reports = {} for pkg in rest_pkgs_old: command = list(cmd) # Package can be <letters><numbers>-<letters>-<and_whatever> regexp = r'^(\w*)(-\D+)?.*$' reg = re.compile(regexp) matched = reg.search(os.path.basename(pkg)) if matched: file_name = matched.group(1) command.append(pkg) find = [x for x in rest_pkgs_new if os.path.basename(x).startswith(file_name)] command.append(find[0]) package_name = os.path.basename(os.path.basename(pkg)) logger.debug('Package name for ABI comparision %s', package_name) regexp_name = r'(\w-)*(\D+)*' reg_name = re.compile(regexp_name) matched = reg_name.search(os.path.basename(pkg)) logger.debug('Found matches %s', matched.groups()) if matched: package_name = matched.group(0) + cls.log_name else: package_name = package_name + '-' + cls.log_name output = os.path.join(cls.results_dir, result_dir, package_name) try: ret_code = ProcessHelper.run_subprocess(command, output=output) except OSError: raise CheckerNotFoundError("Checker '%s' was not found or installed." % cls.CMD) if int(ret_code) & settings.ABIDIFF_ERROR and int(ret_code) & settings.ABIDIFF_USAGE_ERROR: raise RebaseHelperError('Execution of %s failed.\nCommand line is: %s' % (cls.CMD, cmd)) if int(ret_code) == 0: text = 'ABI of the compared binaries in package %s are equal.' % package_name else: text = 'ABI of the compared binaries in package %s are not equal.' % package_name reports[output] = text else: logger.debug("Rebase-helper did not find a package name in '%s'", package_name) return reports
def run_check(cls, results_dir): """Compares old and new RPMs using rpmdiff""" results_dict = {} for tag in settings.CHECKER_TAGS: results_dict[tag] = [] cls.results_dir = results_dir # Only S (size), M(mode) and 5 (checksum) are now important not_catched_flags = ['T', 'F', 'G', 'U', 'V', 'L', 'D', 'N'] old_pkgs = cls._get_rpms(results_store.get_old_build().get('rpm', None)) new_pkgs = cls._get_rpms(results_store.get_new_build().get('rpm', None)) for key, value in six.iteritems(old_pkgs): cmd = [cls.CMD] # TODO modify to online command for x in not_catched_flags: cmd.extend(['-i', x]) cmd.append(value) # We would like to build correct old package against correct new packages cmd.append(new_pkgs[key]) output = StringIO() try: ProcessHelper.run_subprocess(cmd, output=output) except OSError: raise CheckerNotFoundError("Checker '%s' was not found or installed." % cls.CMD) results_dict = cls._analyze_logs(output, results_dict) results_dict = cls.update_added_removed(results_dict) results_dict = dict((k, v) for k, v in six.iteritems(results_dict) if v) lines = [] for key, val in six.iteritems(results_dict): if val: if lines: lines.append('') lines.append('Following files were %s:' % key) lines.extend(val) rpmdiff_report = os.path.join(cls.results_dir, 'report-' + cls.CMD + '.log') try: with open(rpmdiff_report, "w") as f: f.write('\n'.join(lines)) except IOError: raise RebaseHelperError("Unable to write result from %s to '%s'" % (cls.CMD, rpmdiff_report)) return {rpmdiff_report: None}
def run_check(cls, results_dir, **kwargs): """ Compares old and new RPMs using pkgdiff :param results_dir result dir where are stored results """ cls.results_dir = os.path.join(results_dir, cls.NAME) os.makedirs(cls.results_dir) cls.pkgdiff_results_full_path_html = os.path.join( cls.results_dir, cls.pkgdiff_results_filename + '.html') cmd = [cls.NAME] cmd.append('-hide-unchanged') for version in ['old', 'new']: old = results_store.get_build(version) if old: file_name = cls._create_xml(version, input_structure=old) cmd.append(file_name) cmd.append('-extra-info') cmd.append(cls.results_dir) cmd.append('-report-path') cmd.append(cls.pkgdiff_results_full_path_html) try: ret_code = ProcessHelper.run_subprocess( cmd, output_file=ProcessHelper.DEV_NULL) except OSError: raise CheckerNotFoundError( "Checker '{}' was not found or installed.".format(cls.NAME)) # From pkgdiff source code: # ret_code 0 means unchanged # ret_code 1 means Changed # other return codes means error if int(ret_code) != 0 and int(ret_code) != 1: raise RebaseHelperError( 'Execution of {} failed.\nCommand line is: {}'.format( cls.NAME, cmd)) results_dict = cls.process_xml_results(cls.results_dir) lines = [] for key, val in six.iteritems(results_dict): if val: if lines: lines.append('') lines.append('Following files were {}:'.format(key)) lines.extend(val) pkgdiff_report = os.path.join(cls.results_dir, cls.pkgdiff_results_filename + '.txt') try: with open(pkgdiff_report, "w") as f: f.write('\n'.join(lines)) except IOError: raise RebaseHelperError( "Unable to write result from {} to '{}'".format( cls.NAME, pkgdiff_report)) return dict(path=cls.get_checker_output_dir_short())
def test_simple_cmd_with_input_path_and_redirected_output_path(self): with open(self.IN_FILE, "w") as f: f.write(self.PHRASE) assert os.path.exists(self.IN_FILE) assert open(self.IN_FILE).readline().strip("\n") == self.PHRASE ret = ProcessHelper.run_subprocess(self.CAT_COMMAND, input=self.IN_FILE, output=self.OUT_FILE) assert ret == 0 assert os.path.exists(self.OUT_FILE) assert open(self.OUT_FILE).readline().strip("\n") == self.PHRASE
def run_check(cls, results_dir, **kwargs): """Compares old and new RPMs using abipkgdiff""" # Check if ABI changes occured cls.abi_changes = None cls.results_dir = os.path.join(results_dir, cls.NAME) os.makedirs(cls.results_dir) debug_old, rest_pkgs_old = cls._get_packages_for_abipkgdiff( results_store.get_build('old')) debug_new, rest_pkgs_new = cls._get_packages_for_abipkgdiff( results_store.get_build('new')) cmd = [cls.NAME] reports = {} for pkg in rest_pkgs_old: command = list(cmd) debug = cls._find_debuginfo(debug_old, pkg) if debug: command.append('--d1') command.append(debug) old_name = RpmHelper.split_nevra(os.path.basename(pkg))['name'] find = [ x for x in rest_pkgs_new if RpmHelper.split_nevra( os.path.basename(x))['name'] == old_name ] if not find: logger.warning('New version of package %s was not found!', old_name) continue new_pkg = find[0] debug = cls._find_debuginfo(debug_new, new_pkg) if debug: command.append('--d2') command.append(debug) command.append(pkg) command.append(new_pkg) logger.debug('Package name for ABI comparison %s', old_name) output = os.path.join(cls.results_dir, old_name + '.txt') try: ret_code = ProcessHelper.run_subprocess(command, output_file=output) except OSError: raise CheckerNotFoundError( "Checker '{}' was not found or installed.".format( cls.NAME)) if int(ret_code) & cls.ABIDIFF_ERROR and int( ret_code) & cls.ABIDIFF_USAGE_ERROR: raise RebaseHelperError( 'Execution of {} failed.\nCommand line is: {}'.format( cls.NAME, cmd)) reports[old_name] = int(ret_code) return dict(packages=cls.parse_abi_logs(reports), abi_changes=cls.abi_changes, path=cls.get_checker_output_dir_short())
def test_simple_cmd_with_input_path_and_redirected_output_path(self): with open(self.IN_FILE, 'w') as f: f.write(self.PHRASE) assert os.path.exists(self.IN_FILE) assert open(self.IN_FILE).readline().strip('\n') == self.PHRASE ret = ProcessHelper.run_subprocess(self.CAT_COMMAND, input=self.IN_FILE, output=self.OUT_FILE) assert ret == 0 assert os.path.exists(self.OUT_FILE) assert open(self.OUT_FILE).readline().strip('\n') == self.PHRASE
def test_simple_cmd_with_input_fileobject_and_redirected_output_path(self): in_buff = StringIO() in_buff.write(self.PHRASE) assert not os.path.exists(self.IN_FILE) in_buff.seek(0) assert in_buff.readline().strip("\n") == self.PHRASE ret = ProcessHelper.run_subprocess(self.CAT_COMMAND, input=in_buff, output=self.OUT_FILE) in_buff.close() assert ret == 0 assert os.path.exists(self.OUT_FILE) assert open(self.OUT_FILE).readline().strip("\n") == self.PHRASE
def run_check(cls, result_dir): """Compares old and new RPMs using abipkgdiff""" debug_old, rest_pkgs_old = cls._get_packages_for_abipkgdiff( results_store.get_build('old')) debug_new, rest_pkgs_new = cls._get_packages_for_abipkgdiff( results_store.get_build('new')) cmd = [cls.CMD] reports = {} for pkg in rest_pkgs_old: command = list(cmd) debug = cls._find_debuginfo(debug_old, pkg) if debug: command.append('--d1') command.append(debug) old_name = RpmHelper.split_nevra(os.path.basename(pkg))['name'] find = [ x for x in rest_pkgs_new if RpmHelper.split_nevra( os.path.basename(x))['name'] == old_name ] if not find: logger.warning('New version of package %s was not found!', old_name) continue new_pkg = find[0] debug = cls._find_debuginfo(debug_new, new_pkg) if debug: command.append('--d2') command.append(debug) command.append(pkg) command.append(new_pkg) logger.debug('Package name for ABI comparison %s', old_name) output = os.path.join(cls.results_dir, result_dir, old_name + '-' + cls.log_name) try: ret_code = ProcessHelper.run_subprocess(command, output=output) except OSError: raise CheckerNotFoundError( "Checker '%s' was not found or installed." % cls.CMD) if int(ret_code) & settings.ABIDIFF_ERROR and int( ret_code) & settings.ABIDIFF_USAGE_ERROR: raise RebaseHelperError( 'Execution of %s failed.\nCommand line is: %s' % (cls.CMD, cmd)) if int(ret_code) == 0: text = 'ABI of the compared binaries in package %s are equal.' % old_name else: text = 'ABI of the compared binaries in package %s are not equal.' % old_name reports[output] = text return reports
def run_check(cls, results_dir): """ Compares old and new RPMs using pkgdiff :param results_dir result dir where are stored results """ cls.results_dir = results_dir cls.pkgdiff_results_full_path = os.path.join(cls.results_dir, cls.pkgdiff_results_filename) cmd = [cls.CMD] cmd.append('-hide-unchanged') for version in ['old', 'new']: old = results_store.get_build(version) if old: file_name = cls._create_xml(version, input_structure=old) cmd.append(file_name) cmd.append('-extra-info') cmd.append(cls.results_dir) cmd.append('-report-path') cmd.append(cls.pkgdiff_results_full_path) try: ret_code = ProcessHelper.run_subprocess(cmd, output=ProcessHelper.DEV_NULL) except OSError: raise CheckerNotFoundError("Checker '%s' was not found or installed." % cls.CMD) """ From pkgdiff source code: ret_code 0 means unchanged ret_code 1 means Changed other return codes means error """ if int(ret_code) != 0 and int(ret_code) != 1: raise RebaseHelperError('Execution of %s failed.\nCommand line is: %s' % (cls.CMD, cmd)) results_dict = cls.process_xml_results(cls.results_dir) lines = [] for key, val in six.iteritems(results_dict): if val: if lines: lines.append('') lines.append('Following files were %s:' % key) lines.extend(val) pkgdiff_report = os.path.join(cls.results_dir, 'report-' + cls.pkgdiff_results_filename + '.log') try: with open(pkgdiff_report, "w") as f: f.write('\n'.join(lines)) except IOError: raise RebaseHelperError("Unable to write result from %s to '%s'" % (cls.CMD, pkgdiff_report)) return {pkgdiff_report: None}
def _build_srpm(cls, spec, workdir, results_dir, srpm_results_dir, srpm_builder_options): """ Build SRPM using mock. :param spec: abs path to SPEC file inside the rpmbuild/SPECS in workdir. :param workdir: abs path to working directory with rpmbuild directory structure, which will be used as HOME dir. :param results_dir: abs path to dir where the log should be placed. :param srpm_results_dir: path to directory where SRPM will be placed. :param srpm_builder_options: list of additional options for mock build tool(eg. '-r fedora-XX-x86_64'). :return: abs path to built SRPM. """ logger.info("Building SRPM") spec_loc = os.path.dirname(spec) output = os.path.join(results_dir, "build.log") path_to_sources = os.path.join(workdir, 'SOURCES') cmd = ['mock', '--old-chroot', '--buildsrpm'] if srpm_builder_options is not None: cmd.extend(srpm_builder_options) cmd.extend(['--spec', spec]) cmd.extend(['--sources', path_to_sources]) cmd.extend(['--resultdir', results_dir]) ret = ProcessHelper.run_subprocess_cwd_env(cmd, cwd=spec_loc, env={'HOME': workdir}, output_file=output) build_log_path = os.path.join(srpm_results_dir, 'build.log') mock_log_path = os.path.join(srpm_results_dir, 'mock_output.log') root_log_path = os.path.join(srpm_results_dir, 'root.log') if ret == 0: return PathHelper.find_first_file(workdir, '*.src.rpm') if ret == 1: if not os.path.exists(build_log_path) and os.path.exists( mock_log_path): logfile = mock_log_path else: logfile = build_log_path else: logfile = root_log_path cls.logs = [ l for l in PathHelper.find_all_files(srpm_results_dir, '*.log') ] raise SourcePackageBuildError("Building SRPM failed!", logfile=logfile)
def run_check(cls, results_dir): """Compares old and new RPMs using pkgdiff""" csmock_report = {} old_pkgs = OutputLogger.get_old_build().get('srpm', None) new_pkgs = OutputLogger.get_new_build().get('srpm', None) csmock_dir = os.path.join(results_dir, cls.CMD) os.makedirs(csmock_dir) arguments = ['--force', '-a', '-r', 'fedora-rawhide-x86_64', '--base-srpm'] if old_pkgs and new_pkgs: cmd = [cls.CMD] cmd.extend(arguments) cmd.append(old_pkgs) cmd.append(new_pkgs) cmd.extend(['-o', csmock_dir]) output = StringIO() try: ProcessHelper.run_subprocess(cmd, output=output) except OSError: raise CheckerNotFoundError("Checker '%s' was not found or installed." % cls.CMD) csmock_report['error'] = PathHelper.find_all_files_current_dir(csmock_dir, '*.err') csmock_report['txt'] = PathHelper.find_all_files_current_dir(csmock_dir, '*.txt') csmock_report['log'] = PathHelper.find_all_files_current_dir(csmock_dir, '*.log') return csmock_report
def _build_source_from_instructions(cls, instructions, source, logfile): """Run instructions to create source archive""" logger.info("Attempting to create source '%s' using instructions in comments", source) with TemporaryEnvironment() as tmp: script = os.path.join(tmp.path(), 'script.sh') with open(script, 'w') as f: f.write('#!/bin/sh -x\n') f.write(''.join(instructions)) f.write('cp "{}" "{}"\n'.format(source, os.getcwd())) os.chmod(script, 0o755) result = ProcessHelper.run_subprocess_cwd(script, tmp.path(), output=logfile, shell=True) if result == 0 and os.path.isfile(source): logger.info('Source creation succeeded.') else: logger.info('Source creation failed.')
def test_simple_cmd_with_input_fileobject_and_redirected_output_path(self): in_buff = StringIO() in_buff.write(self.PHRASE) assert not os.path.exists(self.IN_FILE) in_buff.seek(0) assert in_buff.readline().strip('\n') == self.PHRASE ret = ProcessHelper.run_subprocess(self.CAT_COMMAND, input=in_buff, output=self.OUT_FILE) in_buff.close() assert ret == 0 assert os.path.exists(self.OUT_FILE) assert open(self.OUT_FILE).readline().strip('\n') == self.PHRASE
def _build_rpm(cls, srpm, results_dir, rpm_results_dir, root=None, arch=None, builder_options=None): """ Build RPM using mock. :param srpm: full path to the srpm. :param results_dir: abs path to dir where the log should be placed. :param rpm_results_dir: directory where rpms will be placed. :param root: path to where chroot should be built. :param arch: target architectures for the build. :param builder_options: builder_options for mock. :return abs paths to RPMs. """ logger.info("Building RPMs") output = os.path.join(results_dir, "mock_output.log") cmd = [ cls.CMD, '--old-chroot', '--rebuild', srpm, '--resultdir', results_dir ] if root is not None: cmd.extend(['--root', root]) if arch is not None: cmd.extend(['--arch', arch]) if builder_options is not None: cmd.extend(builder_options) ret = ProcessHelper.run_subprocess(cmd, output_file=output) if ret == 0: return [ f for f in PathHelper.find_all_files(results_dir, '*.rpm') if not f.endswith('.src.rpm') ] else: logfile = MockBuildTool.get_mock_logfile_path(ret, rpm_results_dir, tmp_path=results_dir) cls.logs.extend( [l for l in PathHelper.find_all_files(rpm_results_dir, '*.log')]) raise BinaryPackageBuildError("Building RPMs failed!", rpm_results_dir, logfile=logfile)
def test_simple_cmd_with_input_path_and_redirected_output_fileobject(self): out_buff = StringIO() with open(self.IN_FILE, 'w') as f: f.write(self.PHRASE) assert os.path.exists(self.IN_FILE) assert open(self.IN_FILE).readline().strip('\n') == self.PHRASE ret = ProcessHelper.run_subprocess(self.CAT_COMMAND, input=self.IN_FILE, output=out_buff) assert ret == 0 assert not os.path.exists(self.OUT_FILE) out_buff.seek(0) assert out_buff.readline().strip('\n') == self.PHRASE out_buff.close()
def _build_rpm(cls, srpm, results_dir, root=None, arch=None): """Build RPM using mock.""" logger.info("Building RPMs") output = os.path.join(results_dir, "mock_output.log") cmd = [cls.CMD, '--rebuild', srpm, '--resultdir', results_dir] if root is not None: cmd.extend(['--root', root]) if arch is not None: cmd.extend(['--arch', arch]) ret = ProcessHelper.run_subprocess(cmd, output=output) if ret != 0: return None else: return [f for f in PathHelper.find_all_files(results_dir, '*.rpm') if not f.endswith('.src.rpm')]
def test_setting_new_env(self): # make copy of existing environment en_variables = os.environ.copy().keys() # pick up non-existing name while True: rand_name = "".join(random.choice(string.ascii_letters) for _ in range(6)).upper() if rand_name not in en_variables: break cmd = 'echo "$' + rand_name + '"' ret = ProcessHelper.run_subprocess_cwd_env( cmd, env={rand_name: self.PHRASE}, output=self.OUT_FILE, shell=True ) assert ret == 0 assert os.path.exists(self.OUT_FILE) assert open(self.OUT_FILE).readline().strip("\n") == self.PHRASE
def test_setting_existing_env(self): # make copy of existing environment en_variables = list(os.environ.copy().keys()) # there are no variables set on the system -> nothing to test if not en_variables: pass assert os.environ.get(en_variables[0]) != self.PHRASE cmd = 'echo "$' + en_variables[0] + '"' ret = ProcessHelper.run_subprocess_cwd_env( cmd, env={en_variables[0]: self.PHRASE}, output=self.OUT_FILE, shell=True ) assert ret == 0 assert os.path.exists(self.OUT_FILE) assert open(self.OUT_FILE).readline().strip("\n") == self.PHRASE
def _build_srpm(cls, spec, sources, results_dir, root=None, arch=None): """Build SRPM using mock.""" logger.info("Building SRPM") output = os.path.join(results_dir, "mock_output.log") cmd = [cls.CMD, '--buildsrpm', '--spec', spec, '--sources', sources, '--resultdir', results_dir] if root is not None: cmd.extend(['--root', root]) if arch is not None: cmd.extend(['--arch', arch]) ret = ProcessHelper.run_subprocess(cmd, output=output) if ret != 0: return None else: return PathHelper.find_first_file(results_dir, '*.src.rpm')
def test_setting_new_env(self): # make copy of existing environment en_variables = os.environ.copy().keys() # pick up non-existing name while True: rand_name = ''.join(random.choice(string.ascii_letters) for _ in range(6)).upper() if rand_name not in en_variables: break cmd = 'echo "$' + rand_name + '"' ret = ProcessHelper.run_subprocess_cwd_env(cmd, env={rand_name: self.PHRASE}, output=self.OUT_FILE, shell=True) assert ret == 0 assert os.path.exists(self.OUT_FILE) assert open(self.OUT_FILE).readline().strip("\n") == self.PHRASE
def test_setting_existing_env(self): # make copy of existing environment en_variables = list(os.environ.copy().keys()) # there are no variables set on the system -> nothing to test if not en_variables: pass assert os.environ.get(en_variables[0]) != self.PHRASE cmd = 'echo "$' + en_variables[0] + '"' ret = ProcessHelper.run_subprocess_cwd_env(cmd, env={en_variables[0]: self.PHRASE}, output=self.OUT_FILE, shell=True) assert ret == 0 assert os.path.exists(self.OUT_FILE) assert open(self.OUT_FILE).readline().strip("\n") == self.PHRASE
def run_check(cls, results_dir): """ Compares old and new RPMs using pkgdiff """ cls.results_dir = results_dir debug_old, rest_pkgs_old = cls._get_packages_for_abipkgdiff(OutputLogger.get_build('old')) debug_new, rest_pkgs_new = cls._get_packages_for_abipkgdiff(OutputLogger.get_build('new')) cmd = [cls.CMD] cmd.append('--d1') cmd.append(debug_old[0]) cmd.append('--d2') cmd.append(debug_new[0]) text = [] for pkg in rest_pkgs_old: command = list(cmd) # Package can be <letters><numbers>-<letters>-<and_whatever> regexp = r'^(\w*)(-\D+)?.*$' reg = re.compile(regexp) matched = reg.search(os.path.basename(pkg)) if matched: file_name = matched.group(1) command.append(pkg) find = [x for x in rest_pkgs_new if os.path.basename(x).startswith(file_name)] command.append(find[0]) package_name = os.path.basename(os.path.basename(pkg)) logger.debug('Package name for ABI comparision %s', package_name) regexp_name = r'(\w-)*(\D+)*' reg_name = re.compile(regexp_name) matched = reg_name.search(os.path.basename(pkg)) logger.debug('Found matches %s', matched.groups()) if matched: package_name = matched.group(0) + cls.log_name else: package_name = package_name + '-' + cls.log_name output = os.path.join(results_dir, package_name) ret_code = ProcessHelper.run_subprocess(command, output=output) if int(ret_code) & settings.ABIDIFF_ERROR and int(ret_code) & settings.ABIDIFF_USAGE_ERROR: raise RebaseHelperError('Execution of %s failed.\nCommand line is: %s', cls.CMD, cmd) if int(ret_code) == 0: text.append('ABI of the compared binaries in package %s are equal.' % package_name) else: text.append('ABI of the compared binaries in package %s are not equal. See file %s' % (package_name, output)) else: logger.debug("Rebase-helper did not find a package name in '%s'", package_name) return text
def _build_srpm(cls, spec, workdir, results_dir, srpm_results_dir, srpm_builder_options): """ Build SRPM using rpmbuild. :param spec: abs path to SPEC file inside the rpmbuild/SPECS in workdir. :param workdir: abs path to working directory with rpmbuild directory structure, which will be used as HOME dir. :param results_dir: abs path to dir where the log should be placed. :param srpm_results_dir: path to directory where SRPM will be placed. :param srpm_builder_options: list of additional options to rpmbuild. :return: abs path to built SRPM. """ logger.info("Building SRPM") spec_loc, spec_name = os.path.split(spec) output = os.path.join(results_dir, "build.log") cmd = ['rpmbuild', '-bs', spec_name] if srpm_builder_options is not None: cmd.extend(srpm_builder_options) ret = ProcessHelper.run_subprocess_cwd_env(cmd, cwd=spec_loc, env={'HOME': workdir}, output_file=output) build_log_path = os.path.join(srpm_results_dir, 'build.log') if ret == 0: return PathHelper.find_first_file(workdir, '*.src.rpm') # An error occurred, raise an exception logfile = build_log_path cls.logs = [ l for l in PathHelper.find_all_files(srpm_results_dir, '*.log') ] raise SourcePackageBuildError("Building SRPM failed!", logfile=logfile)
def test_simple_cmd_with_redirected_output_path(self): ret = ProcessHelper.run_subprocess(self.ECHO_COMMAND, output=self.OUT_FILE) assert ret == 0 assert os.path.exists(self.OUT_FILE) assert open(self.OUT_FILE).readline().strip("\n") == self.PHRASE
def test_simple_cmd_changed_work_dir(self): os.mkdir(self.TEMP_DIR) ret = ProcessHelper.run_subprocess_cwd(self.TOUCH_COMMAND, self.TEMP_DIR) assert ret == 0 assert os.path.exists(os.path.join(self.TEMP_DIR, self.TEMP_FILE))
def get_license_changes(cls, old_dir, new_dir): """ Finds differences in licenses between old and new source files. Args: old_dir(str): Path to the old sources directory. new_dir(str): Path to the new sources directory. Returns: tuple: Changes dictionary, new_licenses set, disappeared_licenses set. """ diffs = [] possible_license_names = r'COPYING|LICENSE|LICENCE|LICENSING|BSD|(L)?GPL(v[23][+]?)?' for source_dir in [old_dir, new_dir]: out = six.StringIO() ProcessHelper.run_subprocess([ "/usr/bin/licensecheck", source_dir, "--machine", "--recursive" ], output_file=out) diff = {} for l in out: # licensecheck output format: 'Filepath\tlicense' file_path, dlicense = l.split('\t') if 'GENERATED FILE' in dlicense: continue file_path = os.path.relpath(file_path, source_dir) diff[file_path] = dlicense.strip() diffs.append(diff) old_lics, new_lics = set(), set() changes = {'appeared': {}, 'transitioned': {}, 'disappeared': {}} # Get changed licenses in existing files for new_file, new_license in six.iteritems(diffs[1]): new_lics.add(new_license) for old_file, old_license in six.iteritems(diffs[0]): old_lics.add(old_license) if (new_file == old_file and (new_license != old_license)): new_key = '{} => {}'.format(old_license, new_license) if new_license == 'UNKNOWN': # Conversion `known license` => `None/Unknown` if re.search(possible_license_names, new_file, re.IGNORECASE): cls.license_files_changes.update({ old_license: 'disappeared in {}'.format(new_file) }) if old_license not in changes['disappeared']: changes['disappeared'][old_license] = [] changes['disappeared'][old_license].append(new_file) elif old_license == 'UNKNOWN': # Conversion `None/Unknown` => `known license` if re.search(possible_license_names, new_file, re.IGNORECASE): cls.license_files_changes.update({ new_license: 'appeared in {}'.format(new_file) }) if new_license not in changes['appeared']: changes['appeared'][new_license] = [] changes['appeared'][new_license].append(new_file) else: # Conversion `known license` => `known license` if re.search(possible_license_names, new_file, re.IGNORECASE): cls.license_files_changes.update( {new_key: 'in {}'.format(new_file)}) if new_key not in changes['transitioned']: changes['transitioned'][new_key] = [] if new_file not in changes['transitioned'][new_key]: changes['transitioned'][new_key].append(new_file) # Get newly appeared files for new_file, new_license in six.iteritems(diffs[1]): if new_file not in diffs[0]: if new_license == 'UNKNOWN': continue if re.search(possible_license_names, new_file, re.IGNORECASE): cls.license_files_changes.update( {new_license: 'appeared in {}'.format(new_file)}) if new_license not in changes['appeared']: changes['appeared'][new_license] = [] changes['appeared'][new_license].append(new_file) # Get removed files for old_file, old_license in six.iteritems(diffs[0]): if old_file not in diffs[1]: if old_license == 'UNKNOWN': continue if re.search(possible_license_names, old_file, re.IGNORECASE): cls.license_files_changes.update( {old_license: 'disappeared in {}'.format(old_file)}) if old_license not in changes['disappeared']: changes['disappeared'][old_license] = [] changes['disappeared'][old_license].append(old_file) new_licenses = new_lics - old_lics disappeared_licenses = old_lics - new_lics if new_licenses or disappeared_licenses: cls.license_changes = True return changes, new_licenses, disappeared_licenses
def test_simple_cmd(self): ret = ProcessHelper.run_subprocess(self.TOUCH_COMMAND) assert ret == 0 assert os.path.exists(self.TEMP_FILE)
def test_simple_cmd_with_redirected_output_path(self): ret = ProcessHelper.run_subprocess(self.ECHO_COMMAND, output=self.OUT_FILE) assert ret == 0 assert os.path.exists(self.OUT_FILE) assert open(self.OUT_FILE).readline().strip('\n') == self.PHRASE