def run_check(cls, results_dir): """ Compares old and new RPMs using pkgdiff """ results_dict = {} for tag in settings.CHECKER_TAGS: results_dict[tag] = [] cls.results_dir = results_dir # Only S (size), M(mode) and 5 (checksum) are now important not_catched_flags = ['T', 'F', 'G', 'U', 'V', 'L', 'D', 'N'] old_pkgs = cls._get_rpms(OutputLogger.get_build('old').get('rpm', None)) new_pkgs = cls._get_rpms(OutputLogger.get_build('new').get('rpm', None)) for key, value in six.iteritems(old_pkgs): cmd = [cls.CMD] # TODO modify to online command for x in not_catched_flags: cmd.extend(['-i', x]) cmd.append(value) # We would like to build correct old package against correct new packages cmd.append(new_pkgs[key]) output = StringIO() ProcessHelper.run_subprocess(cmd, output=output) results_dict = cls._analyze_logs(output, results_dict) results_dict = cls.update_added_removed(results_dict) results_dict = dict((k, v) for k, v in six.iteritems(results_dict) if v) text = [] for key, val in six.iteritems(results_dict): text.append('Following files were %s:\n%s' % (key, '\n'.join(val))) return text
def call_prep_script(cls, prep_script_path): cwd = os.getcwd() os.chdir(cls.old_sources) ProcessHelper.run_subprocess(prep_script_path, output=os.path.join(cls.kwargs['results_dir'], 'prep_script.log')) if not cls.patch_sources_by_prep_script: cls.git_helper.command_add_files(parameters=["--all"]) cls.git_helper.command_commit(message="prep_script prep_corrections") os.chdir(cwd)
def run_check(cls, results_dir): """ Compares old and new RPMs using pkgdiff """ cls.results_dir = results_dir cls.pkgdiff_results_full_path = os.path.join(cls.results_dir, cls.pkgdiff_results_filename) versions = ['old', 'new'] cmd = [cls.CMD] for version in versions: old = OutputLogger.get_build(version) if old: file_name = cls._create_xml(version, input_structure=old) cmd.append(file_name) cmd.append('-extra-info') cmd.append(cls.results_dir) cmd.append('-report-path') cmd.append(cls.pkgdiff_results_full_path) ret_code = ProcessHelper.run_subprocess(cmd, output=ProcessHelper.DEV_NULL) """ From pkgdiff source code: ret_code 0 means unchanged ret_code 1 means Changed other return codes means error """ if int(ret_code) != 0 and int(ret_code) != 1: raise RebaseHelperError('Execution of %s failed.\nCommand line is: %s', cls.CMD, cmd) OutputLogger.set_info_text('Result HTML page from pkgdiff is store in: ', cls.pkgdiff_results_full_path) results_dict = cls.process_xml_results(cls.results_dir) text = [] for key, val in six.iteritems(results_dict): text.append('Following files were %s:\n%s' % (key, '\n'.join(val))) return text
def run_check(cls, results_dir): """ Compares old and new RPMs using pkgdiff """ results_dict = {} for tag in settings.CHECKER_TAGS: results_dict[tag] = [] cls.results_dir = results_dir # Only S (size), M(mode) and 5 (checksum) are now important not_catched_flags = ['T', 'F', 'G', 'U', 'V', 'L', 'D', 'N'] old_pkgs = cls._get_rpms(OutputLogger.get_build('old').get('rpm', None)) new_pkgs = cls._get_rpms(OutputLogger.get_build('new').get('rpm', None)) for key, value in six.iteritems(old_pkgs): cmd = [cls.CMD] # TODO modify to online command for x in not_catched_flags: cmd.extend(['-i', x]) cmd.append(value) # We would like to build correct old package against correct new packages cmd.append(new_pkgs[key]) output = StringIO() ret_code = ProcessHelper.run_subprocess(cmd, output=output) results_dict = cls._analyze_logs(output, results_dict) results_dict = cls.update_added_removed(results_dict) # TODO Check for changed files and # remove them from 'removed' and 'added' #cls._unpack_rpm(old_pkgs) #cls._unpack_rpm(new_pkgs) #cls._find_file_diffs(old_pkgs, new_pkgs) return results_dict
def test_simple_cmd_with_redirected_output_fileobject(self): buff = StringIO() ret = ProcessHelper.run_subprocess(self.ECHO_COMMAND, output=buff) assert ret == 0 assert not os.path.exists(self.OUT_FILE) assert buff.readline().strip("\n") == self.PHRASE buff.close()
def run_check(cls, result_dir): """Compares old and new RPMs using pkgdiff""" debug_old, rest_pkgs_old = cls._get_packages_for_abipkgdiff(OutputLogger.get_build('old')) debug_new, rest_pkgs_new = cls._get_packages_for_abipkgdiff(OutputLogger.get_build('new')) cmd = [cls.CMD] if debug_old is None: logger.warning("Package doesn't contain any debug package") return None try: cmd.append('--d1') cmd.append(debug_old[0]) except IndexError: logger.error('Debuginfo package not found for old package.') return None try: cmd.append('--d2') cmd.append(debug_new[0]) except IndexError: logger.error('Debuginfo package not found for new package.') return None reports = {} for pkg in rest_pkgs_old: command = list(cmd) # Package can be <letters><numbers>-<letters>-<and_whatever> regexp = r'^(\w*)(-\D+)?.*$' reg = re.compile(regexp) matched = reg.search(os.path.basename(pkg)) if matched: file_name = matched.group(1) command.append(pkg) find = [x for x in rest_pkgs_new if os.path.basename(x).startswith(file_name)] command.append(find[0]) package_name = os.path.basename(os.path.basename(pkg)) logger.debug('Package name for ABI comparision %s', package_name) regexp_name = r'(\w-)*(\D+)*' reg_name = re.compile(regexp_name) matched = reg_name.search(os.path.basename(pkg)) logger.debug('Found matches %s', matched.groups()) if matched: package_name = matched.group(0) + cls.log_name else: package_name = package_name + '-' + cls.log_name output = os.path.join(cls.results_dir, result_dir, package_name) try: ret_code = ProcessHelper.run_subprocess(command, output=output) except OSError: raise CheckerNotFoundError("Checker '%s' was not found or installed." % cls.CMD) if int(ret_code) & settings.ABIDIFF_ERROR and int(ret_code) & settings.ABIDIFF_USAGE_ERROR: raise RebaseHelperError('Execution of %s failed.\nCommand line is: %s' % (cls.CMD, cmd)) if int(ret_code) == 0: text = 'ABI of the compared binaries in package %s are equal.' % package_name else: text = 'ABI of the compared binaries in package %s are not equal.' % package_name reports[output] = text else: logger.debug("Rebase-helper did not find a package name in '%s'", package_name) return reports
def run_check(cls, results_dir): """Compares old and new RPMs using rpmdiff""" results_dict = {} for tag in settings.CHECKER_TAGS: results_dict[tag] = [] cls.results_dir = results_dir # Only S (size), M(mode) and 5 (checksum) are now important not_catched_flags = ['T', 'F', 'G', 'U', 'V', 'L', 'D', 'N'] old_pkgs = cls._get_rpms(results_store.get_old_build().get('rpm', None)) new_pkgs = cls._get_rpms(results_store.get_new_build().get('rpm', None)) for key, value in six.iteritems(old_pkgs): cmd = [cls.CMD] # TODO modify to online command for x in not_catched_flags: cmd.extend(['-i', x]) cmd.append(value) # We would like to build correct old package against correct new packages cmd.append(new_pkgs[key]) output = StringIO() try: ProcessHelper.run_subprocess(cmd, output=output) except OSError: raise CheckerNotFoundError("Checker '%s' was not found or installed." % cls.CMD) results_dict = cls._analyze_logs(output, results_dict) results_dict = cls.update_added_removed(results_dict) results_dict = dict((k, v) for k, v in six.iteritems(results_dict) if v) lines = [] for key, val in six.iteritems(results_dict): if val: if lines: lines.append('') lines.append('Following files were %s:' % key) lines.extend(val) rpmdiff_report = os.path.join(cls.results_dir, 'report-' + cls.CMD + '.log') try: with open(rpmdiff_report, "w") as f: f.write('\n'.join(lines)) except IOError: raise RebaseHelperError("Unable to write result from %s to '%s'" % (cls.CMD, rpmdiff_report)) return {rpmdiff_report: None}
def test_simple_cmd_with_input_path_and_redirected_output_path(self): with open(self.IN_FILE, "w") as f: f.write(self.PHRASE) assert os.path.exists(self.IN_FILE) assert open(self.IN_FILE).readline().strip("\n") == self.PHRASE ret = ProcessHelper.run_subprocess(self.CAT_COMMAND, input=self.IN_FILE, output=self.OUT_FILE) assert ret == 0 assert os.path.exists(self.OUT_FILE) assert open(self.OUT_FILE).readline().strip("\n") == self.PHRASE
def test_simple_cmd_with_input_fileobject_and_redirected_output_path(self): in_buff = StringIO() in_buff.write(self.PHRASE) assert not os.path.exists(self.IN_FILE) in_buff.seek(0) assert in_buff.readline().strip("\n") == self.PHRASE ret = ProcessHelper.run_subprocess(self.CAT_COMMAND, input=in_buff, output=self.OUT_FILE) in_buff.close() assert ret == 0 assert os.path.exists(self.OUT_FILE) assert open(self.OUT_FILE).readline().strip("\n") == self.PHRASE
def run_check(cls, result_dir): """Compares old and new RPMs using abipkgdiff""" debug_old, rest_pkgs_old = cls._get_packages_for_abipkgdiff( results_store.get_build('old')) debug_new, rest_pkgs_new = cls._get_packages_for_abipkgdiff( results_store.get_build('new')) cmd = [cls.CMD] reports = {} for pkg in rest_pkgs_old: command = list(cmd) debug = cls._find_debuginfo(debug_old, pkg) if debug: command.append('--d1') command.append(debug) old_name = RpmHelper.split_nevra(os.path.basename(pkg))['name'] find = [ x for x in rest_pkgs_new if RpmHelper.split_nevra( os.path.basename(x))['name'] == old_name ] if not find: logger.warning('New version of package %s was not found!', old_name) continue new_pkg = find[0] debug = cls._find_debuginfo(debug_new, new_pkg) if debug: command.append('--d2') command.append(debug) command.append(pkg) command.append(new_pkg) logger.debug('Package name for ABI comparison %s', old_name) output = os.path.join(cls.results_dir, result_dir, old_name + '-' + cls.log_name) try: ret_code = ProcessHelper.run_subprocess(command, output=output) except OSError: raise CheckerNotFoundError( "Checker '%s' was not found or installed." % cls.CMD) if int(ret_code) & settings.ABIDIFF_ERROR and int( ret_code) & settings.ABIDIFF_USAGE_ERROR: raise RebaseHelperError( 'Execution of %s failed.\nCommand line is: %s' % (cls.CMD, cmd)) if int(ret_code) == 0: text = 'ABI of the compared binaries in package %s are equal.' % old_name else: text = 'ABI of the compared binaries in package %s are not equal.' % old_name reports[output] = text return reports
def run_check(cls, results_dir): """ Compares old and new RPMs using pkgdiff :param results_dir result dir where are stored results """ cls.results_dir = results_dir cls.pkgdiff_results_full_path = os.path.join(cls.results_dir, cls.pkgdiff_results_filename) cmd = [cls.CMD] cmd.append('-hide-unchanged') for version in ['old', 'new']: old = results_store.get_build(version) if old: file_name = cls._create_xml(version, input_structure=old) cmd.append(file_name) cmd.append('-extra-info') cmd.append(cls.results_dir) cmd.append('-report-path') cmd.append(cls.pkgdiff_results_full_path) try: ret_code = ProcessHelper.run_subprocess(cmd, output=ProcessHelper.DEV_NULL) except OSError: raise CheckerNotFoundError("Checker '%s' was not found or installed." % cls.CMD) """ From pkgdiff source code: ret_code 0 means unchanged ret_code 1 means Changed other return codes means error """ if int(ret_code) != 0 and int(ret_code) != 1: raise RebaseHelperError('Execution of %s failed.\nCommand line is: %s' % (cls.CMD, cmd)) results_dict = cls.process_xml_results(cls.results_dir) lines = [] for key, val in six.iteritems(results_dict): if val: if lines: lines.append('') lines.append('Following files were %s:' % key) lines.extend(val) pkgdiff_report = os.path.join(cls.results_dir, 'report-' + cls.pkgdiff_results_filename + '.log') try: with open(pkgdiff_report, "w") as f: f.write('\n'.join(lines)) except IOError: raise RebaseHelperError("Unable to write result from %s to '%s'" % (cls.CMD, pkgdiff_report)) return {pkgdiff_report: None}
def run_check(cls, results_dir): """Compares old and new RPMs using pkgdiff""" csmock_report = {} old_pkgs = results_store.get_old_build().get('srpm', None) new_pkgs = results_store.get_new_build().get('srpm', None) csmock_dir = os.path.join(results_dir, cls.CMD) os.makedirs(csmock_dir) arguments = ['--force', '-a', '-r', 'fedora-rawhide-x86_64', '--base-srpm'] if old_pkgs and new_pkgs: cmd = [cls.CMD] cmd.extend(arguments) cmd.append(old_pkgs) cmd.append(new_pkgs) cmd.extend(['-o', csmock_dir]) output = StringIO() try: ProcessHelper.run_subprocess(cmd, output=output) except OSError: raise CheckerNotFoundError("Checker '%s' was not found or installed." % cls.CMD) csmock_report['error'] = PathHelper.find_all_files_current_dir(csmock_dir, '*.err') csmock_report['txt'] = PathHelper.find_all_files_current_dir(csmock_dir, '*.txt') csmock_report['log'] = PathHelper.find_all_files_current_dir(csmock_dir, '*.log') return csmock_report
def run_check(cls, results_dir): """Compares old and new RPMs using pkgdiff""" csmock_report = {} old_pkgs = OutputLogger.get_old_build().get('srpm', None) new_pkgs = OutputLogger.get_new_build().get('srpm', None) csmock_dir = os.path.join(results_dir, cls.CMD) os.makedirs(csmock_dir) arguments = ['--force', '-a', '-r', 'fedora-rawhide-x86_64', '--base-srpm'] if old_pkgs and new_pkgs: cmd = [cls.CMD] cmd.extend(arguments) cmd.append(old_pkgs) cmd.append(new_pkgs) cmd.extend(['-o', csmock_dir]) output = StringIO() try: ProcessHelper.run_subprocess(cmd, output=output) except OSError: raise CheckerNotFoundError("Checker '%s' was not found or installed." % cls.CMD) csmock_report['error'] = PathHelper.find_all_files_current_dir(csmock_dir, '*.err') csmock_report['txt'] = PathHelper.find_all_files_current_dir(csmock_dir, '*.txt') csmock_report['log'] = PathHelper.find_all_files_current_dir(csmock_dir, '*.log') return csmock_report
def test_simple_cmd_with_input_fileobject_and_redirected_output_path(self): in_buff = StringIO() in_buff.write(self.PHRASE) assert not os.path.exists(self.IN_FILE) in_buff.seek(0) assert in_buff.readline().strip('\n') == self.PHRASE ret = ProcessHelper.run_subprocess(self.CAT_COMMAND, input=in_buff, output=self.OUT_FILE) in_buff.close() assert ret == 0 assert os.path.exists(self.OUT_FILE) assert open(self.OUT_FILE).readline().strip('\n') == self.PHRASE
def _build_rpm(cls, srpm, results_dir, rpm_results_dir, root=None, arch=None, builder_options=None): """ Build RPM using mock. :param srpm: full path to the srpm. :param results_dir: abs path to dir where the log should be placed. :param rpm_results_dir: directory where rpms will be placed. :param root: path to where chroot should be built. :param arch: target architectures for the build. :param builder_options: builder_options for mock. :return abs paths to RPMs. """ logger.info("Building RPMs") output = os.path.join(results_dir, "mock_output.log") cmd = [ cls.CMD, '--old-chroot', '--rebuild', srpm, '--resultdir', results_dir ] if root is not None: cmd.extend(['--root', root]) if arch is not None: cmd.extend(['--arch', arch]) if builder_options is not None: cmd.extend(builder_options) ret = ProcessHelper.run_subprocess(cmd, output_file=output) if ret == 0: return [ f for f in PathHelper.find_all_files(results_dir, '*.rpm') if not f.endswith('.src.rpm') ] else: logfile = MockBuildTool.get_mock_logfile_path(ret, rpm_results_dir, tmp_path=results_dir) cls.logs.extend( [l for l in PathHelper.find_all_files(rpm_results_dir, '*.log')]) raise BinaryPackageBuildError("Building RPMs failed!", rpm_results_dir, logfile=logfile)
def test_simple_cmd_with_input_path_and_redirected_output_fileobject(self): out_buff = StringIO() with open(self.IN_FILE, 'w') as f: f.write(self.PHRASE) assert os.path.exists(self.IN_FILE) assert open(self.IN_FILE).readline().strip('\n') == self.PHRASE ret = ProcessHelper.run_subprocess(self.CAT_COMMAND, input=self.IN_FILE, output=out_buff) assert ret == 0 assert not os.path.exists(self.OUT_FILE) out_buff.seek(0) assert out_buff.readline().strip('\n') == self.PHRASE out_buff.close()
def _build_rpm(cls, srpm, results_dir, root=None, arch=None): """Build RPM using mock.""" logger.info("Building RPMs") output = os.path.join(results_dir, "mock_output.log") cmd = [cls.CMD, '--rebuild', srpm, '--resultdir', results_dir] if root is not None: cmd.extend(['--root', root]) if arch is not None: cmd.extend(['--arch', arch]) ret = ProcessHelper.run_subprocess(cmd, output=output) if ret != 0: return None else: return [f for f in PathHelper.find_all_files(results_dir, '*.rpm') if not f.endswith('.src.rpm')]
def _build_srpm(cls, spec, sources, results_dir, root=None, arch=None): """Build SRPM using mock.""" logger.info("Building SRPM") output = os.path.join(results_dir, "mock_output.log") cmd = [cls.CMD, '--buildsrpm', '--spec', spec, '--sources', sources, '--resultdir', results_dir] if root is not None: cmd.extend(['--root', root]) if arch is not None: cmd.extend(['--arch', arch]) ret = ProcessHelper.run_subprocess(cmd, output=output) if ret != 0: return None else: return PathHelper.find_first_file(results_dir, '*.src.rpm')
def run_check(cls, results_dir): """ Compares old and new RPMs using pkgdiff """ cls.results_dir = results_dir debug_old, rest_pkgs_old = cls._get_packages_for_abipkgdiff(OutputLogger.get_build('old')) debug_new, rest_pkgs_new = cls._get_packages_for_abipkgdiff(OutputLogger.get_build('new')) cmd = [cls.CMD] cmd.append('--d1') cmd.append(debug_old[0]) cmd.append('--d2') cmd.append(debug_new[0]) text = [] for pkg in rest_pkgs_old: command = list(cmd) # Package can be <letters><numbers>-<letters>-<and_whatever> regexp = r'^(\w*)(-\D+)?.*$' reg = re.compile(regexp) matched = reg.search(os.path.basename(pkg)) if matched: file_name = matched.group(1) command.append(pkg) find = [x for x in rest_pkgs_new if os.path.basename(x).startswith(file_name)] command.append(find[0]) package_name = os.path.basename(os.path.basename(pkg)) logger.debug('Package name for ABI comparision %s', package_name) regexp_name = r'(\w-)*(\D+)*' reg_name = re.compile(regexp_name) matched = reg_name.search(os.path.basename(pkg)) logger.debug('Found matches %s', matched.groups()) if matched: package_name = matched.group(0) + cls.log_name else: package_name = package_name + '-' + cls.log_name output = os.path.join(results_dir, package_name) ret_code = ProcessHelper.run_subprocess(command, output=output) if int(ret_code) & settings.ABIDIFF_ERROR and int(ret_code) & settings.ABIDIFF_USAGE_ERROR: raise RebaseHelperError('Execution of %s failed.\nCommand line is: %s', cls.CMD, cmd) if int(ret_code) == 0: text.append('ABI of the compared binaries in package %s are equal.' % package_name) else: text.append('ABI of the compared binaries in package %s are not equal. See file %s' % (package_name, output)) else: logger.debug("Rebase-helper did not find a package name in '%s'", package_name) return text
def get_license_changes(cls, old_dir, new_dir): """ Finds differences in licenses between old and new source files. Args: old_dir(str): Path to the old sources directory. new_dir(str): Path to the new sources directory. Returns: tuple: Changes dictionary, new_licenses set, disappeared_licenses set. """ diffs = [] possible_license_names = r'COPYING|LICENSE|LICENCE|LICENSING|BSD|(L)?GPL(v[23][+]?)?' for source_dir in [old_dir, new_dir]: out = six.StringIO() ProcessHelper.run_subprocess([ "/usr/bin/licensecheck", source_dir, "--machine", "--recursive" ], output_file=out) diff = {} for l in out: # licensecheck output format: 'Filepath\tlicense' file_path, dlicense = l.split('\t') if 'GENERATED FILE' in dlicense: continue file_path = os.path.relpath(file_path, source_dir) diff[file_path] = dlicense.strip() diffs.append(diff) old_lics, new_lics = set(), set() changes = {'appeared': {}, 'transitioned': {}, 'disappeared': {}} # Get changed licenses in existing files for new_file, new_license in six.iteritems(diffs[1]): new_lics.add(new_license) for old_file, old_license in six.iteritems(diffs[0]): old_lics.add(old_license) if (new_file == old_file and (new_license != old_license)): new_key = '{} => {}'.format(old_license, new_license) if new_license == 'UNKNOWN': # Conversion `known license` => `None/Unknown` if re.search(possible_license_names, new_file, re.IGNORECASE): cls.license_files_changes.update({ old_license: 'disappeared in {}'.format(new_file) }) if old_license not in changes['disappeared']: changes['disappeared'][old_license] = [] changes['disappeared'][old_license].append(new_file) elif old_license == 'UNKNOWN': # Conversion `None/Unknown` => `known license` if re.search(possible_license_names, new_file, re.IGNORECASE): cls.license_files_changes.update({ new_license: 'appeared in {}'.format(new_file) }) if new_license not in changes['appeared']: changes['appeared'][new_license] = [] changes['appeared'][new_license].append(new_file) else: # Conversion `known license` => `known license` if re.search(possible_license_names, new_file, re.IGNORECASE): cls.license_files_changes.update( {new_key: 'in {}'.format(new_file)}) if new_key not in changes['transitioned']: changes['transitioned'][new_key] = [] if new_file not in changes['transitioned'][new_key]: changes['transitioned'][new_key].append(new_file) # Get newly appeared files for new_file, new_license in six.iteritems(diffs[1]): if new_file not in diffs[0]: if new_license == 'UNKNOWN': continue if re.search(possible_license_names, new_file, re.IGNORECASE): cls.license_files_changes.update( {new_license: 'appeared in {}'.format(new_file)}) if new_license not in changes['appeared']: changes['appeared'][new_license] = [] changes['appeared'][new_license].append(new_file) # Get removed files for old_file, old_license in six.iteritems(diffs[0]): if old_file not in diffs[1]: if old_license == 'UNKNOWN': continue if re.search(possible_license_names, old_file, re.IGNORECASE): cls.license_files_changes.update( {old_license: 'disappeared in {}'.format(old_file)}) if old_license not in changes['disappeared']: changes['disappeared'][old_license] = [] changes['disappeared'][old_license].append(old_file) new_licenses = new_lics - old_lics disappeared_licenses = old_lics - new_lics if new_licenses or disappeared_licenses: cls.license_changes = True return changes, new_licenses, disappeared_licenses
def test_simple_cmd_with_redirected_output_path(self): ret = ProcessHelper.run_subprocess(self.ECHO_COMMAND, output=self.OUT_FILE) assert ret == 0 assert os.path.exists(self.OUT_FILE) assert open(self.OUT_FILE).readline().strip('\n') == self.PHRASE
def test_simple_cmd(self): ret = ProcessHelper.run_subprocess(self.TOUCH_COMMAND) assert ret == 0 assert os.path.exists(self.TEMP_FILE)
def test_simple_cmd_with_redirected_output_path(self): ret = ProcessHelper.run_subprocess(self.ECHO_COMMAND, output=self.OUT_FILE) assert ret == 0 assert os.path.exists(self.OUT_FILE) assert open(self.OUT_FILE).readline().strip("\n") == self.PHRASE