def run_check(cls, results_dir): """ Compares old and new RPMs using pkgdiff """ results_dict = {} for tag in settings.CHECKER_TAGS: results_dict[tag] = [] cls.results_dir = results_dir # Only S (size), M(mode) and 5 (checksum) are now important not_catched_flags = ['T', 'F', 'G', 'U', 'V', 'L', 'D', 'N'] old_pkgs = cls._get_rpms(OutputLogger.get_build('old').get('rpm', None)) new_pkgs = cls._get_rpms(OutputLogger.get_build('new').get('rpm', None)) for key, value in six.iteritems(old_pkgs): cmd = [cls.CMD] # TODO modify to online command for x in not_catched_flags: cmd.extend(['-i', x]) cmd.append(value) # We would like to build correct old package against correct new packages cmd.append(new_pkgs[key]) output = StringIO() ProcessHelper.run_subprocess(cmd, output=output) results_dict = cls._analyze_logs(output, results_dict) results_dict = cls.update_added_removed(results_dict) results_dict = dict((k, v) for k, v in six.iteritems(results_dict) if v) text = [] for key, val in six.iteritems(results_dict): text.append('Following files were %s:\n%s' % (key, '\n'.join(val))) return text
def _prepare_spec_objects(self): """ Prepare spec files and initialize objects :return: """ self.rebase_spec_file_path = get_rebase_name(self.spec_file_path) self.spec_file = SpecFile(self.spec_file_path, self.execution_dir, download=not self.conf.not_download_sources) # Check whether test suite is enabled at build time if not self.spec_file.is_test_suite_enabled(): OutputLogger.set_info_text('WARNING', 'Test suite is not enabled at build time.') # create an object representing the rebased SPEC file self.rebase_spec_file = self.spec_file.copy(self.rebase_spec_file_path) # check if argument passed as new source is a file or just a version if [True for ext in Archive.get_supported_archives() if self.conf.sources.endswith(ext)]: logger.debug("argument passed as a new source is a file") self.rebase_spec_file.set_version_using_archive(self.conf.sources) else: logger.debug("argument passed as a new source is a version") version, extra_version = SpecFile.split_version_string(self.conf.sources) self.rebase_spec_file.set_version(version) if extra_version: self.rebase_spec_file.set_extra_version(extra_version)
def print_summary(cls, path): """ Function is used for printing summary informations :return: """ for key, value in six.iteritems(OutputLogger.get_summary_info()): logger.info("%s %s\n", key, value) try: LoggerHelper.add_file_handler(logger_report, path) except (OSError, IOError): raise RebaseHelperError("Can not create results file '%s'" % path) type_pkgs = ['old', 'new'] if OutputLogger.get_patches(): cls.print_patches(OutputLogger.get_patches(), '\nSummary information about patches:') for pkg in type_pkgs: type_pkg = OutputLogger.get_build(pkg) if type_pkg: cls.print_rpms(type_pkg, pkg.capitalize()) cls.print_build_logs(type_pkg, pkg.capitalize()) cls.print_pkgdiff_tool()
def run_check(cls, results_dir): """ Compares old and new RPMs using pkgdiff """ cls.results_dir = results_dir cls.pkgdiff_results_full_path = os.path.join(cls.results_dir, cls.pkgdiff_results_filename) versions = ['old', 'new'] cmd = [cls.CMD] for version in versions: old = OutputLogger.get_build(version) if old: file_name = cls._create_xml(version, input_structure=old) cmd.append(file_name) cmd.append('-extra-info') cmd.append(cls.results_dir) cmd.append('-report-path') cmd.append(cls.pkgdiff_results_full_path) ret_code = ProcessHelper.run_subprocess(cmd, output=ProcessHelper.DEV_NULL) """ From pkgdiff source code: ret_code 0 means unchanged ret_code 1 means Changed other return codes means error """ if int(ret_code) != 0 and int(ret_code) != 1: raise RebaseHelperError('Execution of %s failed.\nCommand line is: %s', cls.CMD, cmd) OutputLogger.set_info_text('Result HTML page from pkgdiff is store in: ', cls.pkgdiff_results_full_path) results_dict = cls.process_xml_results(cls.results_dir) text = [] for key, val in six.iteritems(results_dict): text.append('Following files were %s:\n%s' % (key, '\n'.join(val))) return text
def print_summary(cls, path): """ Function is used for printing summary informations :return: """ # First of all we would like to print all # summary information OutputLogger.set_info_text("Summary output is also available in log:", path) logger.info('\n') for key, value in six.iteritems(OutputLogger.get_summary_info()): logger.info("%s %s\n", key, value) try: LoggerHelper.add_file_handler(logger_output, path) except (OSError, IOError): raise RebaseHelperError("Can not create results file '%s'", path) type_pkgs = ['old', 'new'] cls.print_patches(OutputLogger.get_patches(), '\nSummary information about patches:') for pkg in type_pkgs: type_pkg = OutputLogger.get_build(pkg) if type_pkg: cls.print_rpms(type_pkg, pkg.capitalize()) cls.print_build_logs(type_pkg, pkg.capitalize()) cls.print_pkgdiff_tool()
def run_check(cls, results_dir): """ Compares old and new RPMs using pkgdiff """ results_dict = {} for tag in settings.CHECKER_TAGS: results_dict[tag] = [] cls.results_dir = results_dir # Only S (size), M(mode) and 5 (checksum) are now important not_catched_flags = ['T', 'F', 'G', 'U', 'V', 'L', 'D', 'N'] old_pkgs = cls._get_rpms(OutputLogger.get_build('old').get('rpm', None)) new_pkgs = cls._get_rpms(OutputLogger.get_build('new').get('rpm', None)) for key, value in six.iteritems(old_pkgs): cmd = [cls.CMD] # TODO modify to online command for x in not_catched_flags: cmd.extend(['-i', x]) cmd.append(value) # We would like to build correct old package against correct new packages cmd.append(new_pkgs[key]) output = StringIO() ret_code = ProcessHelper.run_subprocess(cmd, output=output) results_dict = cls._analyze_logs(output, results_dict) results_dict = cls.update_added_removed(results_dict) # TODO Check for changed files and # remove them from 'removed' and 'added' #cls._unpack_rpm(old_pkgs) #cls._unpack_rpm(new_pkgs) #cls._find_file_diffs(old_pkgs, new_pkgs) return results_dict
def run_check(cls, result_dir): """Compares old and new RPMs using pkgdiff""" debug_old, rest_pkgs_old = cls._get_packages_for_abipkgdiff(OutputLogger.get_build('old')) debug_new, rest_pkgs_new = cls._get_packages_for_abipkgdiff(OutputLogger.get_build('new')) cmd = [cls.CMD] if debug_old is None: logger.warning("Package doesn't contain any debug package") return None try: cmd.append('--d1') cmd.append(debug_old[0]) except IndexError: logger.error('Debuginfo package not found for old package.') return None try: cmd.append('--d2') cmd.append(debug_new[0]) except IndexError: logger.error('Debuginfo package not found for new package.') return None reports = {} for pkg in rest_pkgs_old: command = list(cmd) # Package can be <letters><numbers>-<letters>-<and_whatever> regexp = r'^(\w*)(-\D+)?.*$' reg = re.compile(regexp) matched = reg.search(os.path.basename(pkg)) if matched: file_name = matched.group(1) command.append(pkg) find = [x for x in rest_pkgs_new if os.path.basename(x).startswith(file_name)] command.append(find[0]) package_name = os.path.basename(os.path.basename(pkg)) logger.debug('Package name for ABI comparision %s', package_name) regexp_name = r'(\w-)*(\D+)*' reg_name = re.compile(regexp_name) matched = reg_name.search(os.path.basename(pkg)) logger.debug('Found matches %s', matched.groups()) if matched: package_name = matched.group(0) + cls.log_name else: package_name = package_name + '-' + cls.log_name output = os.path.join(cls.results_dir, result_dir, package_name) try: ret_code = ProcessHelper.run_subprocess(command, output=output) except OSError: raise CheckerNotFoundError("Checker '%s' was not found or installed." % cls.CMD) if int(ret_code) & settings.ABIDIFF_ERROR and int(ret_code) & settings.ABIDIFF_USAGE_ERROR: raise RebaseHelperError('Execution of %s failed.\nCommand line is: %s' % (cls.CMD, cmd)) if int(ret_code) == 0: text = 'ABI of the compared binaries in package %s are equal.' % package_name else: text = 'ABI of the compared binaries in package %s are not equal.' % package_name reports[output] = text else: logger.debug("Rebase-helper did not find a package name in '%s'", package_name) return reports
def print_pkgdiff_tool(cls): """ Function prints a summary information about pkgcomparetool """ if OutputLogger.get_checkers(): for check, data in six.iteritems(OutputLogger.get_checkers()): if data: logger_output.info("%s:\n%s", check, data)
def __init__(self, cli_conf=None): """ Initialize the application :param cli_conf: CLI object with configuration gathered from commandline :return: """ OutputLogger.clear() self.conf = cli_conf if self.conf.verbose: LoggerHelper.add_stream_handler(logger, logging.DEBUG) else: LoggerHelper.add_stream_handler(logger, logging.INFO) # The directory in which rebase-helper was executed if self.conf.results_dir is None: self.execution_dir = os.getcwd() else: self.execution_dir = self.conf.results_dir # Temporary workspace for Builder, checks, ... self.kwargs['workspace_dir'] = self.workspace_dir = os.path.join(self.execution_dir, settings.REBASE_HELPER_WORKSPACE_DIR) # Directory where results should be put self.kwargs['results_dir'] = self.results_dir = os.path.join(self.execution_dir, settings.REBASE_HELPER_RESULTS_DIR) self.kwargs['non_interactive'] = self.conf.non_interactive # if not continuing, check the results dir if not self.conf.cont and not self.conf.build_only and not self.conf.comparepkgs: self._check_results_dir() # This is used if user executes rebase-helper with --continue # parameter even when directory does not exist if not os.path.exists(self.results_dir): os.makedirs(self.results_dir) os.makedirs(os.path.join(self.results_dir, settings.REBASE_HELPER_LOGS)) self._add_debug_log_file() self._add_report_log_file() logger.debug("Rebase-helper version: %s" % version.VERSION) if self.conf.build_tasks is None: self._get_spec_file() self._prepare_spec_objects() # check the workspace dir if not self.conf.cont: self._check_workspace_dir() # TODO: Remove the value from kwargs and use only CLI attribute! self.kwargs['continue'] = self.conf.cont self._initialize_data() if self.conf.cont or self.conf.build_only: self._delete_old_builds()
def get_checker_outputs(self): checkers = {} if OutputLogger.get_checkers(): for check, data in six.iteritems(OutputLogger.get_checkers()): if data: for log in six.iterkeys(data): if FileHelper.file_available(log): checkers[check] = log else: checkers[check] = None return checkers
def print_pkgdiff_tool(cls): """ Function prints a summary information about pkgcomparetool """ if OutputLogger.get_checkers(): for check, data in six.iteritems(OutputLogger.get_checkers()): if data: logger_output.info("%s", check) for checker, output in six.iteritems(data): logger_output.info("===Checker %s===\n%s\n", checker, '\n'.join(output))
def print_pkgdiff_tool(cls): """ Function prints a summary information about pkgcomparetool """ if OutputLogger.get_checkers(): for check, data in six.iteritems(OutputLogger.get_checkers()): logger_report.info("=== Checker %s results ===", check) for checker, output in six.iteritems(data): if output is None: logger_report.info("Log is available here: %s\n", checker) else: logger_report.info("%s See for more details %s", output, checker)
def pkgdiff_packages(self): """ Function calls pkgdiff class for comparing packages :return: """ try: pkgchecker = Checker(self.conf.pkgcomparetool) except NotImplementedError: raise RebaseHelperError('You have to specify one of these check tools %s', Checker.get_supported_tools()) else: logger.info('Comparing packages using %s...', self.conf.pkgcomparetool) results_dict = pkgchecker.run_check(self.results_dir) for key, val in six.iteritems(results_dict): if val: OutputLogger.set_checker_output('Following files were ' + key, '\n'.join(val))
def pkgdiff_packages(self): """ Function calls pkgdiff class for comparing packages :return: """ pkgdiff_results = {} if not self.conf.pkgcomparetool: for checker in Checker.get_supported_tools(): text = self._execute_checkers(checker) pkgdiff_results[checker] = text else: text = self._execute_checkers(self.conf.pkgcomparetool) pkgdiff_results[self.conf.pkgcomparetool] = text OutputLogger.set_checker_output('\nResults from checker(s):', pkgdiff_results)
def pkgdiff_packages(self): """ Function calls pkgdiff class for comparing packages :return: """ pkgdiff_results = {} if not self.conf.pkgcomparetool: for checker in Checker.get_supported_tools(): results = self._execute_checkers(checker) pkgdiff_results[checker] = results else: text = self._execute_checkers(self.conf.pkgcomparetool) pkgdiff_results[self.conf.pkgcomparetool] = text for diff_name, result in six.iteritems(pkgdiff_results): OutputLogger.set_checker_output(diff_name, result)
def get_rebased_patches(self): """ Function returns a list of patches either '': [list_of_deleted_patches] :return: """ patches = False output_patch_string = [] if OutputLogger.get_patches(): for key, val in six.iteritems(OutputLogger.get_patches()): if key: output_patch_string.append('Following patches have been %s:\n%s' % (key, val)) patches = True if not patches: output_patch_string.append('Patches were not touched. All were applied properly') return output_patch_string
def test_base_output_builds_old(self): """ Test Output logger old builds :return: """ build_results = OutputLogger.get_build('old') assert build_results == self.old_rpm_data
def test_base_output_builds_new(self): """ Test Output logger new builds :return: """ build_results = OutputLogger.get_build('new') assert build_results == self.new_rpm_data
def test_base_output_patches(self): """ Test Output logger patches :return: """ patch_results = OutputLogger.get_patches() expected_patches = self.patches_data assert patch_results == expected_patches
def run_check(cls, results_dir): """ Compares old and new RPMs using pkgdiff """ cls.results_dir = results_dir debug_old, rest_pkgs_old = cls._get_packages_for_abipkgdiff(OutputLogger.get_build('old')) debug_new, rest_pkgs_new = cls._get_packages_for_abipkgdiff(OutputLogger.get_build('new')) cmd = [cls.CMD] cmd.append('--d1') cmd.append(debug_old[0]) cmd.append('--d2') cmd.append(debug_new[0]) text = [] for pkg in rest_pkgs_old: command = list(cmd) # Package can be <letters><numbers>-<letters>-<and_whatever> regexp = r'^(\w*)(-\D+)?.*$' reg = re.compile(regexp) matched = reg.search(os.path.basename(pkg)) if matched: file_name = matched.group(1) command.append(pkg) find = [x for x in rest_pkgs_new if os.path.basename(x).startswith(file_name)] command.append(find[0]) package_name = os.path.basename(os.path.basename(pkg)) logger.debug('Package name for ABI comparision %s', package_name) regexp_name = r'(\w-)*(\D+)*' reg_name = re.compile(regexp_name) matched = reg_name.search(os.path.basename(pkg)) logger.debug('Found matches %s', matched.groups()) if matched: package_name = matched.group(0) + cls.log_name else: package_name = package_name + '-' + cls.log_name output = os.path.join(results_dir, package_name) ret_code = ProcessHelper.run_subprocess(command, output=output) if int(ret_code) & settings.ABIDIFF_ERROR and int(ret_code) & settings.ABIDIFF_USAGE_ERROR: raise RebaseHelperError('Execution of %s failed.\nCommand line is: %s', cls.CMD, cmd) if int(ret_code) == 0: text.append('ABI of the compared binaries in package %s are equal.' % package_name) else: text.append('ABI of the compared binaries in package %s are not equal. See file %s' % (package_name, output)) else: logger.debug("Rebase-helper did not find a package name in '%s'", package_name) return text
def run_download_compare(self, tasks_dict, dir_name): self.set_upstream_monitoring() kh = KojiHelper() for version in ['old', 'new']: rh_dict = {} compare_dirname = os.path.join(dir_name, version) if not os.path.exists(compare_dirname): os.mkdir(compare_dirname, 0o777) (task, upstream_version, package) = tasks_dict[version] rh_dict['rpm'], rh_dict['logs'] = kh.get_koji_tasks([task], compare_dirname) rh_dict['version'] = upstream_version rh_dict['name'] = package OutputLogger.set_build_data(version, rh_dict) if tasks_dict['status'] == 'CLOSED': self.pkgdiff_packages(dir_name) self.print_summary() rh_stuff = self.get_rebasehelper_data() logger.info(rh_stuff) return rh_stuff
def test_base_output_info(self): """ Test Output logger info :return: """ info_results = OutputLogger.get_summary_info() expect_dict = self.info_data expect_dict.update(self.info_data2) assert info_results == expect_dict
def pkgdiff_packages(self, dir_name): """ Function calls pkgdiff class for comparing packages :param dir_name: specify a result dir :return: """ pkgdiff_results = {} checker = Checker(os.path.dirname(__file__)) if not self.conf.pkgcomparetool: for check in checker.get_supported_tools(): try: results = checker.run_check(dir_name, checker_name=check) pkgdiff_results[check] = results except CheckerNotFoundError: logger.info("Rebase-helper did not find checker '%s'." % check) else: pkgdiff_results[self.conf.pkgcomparetool] = checker.run_check(dir_name, checker_name=self.conf.pkgcomparetool) if pkgdiff_results: for diff_name, result in six.iteritems(pkgdiff_results): OutputLogger.set_checker_output(diff_name, result)
def patch_sources(self, sources): # Patch sources git_helper = GitHelper(sources[0]) git_helper.check_git_config() patch = Patcher(self.conf.patchtool) self.rebase_spec_file.update_changelog(self.rebase_spec_file.get_new_log(git_helper)) try: self.rebased_patches = patch.patch(sources[0], sources[1], self.rest_sources, git_helper, self.spec_file.get_applied_patches(), self.spec_file.get_prep_section(), **self.kwargs) except RuntimeError as run_e: raise RebaseHelperError('Patching failed') self.rebase_spec_file.write_updated_patches(self.rebased_patches) if self.conf.non_interactive: OutputLogger.set_patch_output('Unapplied patches:', self.rebased_patches['unapplied']) OutputLogger.set_patch_output('Patches:', self.rebased_patches)
def run_check(cls, results_dir): """Compares old and new RPMs using pkgdiff""" results_dict = {} for tag in settings.CHECKER_TAGS: results_dict[tag] = [] cls.results_dir = results_dir # Only S (size), M(mode) and 5 (checksum) are now important not_catched_flags = ['T', 'F', 'G', 'U', 'V', 'L', 'D', 'N'] old_pkgs = cls._get_rpms(OutputLogger.get_old_build().get('rpm', None)) new_pkgs = cls._get_rpms(OutputLogger.get_new_build().get('rpm', None)) for key, value in six.iteritems(old_pkgs): cmd = [cls.CMD] # TODO modify to online command for x in not_catched_flags: cmd.extend(['-i', x]) cmd.append(value) # We would like to build correct old package against correct new packages cmd.append(new_pkgs[key]) output = StringIO() try: ProcessHelper.run_subprocess(cmd, output=output) except OSError: raise CheckerNotFoundError("Checker '%s' was not found or installed." % cls.CMD) results_dict = cls._analyze_logs(output, results_dict) results_dict = cls.update_added_removed(results_dict) results_dict = dict((k, v) for k, v in six.iteritems(results_dict) if v) text = [] for key, val in six.iteritems(results_dict): text.append('Following files were %s:\n%s' % (key, '\n'.join(val))) pkgdiff_report = os.path.join(cls.results_dir, 'report-' + cls.CMD + '.log') try: with open(pkgdiff_report, "w") as f: f.writelines(text) except IOError: raise RebaseHelperError("Unable to write result from %s to '%s'" % (cls.CMD, pkgdiff_report)) return {pkgdiff_report: None}
def __init__(self, cli_conf, execution_dir, debug_log_file, report_log_file): """ Initialize the application :param cli_conf: CLI object with configuration gathered from commandline :return: """ OutputLogger.clear() self.conf = cli_conf self.execution_dir = execution_dir self.debug_log_file = debug_log_file self.report_log_file = report_log_file # Temporary workspace for Builder, checks, ... self.kwargs['workspace_dir'] = self.workspace_dir = os.path.join(self.execution_dir, settings.REBASE_HELPER_WORKSPACE_DIR) # Directory where results should be put self.kwargs['results_dir'] = self.results_dir = os.path.join(self.execution_dir, settings.REBASE_HELPER_RESULTS_DIR) self.kwargs['non_interactive'] = self.conf.non_interactive logger.debug("Rebase-helper version: %s" % version.VERSION) if self.conf.build_tasks is None: self._get_spec_file() self._prepare_spec_objects() # check the workspace dir if not self.conf.cont: self._check_workspace_dir() # TODO: Remove the value from kwargs and use only CLI attribute! self.kwargs['continue'] = self.conf.cont self._initialize_data() if self.conf.cont or self.conf.build_only: self._delete_old_builds()
def run_package_checkers(self, results_dir): """ Runs checkers on packages and stores results in a given directory. :param results_dir: Path to directory in which to store the results. :type results_dir: str :return: None """ results = dict() if self.conf.pkgcomparetool: results[self.conf.pkgcomparetool] = checkers_runner.run_checker(results_dir, self.conf.pkgcomparetool) else: # no specific checker was given, just run all of them for checker_name in checkers_runner.get_supported_tools(): try: results[checker_name] = checkers_runner.run_checker(results_dir, checker_name) except CheckerNotFoundError: logger.error("Rebase-helper did not find checker '%s'." % checker_name) for diff_name, result in six.iteritems(results): OutputLogger.set_checker_output(diff_name, result)
def get_rpm_packages(self, dirname): """ Function returns RPM packages stored in dirname/old and dirname/new directories :param dirname: directory where are stored old and new RPMS :return: """ found = True for version in ['old', 'new']: data = {} data['name'] = self.spec_file.get_package_name() if version == 'old': spec_version = self.spec_file.get_version() else: spec_version = self.rebase_spec_file.get_version() data['version'] = spec_version data['rpm'] = PathHelper.find_all_files(os.path.join(os.path.realpath(dirname), version, 'RPM'), '*.rpm') if not data['rpm']: logger.error('Your path %s%s/RPM does not contain any RPM packages' % (dirname, version)) found = False OutputLogger.set_build_data(version, data) if not found: return False return True
def test_text_output(self): output = OutputTool('text') data = self.get_data() OutputLogger.set_build_data('old', data['old']) OutputLogger.set_build_data('new', data['new']) OutputLogger.set_patch_output('Patches:', data['patches']) test_output = {'pkgdiff': ['Following files were moved\n%s' % '\n'.join(data['moved'])]} OutputLogger.set_checker_output('Results from checker(s)', test_output) logfile = os.path.join(self.TESTS_DIR, REBASE_HELPER_RESULTS_LOG) output.print_information(logfile) with open(logfile) as f: lines = [y.strip() for y in f.readlines()] assert lines == self.get_expected_output().split('\n') os.unlink(logfile)
def run_check(cls, results_dir): """Compares old and new RPMs using pkgdiff""" csmock_report = {} old_pkgs = OutputLogger.get_old_build().get('srpm', None) new_pkgs = OutputLogger.get_new_build().get('srpm', None) csmock_dir = os.path.join(results_dir, cls.CMD) os.makedirs(csmock_dir) arguments = ['--force', '-a', '-r', 'fedora-rawhide-x86_64', '--base-srpm'] if old_pkgs and new_pkgs: cmd = [cls.CMD] cmd.extend(arguments) cmd.append(old_pkgs) cmd.append(new_pkgs) cmd.extend(['-o', csmock_dir]) output = StringIO() try: ProcessHelper.run_subprocess(cmd, output=output) except OSError: raise CheckerNotFoundError("Checker '%s' was not found or installed." % cls.CMD) csmock_report['error'] = PathHelper.find_all_files_current_dir(csmock_dir, '*.err') csmock_report['txt'] = PathHelper.find_all_files_current_dir(csmock_dir, '*.txt') csmock_report['log'] = PathHelper.find_all_files_current_dir(csmock_dir, '*.log') return csmock_report