def rebase_helper_checkers(self, new_version, old_task_id, new_task_id, tmp_dir): argument = [ '--non-interactive', '--builds-nowait', '--results-dir', tmp_dir, '--fedpkg-build-tasks', old_task_id + ',' + new_task_id, new_version, ] rh_app = None rh_stuff = None try: cli = CLI(argument) rh_app = Application(cli) ret_code = rh_app.run() if int(ret_code) != 0: self.log.warn('Comparing package were not successful') rh_stuff = rh_app.get_rebasehelper_data() except Exception: self.log.exception('Compare packages failed.') self.log.info(rh_stuff) return rh_stuff
def rebase_helper(self, package, upstream, tmp, bz): """ Rebase helper part which does a rebase a inform package maintainer whether package was rebased properly. Output information are in dictionary rh_stuff. """ self.log.info("Rebase-helper is going to rebase package") rh_stuff = {} result_rh = -1 rh_app = None url = self.git_url.format(package=package) self.log.info("Cloning %r to %r" % (url, tmp)) sh.git.clone(url, tmp) os.chdir(tmp) try: argument = [ '--non-interactive', '--builds-nowait', '--buildtool', 'fedpkg', upstream ] cli = CLI(argument) rh_app = Application(cli) rh_app.set_upstream_monitoring() self.log.info("Rebasehelper package %s %s" % (package, upstream)) result_rh = rh_app.run() self.log.info("Rebasehelper finish properly") except Exception as ex: self.log.info( 'builsys.py: Rebase helper failed with unknown reason. %s' % str(ex)) rh_stuff = rh_app.get_rebasehelper_data() self.log.info(rh_stuff) return result_rh, rh_stuff
def test_rebase(self, buildtool, package, version, patches): cli = CLI([ '--non-interactive', '--disable-inapplicable-patches', '--buildtool', buildtool, '--outputtool', 'json', '--pkgcomparetool', 'rpmdiff,pkgdiff,abipkgdiff', '--color=always', '--apply-changes', version ]) config = Config() config.merge(cli) execution_dir, results_dir, debug_log_file = Application.setup(config) app = Application(config, execution_dir, results_dir, debug_log_file) app.run() with open(os.path.join(RESULTS_DIR, 'report.json')) as f: report = json.load(f) for k in ['deleted', 'modified', 'inapplicable']: assert set(report['patches'].get(k, [])) == (patches[k] or set()) changes = os.path.join(RESULTS_DIR, 'changes.patch') patch = unidiff.PatchSet.from_filename(changes, encoding='UTF-8') pf = [pf for pf in patch if pf.path == '{}.spec'.format(package)] assert pf ver = [l for h in pf[0] for l in h.target if l.startswith('+Version')] assert ver assert version in ver[0] repo = git.Repo(execution_dir) assert '- New upstream release {}'.format( version) in repo.commit().summary
def run(): debug_log_file = None try: cli = CLI() if hasattr(cli, 'version'): logger.info(VERSION) sys.exit(0) config = Config(getattr(cli, 'config-file', None)) config.merge(cli) for handler in [main_handler, output_tool_handler]: handler.set_terminal_background(config.background) ConsoleHelper.use_colors = ConsoleHelper.should_use_colors(config) execution_dir, results_dir, debug_log_file = Application.setup( config) traceback_log = os.path.join(results_dir, LOGS_DIR, TRACEBACK_LOG) if config.verbose == 0: main_handler.setLevel(logging.INFO) elif config.verbose == 1: main_handler.setLevel(CustomLogger.VERBOSE) else: main_handler.setLevel(logging.DEBUG) app = Application(config, execution_dir, results_dir, debug_log_file) app.run() except KeyboardInterrupt: logger.info('Interrupted by user') except RebaseHelperError as e: if e.msg: logger.error('%s', e.msg) else: logger.error('%s', six.text_type(e)) sys.exit(1) except SystemExit as e: sys.exit(e.code) except BaseException: if debug_log_file: logger.error( 'rebase-helper failed due to an unexpected error. Please report this problem' '\nusing the following link: %s' '\nand include the content of' '\n\'%s\' and' '\n\'%s\'' '\nin the report.' '\nThank you!', NEW_ISSUE_LINK, debug_log_file, traceback_log) LoggerHelper.add_file_handler(logger_traceback, traceback_log) logger_traceback.trace('', exc_info=1) else: logger.error( 'rebase-helper failed due to an unexpected error. Please report this problem' '\nusing the following link: %s' '\nThank you!', NEW_ISSUE_LINK) sys.exit(1) sys.exit(0)
def test_application_sources(self, workdir): expected_dict = { 'new': { 'sources': [ os.path.join(workdir, 'test-source.sh'), os.path.join(workdir, 'source-tests.sh'), os.path.join(workdir, self.NEW_SOURCES) ], 'version': '1.0.3', 'name': 'test', 'tarball': self.NEW_SOURCES, 'spec': os.path.join(workdir, settings.REBASE_HELPER_RESULTS_DIR, self.SPEC_FILE), 'patches_full': { 1: [os.path.join(workdir, self.PATCH_1), '', 0, False], 2: [os.path.join(workdir, self.PATCH_2), '-p1', 1, False], 3: [os.path.join(workdir, self.PATCH_3), '-p1', 2, False] } }, 'workspace_dir': os.path.join(workdir, settings.REBASE_HELPER_WORKSPACE_DIR), 'old': { 'sources': [ os.path.join(workdir, 'test-source.sh'), os.path.join(workdir, 'source-tests.sh'), os.path.join(workdir, self.OLD_SOURCES) ], 'version': '1.0.2', 'name': 'test', 'tarball': self.OLD_SOURCES, 'spec': os.path.join(workdir, self.SPEC_FILE), 'patches_full': { 1: [os.path.join(workdir, self.PATCH_1), '', 0, False], 2: [os.path.join(workdir, self.PATCH_2), '-p1', 1, False], 3: [os.path.join(workdir, self.PATCH_3), '-p1', 2, False] } }, 'results_dir': os.path.join(workdir, settings.REBASE_HELPER_RESULTS_DIR) } cli = CLI(self.cmd_line_args) execution_dir, results_dir, debug_log_file = Application.setup(cli) app = Application(cli, execution_dir, results_dir, debug_log_file) app.prepare_sources() for key, val in app.kwargs.items(): if key in expected_dict: assert val == expected_dict[key]
def run(): debug_log_file = None try: # be verbose until debug_log_file is created cli = CLI() if hasattr(cli, 'version'): logger.info(VERSION) sys.exit(0) config = Config(getattr(cli, 'config-file', None)) config.merge(cli) for handler in [main_handler, output_tool_handler]: handler.set_terminal_background(config.background) ConsoleHelper.use_colors = ConsoleHelper.should_use_colors(config) execution_dir, results_dir, debug_log_file = Application.setup( config) if not config.verbose: main_handler.setLevel(logging.INFO) app = Application(config, execution_dir, results_dir, debug_log_file) app.run() except KeyboardInterrupt: logger.info('Interrupted by user') except RebaseHelperError as e: if e.msg: logger.error('%s', e.msg) else: logger.error('%s', six.text_type(e)) sys.exit(1) except SystemExit as e: sys.exit(e.code) except BaseException: if debug_log_file: logger.error( 'rebase-helper failed due to an unexpected error. Please report this problem' '\nusing the following link: %s' '\nand include the content of' '\n\'%s\'' '\nfile in the report.' '\nThank you!', NEW_ISSUE_LINK, debug_log_file) else: logger.error( 'rebase-helper failed due to an unexpected error. Please report this problem' '\nusing the following link: %s' '\nand include the traceback following this message in the report.' '\nThank you!', NEW_ISSUE_LINK) logger.trace('', exc_info=1) sys.exit(1) sys.exit(0)
def test_files_build_log_hook(self, buildtool): new_version = '0.3' cli = CLI([ '--non-interactive', '--disable-inapplicable-patches', '--force-build-log-hooks', '--buildtool', buildtool, '--outputtool', 'json', '--pkgcomparetool', '', '--color=always', new_version, ]) config = Config() config.merge(cli) execution_dir, results_dir = Application.setup(config) app = Application(config, os.getcwd(), execution_dir, results_dir) app.run() changes = os.path.join(RESULTS_DIR, CHANGES_PATCH) patch = unidiff.PatchSet.from_filename(changes, encoding='UTF-8') _, _, spec_file = patch assert spec_file.is_modified_file # removed files assert [h for h in spec_file if '-%doc README.md CHANGELOG.md\n' in h.source] assert [h for h in spec_file if '+%doc README.md\n' in h.target] assert [h for h in spec_file if '-%doc %{_docdir}/%{name}/notes.txt\n' in h.source] assert [h for h in spec_file if '-%{_datadir}/%{name}/1.dat\n' in h.source] assert [h for h in spec_file if '-%{_datadir}/%{name}/extra/C.dat\n' in h.source] assert [h for h in spec_file if '-%doc data/extra/README.extra\n' in h.source] # added files assert [h for h in spec_file if '+%{_datadir}/%{name}/2.dat\n' in h.target] assert [h for h in spec_file if '+%{_datadir}/%{name}/extra/D.dat\n' in h.target] with open(os.path.join(RESULTS_DIR, 'report.json')) as f: report = json.load(f) assert 'success' in report['result'] # files build log hook added = report['build_log_hooks']['files']['added'] assert '%{_datadir}/%{name}/2.dat' in added['%files'] assert '%{_datadir}/%{name}/extra/D.dat' in added['%files extra'] removed = report['build_log_hooks']['files']['removed'] assert 'CHANGELOG.md' in removed['%files'] assert '%{_docdir}/%{name}/notes.txt' in removed['%files'] assert '%{_datadir}/%{name}/1.dat' in removed['%files'] assert '%{_datadir}/%{name}/extra/C.dat' in removed['%files extra'] assert 'data/extra/README.extra' in removed['%files extra']
def run(): debug_log_file = None try: # be verbose until debug_log_file is created handler = LoggerHelper.add_stream_handler(logger, logging.DEBUG) if "--builder-options" in sys.argv[1:]: raise RebaseHelperError( 'Wrong format of --builder-options. It must be in the following form:' ' --builder-options="--desired-builder-option".') cli = CLI() if cli.version: logger.info(VERSION) sys.exit(0) ConsoleHelper.use_colors = ConsoleHelper.should_use_colors(cli) execution_dir, results_dir, debug_log_file = Application.setup(cli) if not cli.verbose: handler.setLevel(logging.INFO) app = Application(cli, execution_dir, results_dir, debug_log_file) app.run() except KeyboardInterrupt: logger.info('\nInterrupted by user') except RebaseHelperError as e: if e.msg: logger.error('\n%s', e.msg) else: logger.error('\n%s', six.text_type(e)) sys.exit(1) except SystemExit as e: sys.exit(e.code) except BaseException: if debug_log_file: logger.error( '\nrebase-helper failed due to an unexpected error. Please report this problem' '\nusing the following link: %s' '\nand include the content of' '\n\'%s\'' '\nfile in the report.' '\nThank you!', NEW_ISSUE_LINK, debug_log_file) else: logger.error( '\nrebase-helper failed due to an unexpected error. Please report this problem' '\nusing the following link: %s' '\nand include the traceback following this message in the report.' '\nThank you!', NEW_ISSUE_LINK) logger.debug('\n', exc_info=1) sys.exit(1) sys.exit(0)
def test_files_build_log_hook(self, buildtool): cli = CLI([ '--non-interactive', '--force-build-log-hooks', '--buildtool', buildtool, '--outputtool', 'json', '--color=always', self.NEW_VERSION, ]) config = Config() config.merge(cli) execution_dir, results_dir, debug_log_file = Application.setup(config) app = Application(config, execution_dir, results_dir, debug_log_file) app.run() changes = os.path.join(RESULTS_DIR, 'changes.patch') patch = unidiff.PatchSet.from_filename(changes, encoding='UTF-8') spec_file = patch[0] assert spec_file.is_modified_file # removed files assert '-%license LICENSE README\n' in spec_file[1].source assert '+%license LICENSE\n' in spec_file[1].target assert '-/dirA/fileB\n' in spec_file[1].source assert '-/dirB/fileY\n' in spec_file[1].source assert '-%doc docs_dir/AUTHORS\n' in spec_file[1].source # added files assert '+/dirA/fileC\n' in spec_file[1].target assert '+/dirB/fileW\n' in spec_file[1].target with open(os.path.join(RESULTS_DIR, 'report.json')) as f: report = json.load(f) assert 'success' in report['result'] added = report['build_log_hooks']['files']['added'] assert '/dirA/fileC' in added['%files'] assert '/dirB/fileW' in added['%files devel'] removed = report['build_log_hooks']['files']['removed'] assert 'README' in removed['%files'] assert '/dirA/fileB' in removed['%files'] assert '/dirB/fileY' in removed['%files devel'] assert 'docs_dir/AUTHORS' in removed['%files devel']
def app(self, make_config): config = make_config(self.cmd_line_args) execution_dir, results_dir = Application.setup(config) app = Application(config, os.getcwd(), execution_dir, results_dir) return app
def test_rebase(self, buildtool, favor_on_conflict): new_version = '0.2' cli = CLI([ '--non-interactive', '--disable-inapplicable-patches', '--buildtool', buildtool, '--favor-on-conflict', favor_on_conflict, '--outputtool', 'json', '--pkgcomparetool', 'rpmdiff,pkgdiff,abipkgdiff,licensecheck,sonamecheck', '--color=always', '--apply-changes', new_version, ]) config = Config() config.merge(cli) execution_dir, results_dir = Application.setup(config) app = Application(config, os.getcwd(), execution_dir, results_dir) app.run() changes = os.path.join(RESULTS_DIR, CHANGES_PATCH) patch = unidiff.PatchSet.from_filename(changes, encoding='UTF-8') if favor_on_conflict == 'upstream': backported_patch, conflicting_patch, renamed_patch, spec_file = patch assert conflicting_patch.is_removed_file # conflicting.patch elif favor_on_conflict == 'downstream': backported_patch, conflicting_patch, renamed_patch, spec_file = patch assert conflicting_patch.is_modified_file # conflicting.patch else: backported_patch, renamed_patch, spec_file = patch # Non interactive mode - inapplicable patches are only commented out. assert [ h for h in spec_file if '+#Patch1: conflicting.patch\n' in h.target ] assert [h for h in spec_file if '+#%%patch1 -p1\n' in h.target] assert renamed_patch.is_rename # renamed patch 0.1.patch to 0.2.patch assert os.path.basename( renamed_patch.source_file) == 'renamed-0.1.patch' assert os.path.basename( renamed_patch.target_file) == 'renamed-0.2.patch' # Check that the renamed patch path is unchanged assert not [ h for h in spec_file if '-Patch3: renamed-%{version}.patch\n' in h.source ] assert backported_patch.is_removed_file # backported.patch assert spec_file.is_modified_file # test.spec if favor_on_conflict != 'downstream': assert [ h for h in spec_file if '-Patch1: conflicting.patch\n' in h.source ] assert [h for h in spec_file if '-%patch1 -p1\n' in h.source] assert [ h for h in spec_file if '-Patch2: backported.patch\n' in h.source ] assert [h for h in spec_file if '-%patch2 -p1\n' in h.source] assert [ h for h in spec_file if '+- New upstream release {}\n'.format(new_version) in h.target ] with open(os.path.join(RESULTS_DIR, 'report.json'), encoding=ENCODING) as f: report = json.load(f) assert 'success' in report['result'] # patches assert 'applicable.patch' in report['patches']['untouched'] if favor_on_conflict == 'upstream': # In case of conflict, upstream code is favored, therefore conflicting patch is unused. assert 'conflicting.patch' in report['patches']['deleted'] elif favor_on_conflict == 'downstream': assert 'conflicting.patch' in report['patches']['modified'] else: # Non interactive mode - skipping conflicting patches assert 'conflicting.patch' in report['patches']['inapplicable'] assert 'backported.patch' in report['patches']['deleted'] # licensecheck assert report['checkers']['licensecheck']['license_changes'] assert len(report['checkers']['licensecheck'] ['disappeared_licenses']) == 1 assert len(report['checkers']['licensecheck']['new_licenses']) == 1 # rpmdiff assert report['checkers']['rpmdiff']['files_changes']['added'] == 1 assert report['checkers']['rpmdiff']['files_changes'][ 'changed'] == 3 assert report['checkers']['rpmdiff']['files_changes'][ 'removed'] == 1 # abipkgdiff assert report['checkers']['abipkgdiff']['abi_changes'] lib = report['checkers']['abipkgdiff']['packages']['test'][ 'libtest1.so'] if 'Function symbols changes summary' in lib: assert lib['Function symbols changes summary']['Added'][ 'count'] == 1 elif 'Functions changes summary' in lib: assert lib['Functions changes summary']['Added']['count'] == 1 if favor_on_conflict != 'downstream': if 'Variable symbols changes summary' in lib: assert lib['Variable symbols changes summary']['Removed'][ 'count'] == 1 elif 'Variables changes summary' in lib: assert lib['Variables changes summary']['Removed'][ 'count'] == 1 # sonamecheck change = report['checkers']['sonamecheck']['soname_changes'][ 'test']['changed'][0] assert change['from'] == 'libtest2.so.0.1' assert change['to'] == 'libtest2.so.0.2' repo = git.Repo(execution_dir) assert '- New upstream release {}'.format( new_version) in repo.commit().summary
def run(cls): results_dir = None start_dir = os.getcwd() try: LoggerHelper.setup_memory_handler() main_handler, output_tool_handler = LoggerHelper.create_stream_handlers( ) cli = CLI() if hasattr(cli, 'version'): print(VERSION) sys.exit(0) logger.info('rebase-helper version %s', VERSION) config = Config(getattr(cli, 'config-file', None)) config.merge(cli) for handler in [main_handler, output_tool_handler]: handler.set_terminal_background(config.background) if config.verbose == 0: main_handler.setLevel(logging.INFO) elif config.verbose == 1: main_handler.setLevel(CustomLogger.VERBOSE) else: main_handler.setLevel(logging.DEBUG) ConsoleHelper.use_colors = ConsoleHelper.should_use_colors(config) if config.bugzilla_id: repo_path, config.config[ 'sources'] = BugzillaHelper.prepare_rebase_repository( config.bugzilla_id) try: os.chdir(repo_path) except OSError as e: raise RebaseHelperError( 'Could not change directory to the cloned repository' ) from e # update relative paths in config for option in ('results_dir', 'workspace_dir'): path = getattr(config, option) if path and not os.path.isabs(path): config.config[option] = os.path.join( os.path.relpath(start_dir, os.getcwd()), path) if config.config['rpmmacros']: macros = ' -D ' + ' -D '.join( '"{}"'.format(s) for s in config.config['rpmmacros']) for option in ('builder_options', 'srpm_builder_options'): if config.config[option]: config.config[option] += macros else: config.config[option] = macros config.config['rpmmacros'] = cls.convert_macros_to_dict( config.rpmmacros) execution_dir, results_dir = Application.setup(config) app = Application(config, start_dir, execution_dir, results_dir) app.run() except KeyboardInterrupt: logger.info('Interrupted by user') except RebaseHelperError as e: if e.msg: logger.error('%s', e.msg) else: logger.error('%s', str(e)) sys.exit(1) except SystemExit as e: sys.exit(e.code) except BaseException: logger_traceback: CustomLogger = cast( CustomLogger, logging.getLogger('rebasehelper.traceback')) logger_traceback.propagate = False logger_traceback.setLevel(CustomLogger.TRACE) if results_dir: debug_log = os.path.join(results_dir, LOGS_DIR, DEBUG_LOG) traceback_log = os.path.join(results_dir, LOGS_DIR, TRACEBACK_LOG) logger.error( 'rebase-helper failed due to an unexpected error. Please report this problem' '\nusing the following link: %s' '\nand include the content of' '\n\'%s\' and' '\n\'%s\'' '\nin the report.' '\nThank you!', NEW_ISSUE_LINK, debug_log, traceback_log) LoggerHelper.add_file_handler(logger_traceback, traceback_log) else: logger.error( 'rebase-helper failed due to an unexpected error. Please report this problem' '\nusing the following link: %s' '\nand include the following traceback in the report.' '\nThank you!', NEW_ISSUE_LINK) LoggerHelper.add_stream_handler(logger_traceback, CustomLogger.TRACE) logger_traceback.trace('', exc_info=1) sys.exit(1) sys.exit(0)
def test_rebase(self, buildtool, favor_on_conflict): cli = CLI([ '--non-interactive', '--disable-inapplicable-patches', '--buildtool', buildtool, '--favor-on-conflict', favor_on_conflict, '--outputtool', 'json', '--pkgcomparetool', 'rpmdiff,pkgdiff,abipkgdiff,licensecheck', '--color=always', '--apply-changes', self.NEW_VERSION, ]) config = Config() config.merge(cli) execution_dir, results_dir, debug_log_file = Application.setup(config) app = Application(config, execution_dir, results_dir, debug_log_file) app.run() changes = os.path.join(RESULTS_DIR, 'changes.patch') patch = unidiff.PatchSet.from_filename(changes, encoding='UTF-8') if favor_on_conflict == 'upstream': backported_patch, conflicting_patch, spec_file = patch assert conflicting_patch.is_removed_file # conflicting.patch elif favor_on_conflict == 'downstream': backported_patch, conflicting_patch, spec_file = patch assert conflicting_patch.is_modified_file # conflicting.patch else: backported_patch, spec_file = patch # Non interactive mode - inapplicable patches are only commented out. assert '+#Patch1: conflicting.patch\n' in spec_file[0].target assert '+#%%patch1 -p1\n' in spec_file[1].target assert backported_patch.is_removed_file # backported.patch assert spec_file.is_modified_file # test.spec if favor_on_conflict != 'downstream': assert '-Patch1: conflicting.patch\n' in spec_file[ 0].source assert '-%patch1 -p1\n' in spec_file[1].source assert '-Patch2: backported.patch\n' in spec_file[0].source assert '-%patch2 -p1\n' in spec_file[1].source assert '+- New upstream release {}\n'.format( self.NEW_VERSION) in spec_file[2].target with open(os.path.join(RESULTS_DIR, 'report.json')) as f: report = json.load(f) assert 'success' in report['result'] # patches assert 'applicable.patch' in report['patches']['untouched'] if favor_on_conflict == 'upstream': # In case of conflict, upstream code is favored, therefore conflicting patch is unused. assert 'conflicting.patch' in report['patches']['deleted'] elif favor_on_conflict == 'downstream': assert 'conflicting.patch' in report['patches']['modified'] else: # Non interactive mode - skipping conflicting patches assert 'conflicting.patch' in report['patches']['inapplicable'] assert 'backported.patch' in report['patches']['deleted'] # licensecheck assert report['checkers']['licensecheck']['license_changes'] assert len(report['checkers']['licensecheck'] ['disappeared_licenses']) == 1 assert len(report['checkers']['licensecheck']['new_licenses']) == 1 # rpmdiff assert report['checkers']['rpmdiff']['files_changes'][ 'changed'] == 2 # abipkgdiff assert report['checkers']['abipkgdiff']['abi_changes'] lib = report['checkers']['abipkgdiff']['packages']['test'][ 'libtest.so'] assert lib['Functions changes summary']['Added']['count'] == 1 if favor_on_conflict != 'downstream': assert lib['Variables changes summary']['Removed'][ 'count'] == 1 repo = git.Repo(execution_dir) assert '- New upstream release {}'.format( self.NEW_VERSION) in repo.commit().summary