def get_message(message, default_yes=True, any_input=False): """ Function for command line messages :param message: prompt string :param default_yes: If the default value is YES :param any_input: if True, return input without checking it first :return: True or False, based on user's input """ if default_yes: choice = '[Y/n]' else: choice = '[y/N]' if any_input: msg = '{0} '.format(message) else: msg = '{0} {1}? '.format(message, choice) while True: user_input = input(msg).lower() if not user_input or any_input: return True if default_yes else False try: user_input = strtobool(user_input) except ValueError: logger.error('You have to type y(es) or n(o).') continue if any_input: return True else: return bool(user_input)
def run(cls, package_name): version = cls._get_version(package_name) if version: return version logger.error("Failed to determine latest upstream version!\n" "Check that the package exists on %s.", cls.BASE_URL) return None
def _call_rebase_helper(self): """ Clonning repository and call rebase-helper """ logger.info('Clonning repository %s', self.url) try: # git clone http://pkgs.fedoraproject.org/cgit/emacs.git/ git.Git().clone(self.url) except git.exc.GitCommandError as gce: logger.error(gce.message) return os.chdir(self.package) pprint(self.arguments) cli = CLI(self.arguments) try: app = Application(cli) # TDO After a deep testing app.run() will be used #app.run() logger.info(app.kwargs) sources = app.prepare_sources() app.patch_sources(sources) build = app.build_packages() if build: app.pkgdiff_packages() self.patches = app.rebased_patches self._print_patches() logger.info(app.debug_log_file) except RebaseHelperError as rbe: logger.error(rbe.message)
def _get_initial_patches_list(self): """Method returns a list of patches from a spec file""" patches_applied = [] patches_not_used = [] patches_list = [p for p in self.spc.sources if p[2] == 2] patch_flags = self._get_patches_flags() for filename, num, patch_type in patches_list: patch_path = os.path.join(self.sources_location, filename) if not os.path.exists(patch_path): logger.error('Patch %s does not exist', filename) continue patch_num = num if patch_flags: if num in patch_flags: patch_num, patch_option = patch_flags[num] patches_applied.append( PatchObject(patch_path, patch_num, patch_option)) else: patches_not_used.append( PatchObject(patch_path, patch_num, None)) else: patches_applied.append(PatchObject(patch_path, patch_num, None)) patches_applied = sorted(patches_applied, key=lambda x: x.get_index()) return {"applied": patches_applied, "not_applied": patches_not_used}
def _get_initial_patches_list(self): """ Method returns a list of patches from a spec file """ patches_applied = [] patches_not_used = [] patches_list = [p for p in self.spc.sources if p[2] == 2] patch_flags = self._get_patches_flags() for filename, num, patch_type in patches_list: patch_path = os.path.join(self.sources_location, filename) if not os.path.exists(patch_path): logger.error('Patch %s does not exist', filename) continue patch_num = num if patch_flags: if num in patch_flags: patch_num, patch_option = patch_flags[num] patches_applied.append(PatchObject(patch_path, patch_num, patch_option)) else: patches_not_used.append(PatchObject(patch_path, patch_num, None)) else: patches_applied.append(PatchObject(patch_path, patch_num, None)) patches_applied = sorted(patches_applied, key=lambda x: x.get_index()) return {"applied": patches_applied, "not_applied": patches_not_used}
def get_rpm_packages(self, dirname): """ Function returns RPM packages stored in dirname/old and dirname/new directories :param dirname: directory where are stored old and new RPMS :return: """ found = True for version in ['old', 'new']: data = {} data['name'] = self.spec_file.get_package_name() if version == 'old': spec_version = self.spec_file.get_version() else: spec_version = self.rebase_spec_file.get_version() data['version'] = spec_version data['rpm'] = PathHelper.find_all_files( os.path.join(os.path.realpath(dirname), version, 'RPM'), '*.rpm') if not data['rpm']: logger.error( 'Your path %s%s/RPM does not contain any RPM packages', dirname, version) found = False results_store.set_build_data(version, data) if not found: return False return True
def run(cls, package_name): version = cls._get_version(package_name) if version: return version logger.error("Failed to determine latest upstream version!\n" "Check that the package exists on %s.", cls.BASE_URL) return None
def get_message(message, default_yes=True, any_input=False): """ Function for command line messages :param message: prompt string :param default_yes: If the default value is YES :param any_input: if True, return input without checking it first :return: True or False, based on user's input """ if default_yes: choice = '[Y/n]' else: choice = '[y/N]' if any_input: msg = '{0} '.format(message) else: msg = '{0} {1}? '.format(message, choice) while True: user_input = input(msg).lower() if not user_input or any_input: return True if default_yes else False try: user_input = strtobool(user_input) except ValueError: logger.error('You have to type y(es) or n(o).') continue if any_input: return True else: return bool(user_input)
def check_output_argument(output_tool): """ Function checks whether output_tool argument is allowed """ if output_tool not in output_tools.keys(): logger.error('You have to specify one of these printing output tools %s', output_tools.keys()) sys.exit(0)
def run(self): if self.conf.fedpkg_build_tasks: logger.warning("Option --fedpkg-build-tasks is deprecated, use --build-tasks instead.") if not self.conf.build_tasks: self.conf.build_tasks = self.conf.fedpkg_build_tasks if self.conf.build_tasks and not self.conf.builds_nowait: if self.conf.buildtool in ['fedpkg', 'copr']: logger.error("--builds-nowait has to be specified with --build-tasks.") return 1 else: logger.warning("Options are allowed only for fedpkg or copr build tools. Suppress them.") self.conf.build_tasks = self.conf.builds_nowait = False sources = None if self.conf.build_tasks is None: sources = self.prepare_sources() if not self.conf.build_only and not self.conf.comparepkgs: self.patch_sources(sources) build = False if not self.conf.patch_only: if not self.conf.comparepkgs: # check build dependencies for rpmbuild if self.conf.buildtool == 'rpmbuild': Application.check_build_requires(self.spec_file) # Build packages try: build = self.build_packages() if self.conf.builds_nowait and not self.conf.build_tasks: if self.conf.buildtool == 'fedpkg': self.print_koji_logs() elif self.conf.buildtool == 'copr': self.print_copr_logs() return 0 except RuntimeError: logger.error('Unknown error caused by build log analysis') return 1 # Perform checks else: build = self.get_rpm_packages(self.conf.comparepkgs) # We don't care dirname doesn't contain any RPM packages # Therefore return 1 if build: self.pkgdiff_packages(self.results_dir) else: if not self.upstream_monitoring: logger.info('Rebase package to %s FAILED. See for more details', self.conf.sources) return 1 self.print_summary() if not self.conf.keep_workspace: self._delete_workspace_dir() if self.debug_log_file: logger.info("Detailed debug log is located in '%s'", self.debug_log_file) if not self.upstream_monitoring and not self.conf.patch_only: logger.info('Rebase package to %s was SUCCESSFUL.\n', self.conf.sources) return 0
def _parse_build_log(cls, log_path, nvr): """Parses a build log. Args: log_path (str): Path to the RPM build log. Returns: tuple: The first element is the type of error that was found in the log (missing or deleted). The second element is a list of problematic files. """ try: with open(log_path, 'r') as build_log: lines = build_log.read().splitlines() except IOError: logger.error('There was an error opening %s', log_path) return None, None error_type = None # Use set to avoid duplicate files. files = set() # Two types of error can occur. The file is either in the SPEC file but missing in sources # or the other way around. Example lines of the first case: # File not found: /root/rpmbuild/BUILDROOT/pello-0.1.1-1.fc27.x86_64/should/not/be/here # File not found: /root/rpmbuild/BUILDROOT/pello-0.1.1-1.fc27.x86_64/should/not/be/here2 # Example lines of the second case: # Installed (but unpackaged) file(s) found: # /usr/bin/pello error_re = re.compile( r''' ^ (BUILDSTDERR:)? \s* ( (?P<missing>File\s+not\s+found:\s*)| (?P<unpackaged>Installed\s+\(but\s+unpackaged\)\s+file\(s\)\s+found:) )? (/.*/{}\.\w+)? (?P<path>/.*)? $ '''.format(re.escape(nvr)), re.VERBOSE ) for line in lines: match = error_re.match(line) if match: if match.group('missing'): error_type = 'deleted' files.add(match.group('path')) elif match.group('unpackaged'): error_type = 'missing' elif error_type == 'missing' and match.group('path'): files.add(match.group('path')) elif error_type and not match.group('path'): break return error_type, list(files)
def run_check(cls, result_dir): """Compares old and new RPMs using pkgdiff""" debug_old, rest_pkgs_old = cls._get_packages_for_abipkgdiff(OutputLogger.get_build('old')) debug_new, rest_pkgs_new = cls._get_packages_for_abipkgdiff(OutputLogger.get_build('new')) cmd = [cls.CMD] if debug_old is None: logger.warning("Package doesn't contain any debug package") return None try: cmd.append('--d1') cmd.append(debug_old[0]) except IndexError: logger.error('Debuginfo package not found for old package.') return None try: cmd.append('--d2') cmd.append(debug_new[0]) except IndexError: logger.error('Debuginfo package not found for new package.') return None reports = {} for pkg in rest_pkgs_old: command = list(cmd) # Package can be <letters><numbers>-<letters>-<and_whatever> regexp = r'^(\w*)(-\D+)?.*$' reg = re.compile(regexp) matched = reg.search(os.path.basename(pkg)) if matched: file_name = matched.group(1) command.append(pkg) find = [x for x in rest_pkgs_new if os.path.basename(x).startswith(file_name)] command.append(find[0]) package_name = os.path.basename(os.path.basename(pkg)) logger.debug('Package name for ABI comparision %s', package_name) regexp_name = r'(\w-)*(\D+)*' reg_name = re.compile(regexp_name) matched = reg_name.search(os.path.basename(pkg)) logger.debug('Found matches %s', matched.groups()) if matched: package_name = matched.group(0) + cls.log_name else: package_name = package_name + '-' + cls.log_name output = os.path.join(cls.results_dir, result_dir, package_name) try: ret_code = ProcessHelper.run_subprocess(command, output=output) except OSError: raise CheckerNotFoundError("Checker '%s' was not found or installed." % cls.CMD) if int(ret_code) & settings.ABIDIFF_ERROR and int(ret_code) & settings.ABIDIFF_USAGE_ERROR: raise RebaseHelperError('Execution of %s failed.\nCommand line is: %s' % (cls.CMD, cmd)) if int(ret_code) == 0: text = 'ABI of the compared binaries in package %s are equal.' % package_name else: text = 'ABI of the compared binaries in package %s are not equal.' % package_name reports[output] = text else: logger.debug("Rebase-helper did not find a package name in '%s'", package_name) return reports
def run_check(cls, result_dir): """Compares old and new RPMs using pkgdiff""" debug_old, rest_pkgs_old = cls._get_packages_for_abipkgdiff(results_store.get_build('old')) debug_new, rest_pkgs_new = cls._get_packages_for_abipkgdiff(results_store.get_build('new')) cmd = [cls.CMD] if debug_old is None: logger.warning("Package doesn't contain any debug package") return None try: cmd.append('--d1') cmd.append(debug_old[0]) except IndexError: logger.error('Debuginfo package not found for old package.') return None try: cmd.append('--d2') cmd.append(debug_new[0]) except IndexError: logger.error('Debuginfo package not found for new package.') return None reports = {} for pkg in rest_pkgs_old: command = list(cmd) # Package can be <letters><numbers>-<letters>-<and_whatever> regexp = r'^(\w*)(-\D+)?.*$' reg = re.compile(regexp) matched = reg.search(os.path.basename(pkg)) if matched: file_name = matched.group(1) command.append(pkg) find = [x for x in rest_pkgs_new if os.path.basename(x).startswith(file_name)] command.append(find[0]) package_name = os.path.basename(os.path.basename(pkg)) logger.debug('Package name for ABI comparision %s', package_name) regexp_name = r'(\w-)*(\D+)*' reg_name = re.compile(regexp_name) matched = reg_name.search(os.path.basename(pkg)) logger.debug('Found matches %s', matched.groups()) if matched: package_name = matched.group(0) + cls.log_name else: package_name = package_name + '-' + cls.log_name output = os.path.join(cls.results_dir, result_dir, package_name) try: ret_code = ProcessHelper.run_subprocess(command, output=output) except OSError: raise CheckerNotFoundError("Checker '%s' was not found or installed." % cls.CMD) if int(ret_code) & settings.ABIDIFF_ERROR and int(ret_code) & settings.ABIDIFF_USAGE_ERROR: raise RebaseHelperError('Execution of %s failed.\nCommand line is: %s' % (cls.CMD, cmd)) if int(ret_code) == 0: text = 'ABI of the compared binaries in package %s are equal.' % package_name else: text = 'ABI of the compared binaries in package %s are not equal.' % package_name reports[output] = text else: logger.debug("Rebase-helper did not find a package name in '%s'", package_name) return reports
def _parse_build_log(cls, log_path, nvr): """Parses a build log. Args: log_path (str): Path to the RPM build log. Returns: tuple: The first element is the type of error that was found in the log (missing or deleted). The second element is a list of problematic files. """ try: with open(log_path, 'r') as build_log: lines = build_log.read().splitlines() except IOError: logger.error('There was an error opening %s', log_path) return None, None error_type = None # Use set to avoid duplicate files. files = set() # Two types of error can occur. The file is either in the SPEC file but missing in sources # or the other way around. Example lines of the first case: # File not found: /root/rpmbuild/BUILDROOT/pello-0.1.1-1.fc27.x86_64/should/not/be/here # File not found: /root/rpmbuild/BUILDROOT/pello-0.1.1-1.fc27.x86_64/should/not/be/here2 # Example lines of the second case: # Installed (but unpackaged) file(s) found: # /usr/bin/pello error_re = re.compile( r''' ^ (BUILDSTDERR:)? \s* ( (?P<missing>File\s+not\s+found:\s*)| (?P<unpackaged>Installed\s+\(but\s+unpackaged\)\s+file\(s\)\s+found:) )? (/.*/{}\.\w+)? (?P<path>/.*)? $ '''.format(re.escape(nvr)), re.VERBOSE) for line in lines: match = error_re.match(line) if match: if match.group('missing'): error_type = 'deleted' files.add(match.group('path')) elif match.group('unpackaged'): error_type = 'missing' elif error_type == 'missing' and match.group('path'): files.add(match.group('path')) elif error_type and not match.group('path'): break return error_type, list(files)
def build_source_packages(self): try: builder = plugin_manager.srpm_build_tools.get_plugin(self.conf.srpm_buildtool) except NotImplementedError as e: raise RebaseHelperError('{}. Supported SRPM build tools are {}'.format( str(e), plugin_manager.srpm_build_tools.get_supported_tools())) for version in ['old', 'new']: koji_build_id = None results_dir = '{}-build'.format(os.path.join(self.results_dir, version)) spec = self.spec_file if version == 'old' else self.rebase_spec_file package_name = spec.get_package_name() package_version = spec.get_version() package_full_version = spec.get_full_version() logger.info('Building source package for %s version %s', package_name, package_full_version) if version == 'old' and self.conf.get_old_build_from_koji: koji_build_id, package_version, package_full_version = KojiHelper.get_old_build_info(package_name, package_version) build_dict = dict( name=package_name, version=package_version, srpm_buildtool=self.conf.srpm_buildtool, srpm_builder_options=self.conf.srpm_builder_options) try: if koji_build_id: session = KojiHelper.create_session() build_dict['srpm'], build_dict['logs'] = KojiHelper.download_build(session, koji_build_id, os.path.join( results_dir, 'SRPM' ), arches=['src']) else: build_dict.update(builder.build(spec, results_dir, **build_dict)) build_dict = self._sanitize_build_dict(build_dict) results_store.set_build_data(version, build_dict) except RebaseHelperError: # pylint: disable=try-except-raise raise except SourcePackageBuildError as e: build_dict.update(builder.get_logs()) build_dict['source_package_build_error'] = str(e) build_dict = self._sanitize_build_dict(build_dict) results_store.set_build_data(version, build_dict) if e.logfile: msg = 'Building {} SRPM packages failed; see {} for more information'.format(version, e.logfile) else: msg = 'Building {} SRPM packages failed; see logs in {} for more information'.format( version, os.path.join(results_dir, 'SRPM')) logger.error(msg) raise RebaseHelperError(msg, logfiles=builder.get_logs().get('logs')) except Exception: raise RebaseHelperError('Building package failed with unknown reason. ' 'Check all available log files.')
def run(): try: cli = CLI(sys.argv[1:]) app = Application(cli) app.run() except KeyboardInterrupt: logger.info('\nInterrupted by user') except RebaseHelperError as e: logger.error('\n%s', exc_as_decode_string(e)) sys.exit(1) sys.exit(0)
def _call_rebase_helper(self): """ Clonning repository and call rebase-helper """ logger.info('Clonning repository %s', self.url) git.Git().clone(self.url) os.chdir(self.package) cli = CLI(self.arguments) try: app = Application(cli) app.run() self.patches = app.rebased_patches self._print_patches() logger.info(app.debug_log_file) except RebaseHelperError as rbe: logger.error(rbe.message)
def request(url, **kwargs): """Performs an HTTP request or an FTP RETR command. Args: url (str): HTTP, HTTPS or FTP URL. **kwargs: Keyword arguments to be passed to requests.session.get(). Returns: requests.Response: Response object. """ class FTPAdapter(requests.adapters.BaseAdapter): def send( self, request, stream=False, timeout=None, verify=True, # pylint: disable=unused-argument cert=None, proxies=None): # pylint: disable=unused-argument response = requests.models.Response() response.request = request response.connection = self try: resp = urllib.request.urlopen(request.url) except urllib.error.URLError as e: response.status_code = 400 response.reason = e.reason else: response.status_code = 200 response.headers = requests.structures.CaseInsensitiveDict( getattr(resp, 'headers', {})) response.raw = resp response.url = resp.url return response def close(self): pass session = requests.Session() session.mount('ftp://', FTPAdapter()) try: return session.get(url, **kwargs) except requests.exceptions.RequestException as e: logger.error('%s: %s', type(e).__name__, six.text_type(e)) return None
def run(): debug_log_file = None try: cli = CLI() if hasattr(cli, 'version'): logger.info(VERSION) sys.exit(0) config = Config(getattr(cli, 'config-file', None)) config.merge(cli) for handler in [main_handler, output_tool_handler]: handler.set_terminal_background(config.background) ConsoleHelper.use_colors = ConsoleHelper.should_use_colors(config) execution_dir, results_dir, debug_log_file = Application.setup( config) traceback_log = os.path.join(results_dir, LOGS_DIR, TRACEBACK_LOG) if config.verbose == 0: main_handler.setLevel(logging.INFO) elif config.verbose == 1: main_handler.setLevel(CustomLogger.VERBOSE) else: main_handler.setLevel(logging.DEBUG) app = Application(config, execution_dir, results_dir, debug_log_file) app.run() except KeyboardInterrupt: logger.info('Interrupted by user') except RebaseHelperError as e: if e.msg: logger.error('%s', e.msg) else: logger.error('%s', six.text_type(e)) sys.exit(1) except SystemExit as e: sys.exit(e.code) except BaseException: if debug_log_file: logger.error( 'rebase-helper failed due to an unexpected error. Please report this problem' '\nusing the following link: %s' '\nand include the content of' '\n\'%s\' and' '\n\'%s\'' '\nin the report.' '\nThank you!', NEW_ISSUE_LINK, debug_log_file, traceback_log) LoggerHelper.add_file_handler(logger_traceback, traceback_log) logger_traceback.trace('', exc_info=1) else: logger.error( 'rebase-helper failed due to an unexpected error. Please report this problem' '\nusing the following link: %s' '\nThank you!', NEW_ISSUE_LINK) sys.exit(1) sys.exit(0)
def run_package_checkers(self, results_dir): """ Runs checkers on packages and stores results in a given directory. :param results_dir: Path to directory in which to store the results. :type results_dir: str :return: None """ results = dict() for checker_name in self.conf.pkgcomparetool: try: results[checker_name] = checkers_runner.run_checker(results_dir, checker_name) except CheckerNotFoundError: logger.error("Rebase-helper did not find checker '%s'." % checker_name) for diff_name, result in six.iteritems(results): results_store.set_checker_output(diff_name, result)
def run(): debug_log_file = None try: cli = CLI() if hasattr(cli, 'version'): logger.info(VERSION) sys.exit(0) config = Config(getattr(cli, 'config-file', None)) config.merge(cli) for handler in [main_handler, output_tool_handler]: handler.set_terminal_background(config.background) ConsoleHelper.use_colors = ConsoleHelper.should_use_colors(config) execution_dir, results_dir, debug_log_file = Application.setup(config) traceback_log = os.path.join(results_dir, LOGS_DIR, TRACEBACK_LOG) if config.verbose == 0: main_handler.setLevel(logging.INFO) elif config.verbose == 1: main_handler.setLevel(CustomLogger.VERBOSE) else: main_handler.setLevel(logging.DEBUG) app = Application(config, execution_dir, results_dir, debug_log_file) app.run() except KeyboardInterrupt: logger.info('Interrupted by user') except RebaseHelperError as e: if e.msg: logger.error('%s', e.msg) else: logger.error('%s', str(e)) sys.exit(1) except SystemExit as e: sys.exit(e.code) except BaseException: if debug_log_file: logger.error('rebase-helper failed due to an unexpected error. Please report this problem' '\nusing the following link: %s' '\nand include the content of' '\n\'%s\' and' '\n\'%s\'' '\nin the report.' '\nThank you!', NEW_ISSUE_LINK, debug_log_file, traceback_log) LoggerHelper.add_file_handler(logger_traceback, traceback_log) logger_traceback.trace('', exc_info=1) else: logger.error('rebase-helper failed due to an unexpected error. Please report this problem' '\nusing the following link: %s' '\nThank you!', NEW_ISSUE_LINK) sys.exit(1) sys.exit(0)
def request(url, **kwargs): """Performs an HTTP request or an FTP RETR command. Args: url (str): HTTP, HTTPS or FTP URL. **kwargs: Keyword arguments to be passed to requests.session.get(). Returns: requests.Response: Response object. """ class FTPAdapter(requests.adapters.BaseAdapter): def send(self, request, stream=False, timeout=None, verify=True, # pylint: disable=unused-argument cert=None, proxies=None): # pylint: disable=unused-argument response = requests.models.Response() response.request = request response.connection = self try: resp = urllib.request.urlopen(request.url) except urllib.error.URLError as e: response.status_code = 400 response.reason = e.reason else: response.status_code = 200 response.headers = requests.structures.CaseInsensitiveDict(getattr(resp, 'headers', {})) response.raw = resp response.url = resp.url return response def close(self): pass session = requests.Session() session.mount('ftp://', FTPAdapter()) try: return session.get(url, **kwargs) except requests.exceptions.RequestException as e: logger.error('%s: %s', type(e).__name__, str(e)) return None
def run_package_checkers(self, results_dir): """ Runs checkers on packages and stores results in a given directory. :param results_dir: Path to directory in which to store the results. :type results_dir: str :return: None """ results = dict() for checker_name in self.conf.pkgcomparetool: try: results[checker_name] = checkers_runner.run_checker( os.path.join(results_dir, 'checkers'), checker_name) except CheckerNotFoundError: logger.error("Rebase-helper did not find checker '%s'." % checker_name) for diff_name, result in six.iteritems(results): results_store.set_checker_output(diff_name, result)
def run(): debug_log_file = None try: # be verbose until debug_log_file is created cli = CLI() if hasattr(cli, 'version'): logger.info(VERSION) sys.exit(0) config = Config(getattr(cli, 'config-file', None)) config.merge(cli) for handler in [main_handler, output_tool_handler]: handler.set_terminal_background(config.background) ConsoleHelper.use_colors = ConsoleHelper.should_use_colors(config) execution_dir, results_dir, debug_log_file = Application.setup( config) if not config.verbose: main_handler.setLevel(logging.INFO) app = Application(config, execution_dir, results_dir, debug_log_file) app.run() except KeyboardInterrupt: logger.info('Interrupted by user') except RebaseHelperError as e: if e.msg: logger.error('%s', e.msg) else: logger.error('%s', six.text_type(e)) sys.exit(1) except SystemExit as e: sys.exit(e.code) except BaseException: if debug_log_file: logger.error( 'rebase-helper failed due to an unexpected error. Please report this problem' '\nusing the following link: %s' '\nand include the content of' '\n\'%s\'' '\nfile in the report.' '\nThank you!', NEW_ISSUE_LINK, debug_log_file) else: logger.error( 'rebase-helper failed due to an unexpected error. Please report this problem' '\nusing the following link: %s' '\nand include the traceback following this message in the report.' '\nThank you!', NEW_ISSUE_LINK) logger.trace('', exc_info=1) sys.exit(1) sys.exit(0)
def run(): debug_log_file = None try: # be verbose until debug_log_file is created handler = LoggerHelper.add_stream_handler(logger, logging.DEBUG) if "--builder-options" in sys.argv[1:]: raise RebaseHelperError( "Wrong format of --builder-options. It must be in the following form:" ' --builder-options="--desired-builder-option".' ) cli = CLI() execution_dir, results_dir, debug_log_file, report_log_file = Application.setup(cli) if not cli.verbose: handler.setLevel(logging.INFO) app = Application(cli, execution_dir, results_dir, debug_log_file, report_log_file) app.run() except KeyboardInterrupt: logger.info("\nInterrupted by user") except RebaseHelperError as e: if e.args: logger.error("\n%s", e.args[0] % e.args[1:]) else: logger.error("\n%s", six.text_type(e)) sys.exit(1) except SystemExit as e: sys.exit(e.code) except BaseException: if debug_log_file: logger.error( "\nrebase-helper failed due to an unexpected error. Please report this problem" "\nusing the following link: %s" "\nand include the content of" "\n'%s'" "\nfile in the report." "\nThank you!", NEW_ISSUE_LINK, debug_log_file, ) else: logger.error( "\nrebase-helper failed due to an unexpected error. Please report this problem" "\nusing the following link: %s" "\nand include the traceback following this message in the report." "\nThank you!", NEW_ISSUE_LINK, ) logger.debug("\n", exc_info=1) sys.exit(1) sys.exit(0)
def run_package_checkers(self, results_dir): """ Runs checkers on packages and stores results in a given directory. :param results_dir: Path to directory in which to store the results. :type results_dir: str :return: None """ results = dict() if self.conf.pkgcomparetool: results[self.conf.pkgcomparetool] = checkers_runner.run_checker(results_dir, self.conf.pkgcomparetool) else: # no specific checker was given, just run all of them for checker_name in checkers_runner.get_supported_tools(): try: results[checker_name] = checkers_runner.run_checker(results_dir, checker_name) except CheckerNotFoundError: logger.error("Rebase-helper did not find checker '%s'." % checker_name) for diff_name, result in six.iteritems(results): OutputLogger.set_checker_output(diff_name, result)
def get_message(message, default_yes=True, any_input=False): """Prompts a user with yes/no message and gets the response. Args: message (str): Prompt string. default_yes (bool): If the default value should be YES. any_input (bool): Whether to return default value regardless of input. Returns: bool: True or False, based on user's input. """ if default_yes: choice = '[Y/n]' else: choice = '[y/N]' if any_input: msg = '{0} '.format(message) else: msg = '{0} {1}? '.format(message, choice) while True: user_input = input(msg).lower() if not user_input or any_input: return True if default_yes else False try: user_input = strtobool(user_input) except ValueError: logger.error('You have to type y(es) or n(o).') continue if any_input: return True else: return bool(user_input)
def run(self): sources = self.prepare_sources() if not self.conf.build_only and not self.conf.comparepkgs: self.patch_sources(sources) if not self.conf.patch_only: if not self.conf.comparepkgs: # check build dependencies for rpmbuild if self.conf.buildtool == 'rpmbuild': Application.check_build_requires(self.spec_file) # Build packages try: build = self.build_packages() except RuntimeError: logger.error('Not know error caused by build log analysis') return 1 # Perform checks else: build = self.get_rpm_packages(self.conf.comparepkgs) # We don't care dirname doesn't contain any RPM packages # Therefore return 1 if build: self.pkgdiff_packages() else: if not self.upstream_monitoring: logger.info('Rebase package to %s FAILED. See for more details' % self.conf.sources) return 1 self.print_summary() if not self.conf.keep_workspace: self._delete_workspace_dir() if self.debug_log_file: logger.info("Detailed debug log is located in '%s'", self.debug_log_file) if not self.upstream_monitoring and not self.conf.patch_only: logger.info('Rebase package to %s was SUCCESSFUL.\n' % self.conf.sources) return 0
def get_message(message, default_yes=True, any_input=False): """Prompts a user with yes/no message and gets the response. Args: message (str): Prompt string. default_yes (bool): If the default value should be YES. any_input (bool): Whether to return default value regardless of input. Returns: bool: True or False, based on user's input. """ if default_yes: choice = '[Y/n]' else: choice = '[y/N]' if any_input: msg = '{0} '.format(message) else: msg = '{0} {1}? '.format(message, choice) while True: user_input = input(msg).lower() if not user_input or any_input: return True if default_yes else False try: user_input = strtobool(user_input) except ValueError: logger.error('You have to type y(es) or n(o).') continue if any_input: return True else: return bool(user_input)
def run(): debug_log_file = None try: # be verbose until debug_log_file is created handler = LoggerHelper.add_stream_handler(logger, logging.DEBUG) if "--builder-options" in sys.argv[1:]: raise RebaseHelperError( 'Wrong format of --builder-options. It must be in the following form:' ' --builder-options="--desired-builder-option".') cli = CLI() if cli.version: logger.info(VERSION) sys.exit(0) ConsoleHelper.use_colors = ConsoleHelper.should_use_colors(cli) execution_dir, results_dir, debug_log_file = Application.setup(cli) if not cli.verbose: handler.setLevel(logging.INFO) app = Application(cli, execution_dir, results_dir, debug_log_file) app.run() except KeyboardInterrupt: logger.info('\nInterrupted by user') except RebaseHelperError as e: if e.msg: logger.error('\n%s', e.msg) else: logger.error('\n%s', six.text_type(e)) sys.exit(1) except SystemExit as e: sys.exit(e.code) except BaseException: if debug_log_file: logger.error( '\nrebase-helper failed due to an unexpected error. Please report this problem' '\nusing the following link: %s' '\nand include the content of' '\n\'%s\'' '\nfile in the report.' '\nThank you!', NEW_ISSUE_LINK, debug_log_file) else: logger.error( '\nrebase-helper failed due to an unexpected error. Please report this problem' '\nusing the following link: %s' '\nand include the traceback following this message in the report.' '\nThank you!', NEW_ISSUE_LINK) logger.debug('\n', exc_info=1) sys.exit(1) sys.exit(0)
def get_rpm_packages(self, dirname): """ Function returns RPM packages stored in dirname/old and dirname/new directories :param dirname: directory where are stored old and new RPMS :return: """ found = True for version in ['old', 'new']: data = {} data['name'] = self.spec_file.get_package_name() if version == 'old': spec_version = self.spec_file.get_version() else: spec_version = self.rebase_spec_file.get_version() data['version'] = spec_version data['rpm'] = PathHelper.find_all_files(os.path.join(os.path.realpath(dirname), version, 'RPM'), '*.rpm') if not data['rpm']: logger.error('Your path %s%s/RPM does not contain any RPM packages' % (dirname, version)) found = False OutputLogger.set_build_data(version, data) if not found: return False return True
def run_package_checkers(self, results_dir, **kwargs): """ Runs checkers on packages and stores results in a given directory. :param results_dir: Path to directory in which to store the results. :type results_dir: str :param category: checker type(SOURCE/SRPM/RPM) :type category: str :return: None """ results = dict() for checker_name in self.conf.pkgcomparetool: try: data = plugin_manager.checkers.run(os.path.join(results_dir, 'checkers'), checker_name, **kwargs) if data: results[checker_name] = data except CheckerNotFoundError: logger.error("Rebase-helper did not find checker '%s'.", checker_name) for diff_name, result in results.items(): results_store.set_checker_output(diff_name, result)
def build_packages(self): """Function calls build class for building packages""" if self.conf.buildtool == 'fedpkg' and not koji_builder: print ('Importing module koji failed. Switching to mockbuild.') self.conf.buildtool = 'mock' try: builder = Builder(self.conf.buildtool) except NotImplementedError as ni_e: raise RebaseHelperError('%s. Supported build tools are %s' % six.text_type(ni_e), Builder.get_supported_tools()) for version in ['old', 'new']: spec_object = self.spec_file if version == 'old' else self.rebase_spec_file build_dict = {} task_id = None if self.conf.build_tasks is None: build_dict['name'] = spec_object.get_package_name() build_dict['version'] = spec_object.get_version() patches = [x.get_path() for x in spec_object.get_patches()] spec = spec_object.get_path() sources = spec_object.get_sources() logger.info('Building packages for %s version %s', spec_object.get_package_name(), spec_object.get_version()) else: if version == 'old': task_id = self.conf.build_tasks.split(',')[0] else: task_id = self.conf.build_tasks.split(',')[1] results_dir = os.path.join(self.results_dir, version) build_dict['builds_nowait'] = self.conf.builds_nowait build_dict['build_tasks'] = self.conf.build_tasks files = {} number_retries = 0 while self.conf.build_retries != number_retries: try: if self.conf.build_tasks is None: build_dict.update(builder.build(spec, sources, patches, results_dir, **build_dict)) if not self.conf.builds_nowait: if self.conf.buildtool == 'fedpkg': while True: kh = KojiHelper() build_dict['rpm'], build_dict['logs'] = kh.get_koji_tasks(build_dict['koji_task_id'], results_dir) if build_dict['rpm']: break else: if self.conf.build_tasks: if self.conf.buildtool == 'fedpkg': kh = KojiHelper() try: build_dict['rpm'], build_dict['logs'] = kh.get_koji_tasks(task_id, results_dir) OutputLogger.set_build_data(version, build_dict) if not build_dict['rpm']: return False except TypeError: logger.info('Koji tasks are not finished yet. Try again later') return False elif self.conf.buildtool == 'copr': copr_helper = CoprHelper() client = copr_helper.get_client() build_id = int(task_id) status = copr_helper.get_build_status(client, build_id) if status in ['importing', 'pending', 'starting', 'running']: logger.info('Copr build is not finished yet. Try again later') return False else: build_dict['rpm'], build_dict['logs'] = copr_helper.download_build(client, build_id, results_dir) if status not in ['succeeded', 'skipped']: logger.info('Copr build {} did not complete successfully'.format(build_id)) return False # Build finishes properly. Go out from while cycle OutputLogger.set_build_data(version, build_dict) break except SourcePackageBuildError: # always fail for original version if version == 'old': raise RebaseHelperError('Creating old SRPM package failed.') logger.error('Building source package failed.') # TODO: implement log analyzer for SRPMs and add the checks here!!! raise except BinaryPackageBuildError: # always fail for original version rpm_dir = os.path.join(results_dir, 'RPM') build_dict.update(builder.get_logs()) OutputLogger.set_build_data(version, build_dict) build_log = 'build.log' build_log_path = os.path.join(rpm_dir, build_log) if version == 'old': raise RebaseHelperError('Building old RPM package failed. Check log %s', build_log_path) logger.error('Building binary packages failed.') msg = 'Building package failed' try: files = BuildLogAnalyzer.parse_log(rpm_dir, build_log) except BuildLogAnalyzerMissingError: raise RebaseHelperError('Build log %s does not exist', build_log_path) except BuildLogAnalyzerMakeError: raise RebaseHelperError('%s during build. Check log %s', msg, build_log_path) except BuildLogAnalyzerPatchError: raise RebaseHelperError('%s during patching. Check log %s', msg, build_log_path) except RuntimeError: if self.conf.build_retries == number_retries: raise RebaseHelperError('%s with unknown reason. Check log %s', msg, build_log_path) if 'missing' in files: missing_files = '\n'.join(files['missing']) logger.info('Files not packaged in the SPEC file:\n%s', missing_files) elif 'deleted' in files: deleted_files = '\n'.join(files['deleted']) logger.warning('Removed files packaged in SPEC file:\n%s', deleted_files) else: if self.conf.build_retries == number_retries: raise RebaseHelperError("Build failed, but no issues were found in the build log %s", build_log) self.rebase_spec_file.modify_spec_files_section(files) if not self.conf.non_interactive: msg = 'Do you want rebase-helper to try build the packages one more time' if not ConsoleHelper.get_message(msg): raise KeyboardInterrupt else: logger.warning('Some patches were not successfully applied') # build just failed, otherwise we would break out of the while loop logger.debug('Number of retries is %s', self.conf.build_retries) if os.path.exists(os.path.join(results_dir, 'RPM')): shutil.rmtree(os.path.join(results_dir, 'RPM')) if os.path.exists(os.path.join(results_dir, 'SRPM')): shutil.rmtree(os.path.join(results_dir, 'SRPM')) number_retries += 1 if self.conf.build_retries == number_retries: raise RebaseHelperError('Building package failed with unknow reason. Check all available log files.') return True
def build_packages(self): """ Function calls build class for building packages """ try: builder = Builder(self.conf.buildtool) except NotImplementedError as ni_e: raise RebaseHelperError('%s. Supported build tools are %s', ni_e.message, Builder.get_supported_tools()) for version in ['old', 'new']: spec_object = self.spec_file if version == 'old' else self.rebase_spec_file build_dict = {} build_dict['name'] = spec_object.get_package_name() build_dict['version'] = spec_object.get_version() logger.debug(build_dict) patches = [x.get_path() for x in spec_object.get_patches()] results_dir = os.path.join(self.results_dir, version) spec = spec_object.get_path() sources = spec_object.get_sources() failed_before = False while True: try: build_dict.update(builder.build(spec, sources, patches, results_dir, **build_dict)) OutputLogger.set_build_data(version, build_dict) break except SourcePackageBuildError: # always fail for original version if version == 'old': raise RebaseHelperError('Creating old SRPM package failed.') logger.error('Building source package failed.') # TODO: implement log analyzer for SRPMs and add the checks here!!! raise except BinaryPackageBuildError: # always fail for original version rpm_dir = os.path.join(results_dir, 'RPM') build_log = 'build.log' build_log_path = os.path.join(rpm_dir, build_log) if version == 'old': raise RebaseHelperError('Building old RPM package failed. Check log %s', build_log_path) logger.error('Building binary packages failed.') try: files = BuildLogAnalyzer.parse_log(rpm_dir, build_log) except BuildLogAnalyzerMissingError: raise RebaseHelperError('Build log %s does not exist', build_log_path) except BuildLogAnalyzerMakeError: raise RebaseHelperError('Building package failed during build. Check log %s', build_log_path) except BuildLogAnalyzerPatchError: raise RebaseHelperError('Building package failed during patching. Check log %s' % build_log_path) if files['missing']: missing_files = '\n'.join(files['added']) logger.info('Files not packaged in the SPEC file:\n%s', missing_files) elif files['deleted']: deleted_files = '\n'.join(files['deleted']) logger.warning('Removed files packaged in SPEC file:\n%s', deleted_files) else: raise RebaseHelperError("Build failed, but no issues were found in the build log %s", build_log) self.rebase_spec_file.modify_spec_files_section(files) if not self.conf.non_interactive: if failed_before: if not ConsoleHelper.get_message('Do you want rebase-helper to try build the packages one more time'): raise KeyboardInterrupt else: logger.warning('Some patches were not successfully applied') shutil.rmtree(os.path.join(results_dir, 'RPM')) shutil.rmtree(os.path.join(results_dir, 'SRPM')) return False # build just failed, otherwise we would break out of the while loop failed_before = True shutil.rmtree(os.path.join(results_dir, 'RPM')) shutil.rmtree(os.path.join(results_dir, 'SRPM')) return True
def watch_koji_tasks(cls, session, tasklist): """Function is copy/paste from pyrpkg/cli.py""" if not tasklist: return # Place holder for return value rh_tasks = {} try: tasks = {} for task_id in tasklist: tasks[task_id] = TaskWatcher(task_id, session, logger, quiet=False) while True: all_done = True for task_id, task in tasks.items(): changed = task.update() info = session.getTaskInfo(task_id) state = task.info['state'] if state == koji.TASK_STATES['FAILED']: return {info['id']: state} else: if info['arch'] == 'x86_64' or info['arch'] == 'noarch': rh_tasks[info['id']] = state if not task.is_done(): all_done = False else: if changed: cls.display_task_results(tasks) if not task.is_success(): rh_tasks = None try: for child in session.getTaskChildren(task_id): child_id = child['id'] if child_id not in tasks.keys(): tasks[child_id] = TaskWatcher(child_id, session, logger, task.level + 1, quiet=False) tasks[child_id].update() # If we found new children, go through the list # again, in case they have children also info = session.getTaskInfo(child_id) state = task.info['state'] if state == koji.TASK_STATES['FAILED']: return {info['id']: state} else: if info['arch'] == 'x86_64' or info[ 'arch'] == 'noarch': rh_tasks[info['id']] = state all_done = False except SSL.SysCallError as e: logger.error('Detected SSL error: %s', six.text_type(e)) if all_done: cls.display_task_results(tasks) break time.sleep(1) except (KeyboardInterrupt): # A ^c should return non-zero so that it doesn't continue # on to any && commands. rh_tasks = None return rh_tasks
def build_packages(self): """Function calls build class for building packages""" if self.conf.buildtool == KojiBuildTool.CMD and not koji_builder: logger.info( 'Importing module koji failed. Switching to mock builder.') self.conf.buildtool = MockBuildTool.CMD try: builder = Builder(self.conf.buildtool) except NotImplementedError as ni_e: raise RebaseHelperError( '%s. Supported build tools are %s' % six.text_type(ni_e), Builder.get_supported_tools()) for version in ['old', 'new']: spec_object = self.spec_file if version == 'old' else self.rebase_spec_file build_dict = {} task_id = None if self.conf.build_tasks is None: build_dict['name'] = spec_object.get_package_name() build_dict['version'] = spec_object.get_version() patches = [x.get_path() for x in spec_object.get_patches()] spec = spec_object.get_path() sources = spec_object.get_sources() logger.info('Building packages for %s version %s', spec_object.get_package_name(), spec_object.get_full_version()) else: if version == 'old': task_id = self.conf.build_tasks[0] else: task_id = self.conf.build_tasks[1] results_dir = os.path.join(self.results_dir, version) build_dict['builds_nowait'] = self.conf.builds_nowait build_dict['build_tasks'] = self.conf.build_tasks build_dict['builder_options'] = self.conf.builder_options files = {} number_retries = 0 while self.conf.build_retries != number_retries: try: if self.conf.build_tasks is None: build_dict.update( builder.build(spec, sources, patches, results_dir, **build_dict)) if not self.conf.builds_nowait: if self.conf.buildtool == KojiBuildTool.CMD: while not build_dict['rpm']: kh = KojiHelper() build_dict['rpm'], build_dict[ 'logs'] = kh.get_koji_tasks( build_dict['koji_task_id'], results_dir) else: if self.conf.build_tasks: if self.conf.buildtool == KojiBuildTool.CMD: kh = KojiHelper() try: build_dict['rpm'], build_dict[ 'logs'] = kh.get_koji_tasks( task_id, results_dir) results_store.set_build_data( version, build_dict) if not build_dict['rpm']: return False except TypeError: logger.info( 'Koji tasks are not finished yet. Try again later' ) return False elif self.conf.buildtool == CoprBuildTool.CMD: copr_helper = CoprHelper() client = copr_helper.get_client() build_id = int(task_id) status = copr_helper.get_build_status( client, build_id) if status in [ 'importing', 'pending', 'starting', 'running' ]: logger.info( 'Copr build is not finished yet. Try again later' ) return False else: build_dict['rpm'], build_dict[ 'logs'] = copr_helper.download_build( client, build_id, results_dir) if status not in ['succeeded', 'skipped']: logger.info( 'Copr build {} did not complete successfully' .format(build_id)) return False # Build finishes properly. Go out from while cycle results_store.set_build_data(version, build_dict) break except SourcePackageBuildError: build_dict.update(builder.get_logs()) results_store.set_build_data(version, build_dict) # always fail for original version if version == 'old': raise RebaseHelperError( 'Creating old SRPM package failed.') logger.error('Building source package failed.') # TODO: implement log analyzer for SRPMs and add the checks here!!! raise except BinaryPackageBuildError: # always fail for original version rpm_dir = os.path.join(results_dir, 'RPM') build_dict.update(builder.get_logs()) results_store.set_build_data(version, build_dict) build_log = 'build.log' build_log_path = os.path.join(rpm_dir, build_log) if version == 'old': error_message = 'Building old RPM package failed. Check logs: {} '.format( builder.get_logs().get('logs', 'N/A')) raise RebaseHelperError(error_message) logger.error('Building binary packages failed.') msg = 'Building package failed' try: files = BuildLogAnalyzer.parse_log(rpm_dir, build_log) except BuildLogAnalyzerMissingError: raise RebaseHelperError('Build log %s does not exist', build_log_path) except BuildLogAnalyzerMakeError: raise RebaseHelperError( '%s during build. Check log %s', msg, build_log_path) except BuildLogAnalyzerPatchError: raise RebaseHelperError( '%s during patching. Check log %s', msg, build_log_path) except RuntimeError: if self.conf.build_retries == number_retries: raise RebaseHelperError( '%s with unknown reason. Check log %s', msg, build_log_path) if 'missing' in files: missing_files = '\n'.join(files['missing']) logger.info('Files not packaged in the SPEC file:\n%s', missing_files) elif 'deleted' in files: deleted_files = '\n'.join(files['deleted']) logger.warning( 'Removed files packaged in SPEC file:\n%s', deleted_files) else: if self.conf.build_retries == number_retries: raise RebaseHelperError( "Build failed, but no issues were found in the build log %s", build_log) self.rebase_spec_file.modify_spec_files_section(files) if not self.conf.non_interactive: msg = 'Do you want rebase-helper to try build the packages one more time' if not ConsoleHelper.get_message(msg): raise KeyboardInterrupt else: logger.warning( 'Some patches were not successfully applied') # build just failed, otherwise we would break out of the while loop logger.debug('Number of retries is %s', self.conf.build_retries) if os.path.exists(os.path.join(results_dir, 'RPM')): shutil.rmtree(os.path.join(results_dir, 'RPM')) if os.path.exists(os.path.join(results_dir, 'SRPM')): shutil.rmtree(os.path.join(results_dir, 'SRPM')) number_retries += 1 if self.conf.build_retries == number_retries: raise RebaseHelperError( 'Building package failed with unknown reason. Check all available log files.' ) return True
def run(self): # TODO: Move this check to CliHelper OR possibly to a private method validating the configuration. if self.conf.fedpkg_build_tasks: logger.warning( "Option --fedpkg-build-tasks is deprecated, use --build-tasks instead." ) if not self.conf.build_tasks: self.conf.build_tasks = self.conf.fedpkg_build_tasks # Certain options can be used only with specific build tools # here are checks for remote build tools if self.conf.buildtool not in [KojiBuildTool.CMD, CoprBuildTool.CMD]: options_used = [] if self.conf.build_tasks is not None: options_used.append('--build-tasks') if self.conf.builds_nowait is True: options_used.append('--builds-nowait') if options_used: raise RebaseHelperError( "%s can be used only with the following build tools: %s", ' and '.join(options_used), ', '.join([KojiBuildTool.CMD, CoprBuildTool.CMD])) # here are checks for local builders elif self.conf.buildtool not in [ RpmbuildBuildTool.CMD, MockBuildTool.CMD ]: options_used = [] if self.conf.builder_options is not None: options_used.append('--builder-options') if options_used: raise RebaseHelperError( "%s can be used only with the following build tools: %s", ' and '.join(options_used), ', '.join([RpmbuildBuildTool.CMD, MockBuildTool.CMD])) sources = None if self.conf.build_tasks is None: sources = self.prepare_sources() if not self.conf.build_only and not self.conf.comparepkgs: self.patch_sources(sources) build = False if not self.conf.patch_only: if not self.conf.comparepkgs: # check build dependencies for rpmbuild if self.conf.buildtool == RpmbuildBuildTool.CMD: Application.check_build_requires(self.spec_file) # Build packages try: build = self.build_packages() if self.conf.builds_nowait and not self.conf.build_tasks: if self.conf.buildtool == KojiBuildTool.CMD: self.print_koji_logs() elif self.conf.buildtool == CoprBuildTool.CMD: self.print_copr_logs() return except RuntimeError: logger.error('Unknown error caused by build log analysis') # TODO: exception should be raised instead of returning a value - it is never checked! return 1 # Perform checks else: build = self.get_rpm_packages(self.conf.comparepkgs) # We don't care dirname doesn't contain any RPM packages # Therefore return 1 if build: self.run_package_checkers(self.results_dir) else: if not self.upstream_monitoring: # TODO: This should be an ERROR logger.info( 'Rebase package to %s FAILED. See for more details', self.conf.sources) # TODO: exception should be raised instead of returning a value - it is never checked! return 1 self.print_summary() if not self.conf.keep_workspace: self._delete_workspace_dir() if self.debug_log_file: logger.info("Detailed debug log is located in '%s'", self.debug_log_file) if not self.upstream_monitoring and not self.conf.patch_only: logger.info('Rebase package to %s was SUCCESSFUL.\n', self.conf.sources) return 0
def watch_koji_tasks(cls, session, tasklist): """Function is copy/paste from pyrpkg/cli.py""" if not tasklist: return # Place holder for return value rh_tasks = {} try: tasks = {} for task_id in tasklist: tasks[task_id] = TaskWatcher(task_id, session, logger, quiet=False) while True: all_done = True for task_id, task in tasks.items(): changed = task.update() info = session.getTaskInfo(task_id) state = task.info['state'] if state == koji.TASK_STATES['FAILED']: return {info['id']: state} else: if info['arch'] == 'x86_64' or info['arch'] == 'noarch': rh_tasks[info['id']] = state if not task.is_done(): all_done = False else: if changed: cls.display_task_results(tasks) if not task.is_success(): rh_tasks = None try: for child in session.getTaskChildren(task_id): child_id = child['id'] if child_id not in tasks.keys(): tasks[child_id] = TaskWatcher(child_id, session, logger, task.level + 1, quiet=False) tasks[child_id].update() # If we found new children, go through the list # again, in case they have children also info = session.getTaskInfo(child_id) state = task.info['state'] if state == koji.TASK_STATES['FAILED']: return {info['id']: state} else: if info['arch'] == 'x86_64' or info['arch'] == 'noarch': rh_tasks[info['id']] = state all_done = False except SSL.SysCallError as e: logger.error('Detected SSL error: %s', six.text_type(e)) if all_done: cls.display_task_results(tasks) break time.sleep(1) except (KeyboardInterrupt): # A ^c should return non-zero so that it doesn't continue # on to any && commands. rh_tasks = None return rh_tasks
def run(self): # TODO: Move this check to CliHelper OR possibly to a private method validating the configuration. if self.conf.fedpkg_build_tasks: logger.warning("Option --fedpkg-build-tasks is deprecated, use --build-tasks instead.") if not self.conf.build_tasks: self.conf.build_tasks = self.conf.fedpkg_build_tasks # Certain options can be used only with specific build tools # here are checks for remote build tools if self.conf.buildtool not in [KojiBuildTool.CMD, CoprBuildTool.CMD]: options_used = [] if self.conf.build_tasks is not None: options_used.append('--build-tasks') if self.conf.builds_nowait is True: options_used.append('--builds-nowait') if options_used: raise RebaseHelperError("%s can be used only with the following build tools: %s", ' and '.join(options_used), ', '.join([KojiBuildTool.CMD, CoprBuildTool.CMD]) ) # here are checks for local builders elif self.conf.buildtool not in [RpmbuildBuildTool.CMD, MockBuildTool.CMD]: options_used = [] if self.conf.builder_options is not None: options_used.append('--builder-options') if options_used: raise RebaseHelperError("%s can be used only with the following build tools: %s", ' and '.join(options_used), ', '.join([RpmbuildBuildTool.CMD, MockBuildTool.CMD]) ) sources = None if self.conf.build_tasks is None: sources = self.prepare_sources() if not self.conf.build_only and not self.conf.comparepkgs: self.patch_sources(sources) build = False if not self.conf.patch_only: if not self.conf.comparepkgs: # check build dependencies for rpmbuild if self.conf.buildtool == RpmbuildBuildTool.CMD: Application.check_build_requires(self.spec_file) # Build packages try: build = self.build_packages() if self.conf.builds_nowait and not self.conf.build_tasks: if self.conf.buildtool == KojiBuildTool.CMD: self.print_koji_logs() elif self.conf.buildtool == CoprBuildTool.CMD: self.print_copr_logs() return except RuntimeError: logger.error('Unknown error caused by build log analysis') # TODO: exception should be raised instead of returning a value - it is never checked! return 1 # Perform checks else: build = self.get_rpm_packages(self.conf.comparepkgs) # We don't care dirname doesn't contain any RPM packages # Therefore return 1 if build: self.run_package_checkers(self.results_dir) else: if not self.upstream_monitoring: # TODO: This should be an ERROR logger.info('Rebase package to %s FAILED. See for more details', self.conf.sources) # TODO: exception should be raised instead of returning a value - it is never checked! return 1 self.print_summary() if not self.conf.keep_workspace: self._delete_workspace_dir() if self.debug_log_file: logger.info("Detailed debug log is located in '%s'", self.debug_log_file) if not self.upstream_monitoring and not self.conf.patch_only: logger.info('Rebase package to %s was SUCCESSFUL.\n', self.conf.sources) return 0
def build_binary_packages(self): """Function calls build class for building packages""" try: builder = plugin_manager.build_tools.get_plugin(self.conf.buildtool) except NotImplementedError as e: raise RebaseHelperError('{}. Supported build tools are {}'.format( str(e), plugin_manager.build_tools.get_supported_tools())) for version in ['old', 'new']: results_dir = '{}-build'.format(os.path.join(self.results_dir, version)) spec = None task_id = None koji_build_id = None build_dict = {} if self.conf.build_tasks is None: spec = self.spec_file if version == 'old' else self.rebase_spec_file package_name = spec.get_package_name() package_version = spec.get_version() package_full_version = spec.get_full_version() if version == 'old' and self.conf.get_old_build_from_koji: koji_build_id, package_version, package_full_version = KojiHelper.get_old_build_info( package_name, package_version) build_dict = dict( name=package_name, version=package_version, builds_nowait=self.conf.builds_nowait, build_tasks=self.conf.build_tasks, builder_options=self.conf.builder_options, srpm=results_store.get_build(version).get('srpm'), srpm_logs=results_store.get_build(version).get('logs')) # prepare for building builder.prepare(spec, self.conf) logger.info('Building binary packages for %s version %s', package_name, package_full_version) else: task_id = self.conf.build_tasks[0] if version == 'old' else self.conf.build_tasks[1] try: if self.conf.build_tasks is None: if koji_build_id: session = KojiHelper.create_session() build_dict['rpm'], build_dict['logs'] = KojiHelper.download_build(session, koji_build_id, os.path.join( results_dir, 'RPM', ), arches=['noarch', 'x86_64']) else: build_dict.update(builder.build(spec, results_dir, **build_dict)) if builder.CREATES_TASKS and task_id and not koji_build_id: if not self.conf.builds_nowait: build_dict['rpm'], build_dict['logs'] = builder.wait_for_task(build_dict, task_id, results_dir) elif self.conf.build_tasks: build_dict['rpm'], build_dict['logs'] = builder.get_detached_task(task_id, results_dir) build_dict = self._sanitize_build_dict(build_dict) results_store.set_build_data(version, build_dict) except RebaseHelperError: # pylint: disable=try-except-raise # Proper RebaseHelperError instance was created already. Re-raise it. raise except BinaryPackageBuildError as e: build_dict.update(builder.get_logs()) build_dict['binary_package_build_error'] = str(e) build_dict = self._sanitize_build_dict(build_dict) results_store.set_build_data(version, build_dict) if e.logfile is None: msg = 'Building {} RPM packages failed; see logs in {} for more information'.format( version, os.path.join(results_dir, 'RPM') ) else: msg = 'Building {} RPM packages failed; see {} for more information'.format(version, e.logfile) logger.error(msg) raise RebaseHelperError(msg, logfiles=builder.get_logs().get('logs')) except Exception: raise RebaseHelperError('Building package failed with unknown reason. ' 'Check all available log files.') if self.conf.builds_nowait and not self.conf.build_tasks: if builder.CREATES_TASKS: self.print_task_info(builder)
def build_packages(self): """Function calls build class for building packages""" try: builder = Builder(self.conf.buildtool) except NotImplementedError as ni_e: raise RebaseHelperError( '%s. Supported build tools are %s' % six.text_type(ni_e), Builder.get_supported_tools()) for version in ['old', 'new']: spec_object = self.spec_file if version == 'old' else self.rebase_spec_file build_dict = {} task_id = None koji_build_id = None if self.conf.build_tasks is None: pkg_name = spec_object.get_package_name() pkg_version = spec_object.get_version() pkg_full_version = spec_object.get_full_version() if version == 'old' and self.conf.get_old_build_from_koji: if KojiHelper.functional: koji_version, koji_build_id = KojiHelper.get_latest_build( pkg_name) if koji_version: if koji_version != pkg_version: logger.warning( 'Version of the latest Koji build (%s) with id (%s) ' 'differs from version in SPEC file (%s)!', koji_version, koji_build_id, pkg_version) pkg_version = pkg_full_version = koji_version else: logger.warning( 'Unable to find the latest Koji build!') else: logger.warning('Unable to get the latest Koji build!') # prepare for building builder.prepare(spec_object, self.conf) build_dict['name'] = pkg_name build_dict['version'] = pkg_version patches = [x.get_path() for x in spec_object.get_patches()] spec = spec_object.get_path() sources = spec_object.get_sources() logger.info('Building packages for %s version %s', pkg_name, pkg_full_version) else: if version == 'old': task_id = self.conf.build_tasks[0] else: task_id = self.conf.build_tasks[1] results_dir = os.path.join(self.results_dir, version) + '-build' files = {} number_retries = 0 while self.conf.build_retries != number_retries: try: if self.conf.build_tasks is None: if koji_build_id: build_dict['rpm'], build_dict[ 'logs'] = KojiHelper.download_build( koji_build_id, results_dir) else: build_dict.update( builder.build(spec, sources, patches, results_dir, **build_dict)) if builder.creates_tasks() and not koji_build_id: if not self.conf.builds_nowait: build_dict['rpm'], build_dict[ 'logs'] = builder.wait_for_task( build_dict, results_dir) if build_dict['rpm'] is None: return False elif self.conf.build_tasks: build_dict['rpm'], build_dict[ 'logs'] = builder.get_detached_task( task_id, results_dir) if build_dict['rpm'] is None: return False # Build finishes properly. Go out from while cycle results_store.set_build_data(version, build_dict) break except SourcePackageBuildError as e: build_dict.update(builder.get_logs()) build_dict['source_package_build_error'] = six.text_type(e) results_store.set_build_data(version, build_dict) # always fail for original version if version == 'old': raise RebaseHelperError( 'Creating old SRPM package failed.') logger.error('Building source package failed.') # TODO: implement log analyzer for SRPMs and add the checks here!!! raise except BinaryPackageBuildError as e: # always fail for original version rpm_dir = os.path.join(results_dir, 'RPM') build_dict.update(builder.get_logs()) build_dict['binary_package_build_error'] = six.text_type(e) results_store.set_build_data(version, build_dict) build_log = 'build.log' build_log_path = os.path.join(rpm_dir, build_log) if version == 'old': error_message = 'Building old RPM package failed. Check logs: {} '.format( builder.get_logs().get('logs', 'N/A')) raise RebaseHelperError( error_message, logfiles=builder.get_logs().get('logs')) logger.error('Building binary packages failed.') msg = 'Building package failed' try: files = BuildLogAnalyzer.parse_log(rpm_dir, build_log) except BuildLogAnalyzerMissingError: raise RebaseHelperError('Build log %s does not exist', build_log_path) except BuildLogAnalyzerMakeError: raise RebaseHelperError( '%s during build. Check log %s', msg, build_log_path, logfiles=[build_log_path]) except BuildLogAnalyzerPatchError: raise RebaseHelperError( '%s during patching. Check log %s', msg, build_log_path, logfiles=[build_log_path]) except RuntimeError: if self.conf.build_retries == number_retries: raise RebaseHelperError( '%s with unknown reason. Check log %s', msg, build_log_path, logfiles=[build_log_path]) if 'missing' in files: missing_files = '\n'.join(files['missing']) logger.info('Files not packaged in the SPEC file:\n%s', missing_files) elif 'deleted' in files: deleted_files = '\n'.join(files['deleted']) logger.warning( 'Removed files packaged in SPEC file:\n%s', deleted_files) else: if self.conf.build_retries == number_retries: raise RebaseHelperError( "Build failed, but no issues were found in the build log %s", build_log, logfiles=[build_log]) self.rebase_spec_file.modify_spec_files_section(files) if not self.conf.non_interactive: msg = 'Do you want rebase-helper to try to build the packages one more time' if not ConsoleHelper.get_message(msg): raise KeyboardInterrupt else: logger.warning( 'Some patches were not successfully applied') # build just failed, otherwise we would break out of the while loop logger.debug('Number of retries is %s', self.conf.build_retries) number_retries += 1 if self.conf.build_retries > number_retries: # only remove builds if this retry is not the last one if os.path.exists(os.path.join(results_dir, 'RPM')): shutil.rmtree(os.path.join(results_dir, 'RPM')) if os.path.exists(os.path.join(results_dir, 'SRPM')): shutil.rmtree(os.path.join(results_dir, 'SRPM')) if self.conf.build_retries == number_retries: raise RebaseHelperError( 'Building package failed with unknown reason. Check all available log files.' ) if self.conf.builds_nowait and not self.conf.build_tasks: if builder.creates_tasks(): self.print_task_info(builder) return True