コード例 #1
0
ファイル: application.py プロジェクト: FrNecas/rebase-helper
    def get_rpm_packages(self, dirname):
        """
        Function returns RPM packages stored in dirname/old and dirname/new directories

        :param dirname: directory where are stored old and new RPMS
        :return: 
        """
        found = True
        for version in ['old', 'new']:
            data = {}
            data['name'] = self.spec_file.get_package_name()
            if version == 'old':
                spec_version = self.spec_file.get_version()
            else:
                spec_version = self.rebase_spec_file.get_version()
            data['version'] = spec_version
            data['rpm'] = PathHelper.find_all_files(
                os.path.join(os.path.realpath(dirname), version, 'RPM'),
                '*.rpm')
            if not data['rpm']:
                logger.error(
                    'Your path %s%s/RPM does not contain any RPM packages',
                    dirname, version)
                found = False
            results_store.set_build_data(version, data)
        if not found:
            return False
        return True
コード例 #2
0
    def build_source_packages(self):
        try:
            builder = plugin_manager.srpm_build_tools.get_plugin(self.conf.srpm_buildtool)
        except NotImplementedError as e:
            raise RebaseHelperError('{}. Supported SRPM build tools are {}'.format(
                str(e), plugin_manager.srpm_build_tools.get_supported_tools()))

        for version in ['old', 'new']:
            koji_build_id = None
            results_dir = '{}-build'.format(os.path.join(self.results_dir, version))
            spec = self.spec_file if version == 'old' else self.rebase_spec_file
            package_name = spec.get_package_name()
            package_version = spec.get_version()
            package_full_version = spec.get_full_version()
            logger.info('Building source package for %s version %s', package_name, package_full_version)

            if version == 'old' and self.conf.get_old_build_from_koji:
                koji_build_id, package_version, package_full_version = KojiHelper.get_old_build_info(package_name,
                                                                                                     package_version)

            build_dict = dict(
                name=package_name,
                version=package_version,
                srpm_buildtool=self.conf.srpm_buildtool,
                srpm_builder_options=self.conf.srpm_builder_options)
            try:
                if koji_build_id:
                    session = KojiHelper.create_session()
                    build_dict['srpm'], build_dict['logs'] = KojiHelper.download_build(session,
                                                                                       koji_build_id,
                                                                                       os.path.join(
                                                                                           results_dir,
                                                                                           'SRPM'
                                                                                       ),
                                                                                       arches=['src'])

                else:
                    build_dict.update(builder.build(spec, results_dir, **build_dict))
                build_dict = self._sanitize_build_dict(build_dict)
                results_store.set_build_data(version, build_dict)
            except RebaseHelperError:  # pylint: disable=try-except-raise
                raise
            except SourcePackageBuildError as e:
                build_dict.update(builder.get_logs())
                build_dict['source_package_build_error'] = str(e)
                build_dict = self._sanitize_build_dict(build_dict)
                results_store.set_build_data(version, build_dict)
                if e.logfile:
                    msg = 'Building {} SRPM packages failed; see {} for more information'.format(version, e.logfile)
                else:
                    msg = 'Building {} SRPM packages failed; see logs in {} for more information'.format(
                        version, os.path.join(results_dir, 'SRPM'))
                logger.error(msg)
                raise RebaseHelperError(msg, logfiles=builder.get_logs().get('logs'))
            except Exception:
                raise RebaseHelperError('Building package failed with unknown reason. '
                                        'Check all available log files.')
コード例 #3
0
    def build_source_packages(self):
        try:
            builder = srpm_build_helper.get_tool(self.conf.srpm_buildtool)
        except NotImplementedError as e:
            raise RebaseHelperError('{}. Supported SRPM build tools are {}'.format(
                six.text_type(e), srpm_build_helper.get_supported_tools()))

        for version in ['old', 'new']:
            koji_build_id = None
            results_dir = '{}-build'.format(os.path.join(self.results_dir, version))
            spec = self.spec_file if version == 'old' else self.rebase_spec_file
            package_name = spec.get_package_name()
            package_version = spec.get_version()
            package_full_version = spec.get_full_version()
            logger.info('Building source package for %s version %s', package_name, package_full_version)

            if version == 'old' and self.conf.get_old_build_from_koji:
                koji_build_id, package_version, package_full_version = KojiHelper.get_old_build_info(package_name,
                                                                                                     package_version)

            build_dict = dict(
                name=package_name,
                version=package_version,
                srpm_buildtool=self.conf.srpm_buildtool,
                srpm_builder_options=self.conf.srpm_builder_options)
            try:
                if koji_build_id:
                    session = KojiHelper.create_session()
                    build_dict['srpm'], build_dict['logs'] = KojiHelper.download_build(session,
                                                                                       koji_build_id,
                                                                                       os.path.join(
                                                                                           results_dir,
                                                                                           'SRPM'
                                                                                       ),
                                                                                       arches=['src'])

                else:
                    build_dict.update(builder.build(spec, results_dir, **build_dict))
                build_dict = self._sanitize_build_dict(build_dict)
                results_store.set_build_data(version, build_dict)
            except RebaseHelperError:  # pylint: disable=try-except-raise
                raise
            except SourcePackageBuildError as e:
                build_dict.update(builder.get_logs())
                build_dict['source_package_build_error'] = six.text_type(e)
                build_dict = self._sanitize_build_dict(build_dict)
                results_store.set_build_data(version, build_dict)
                if e.logfile:
                    msg = 'Building {} SRPM packages failed; see {} for more information'.format(version, e.logfile)
                else:
                    msg = 'Building {} SRPM packages failed; see logs in {} for more information'.format(
                        version, os.path.join(results_dir, 'SRPM'))
                raise RebaseHelperError(msg, logfiles=builder.get_logs().get('logs'))
            except Exception:
                raise RebaseHelperError('Building package failed with unknown reason. '
                                        'Check all available log files.')
コード例 #4
0
 def run_download_compare(self, tasks_dict, dir_name):
     # TODO: Add doc text with explanation
     self.set_upstream_monitoring()
     kh = KojiHelper()
     for version in ['old', 'new']:
         rh_dict = {}
         compare_dirname = os.path.join(dir_name, version)
         if not os.path.exists(compare_dirname):
             os.mkdir(compare_dirname, 0o777)
         (task, upstream_version, package) = tasks_dict[version]
         rh_dict['rpm'], rh_dict['logs'] = kh.get_koji_tasks([task], compare_dirname)
         rh_dict['version'] = upstream_version
         rh_dict['name'] = package
         results_store.set_build_data(version, rh_dict)
     if tasks_dict['status'] == 'CLOSED':
         self.run_package_checkers(dir_name)
     self.print_summary()
     rh_stuff = self.get_rebasehelper_data()
     logger.info(rh_stuff)
     return rh_stuff
コード例 #5
0
ファイル: application.py プロジェクト: FrNecas/rebase-helper
 def run_download_compare(self, tasks_dict, dir_name):
     # TODO: Add doc text with explanation
     self.set_upstream_monitoring()
     kh = KojiHelper()
     for version in ['old', 'new']:
         rh_dict = {}
         compare_dirname = os.path.join(dir_name, version)
         if not os.path.exists(compare_dirname):
             os.mkdir(compare_dirname, 0o777)
         (task, upstream_version, package) = tasks_dict[version]
         rh_dict['rpm'], rh_dict['logs'] = kh.get_koji_tasks(
             [task], compare_dirname)
         rh_dict['version'] = upstream_version
         rh_dict['name'] = package
         results_store.set_build_data(version, rh_dict)
     if tasks_dict['status'] == 'CLOSED':
         self.run_package_checkers(dir_name)
     self.print_summary()
     rh_stuff = self.get_rebasehelper_data()
     logger.info(rh_stuff)
     return rh_stuff
コード例 #6
0
    def get_rpm_packages(self, dirname):
        """
        Function returns RPM packages stored in dirname/old and dirname/new directories

        :param dirname: directory where are stored old and new RPMS
        :return:
        """
        found = True
        for version in ['old', 'new']:
            data = {}
            data['name'] = self.spec_file.get_package_name()
            if version == 'old':
                spec_version = self.spec_file.get_version()
            else:
                spec_version = self.rebase_spec_file.get_version()
            data['version'] = spec_version
            data['rpm'] = PathHelper.find_all_files(os.path.join(os.path.realpath(dirname), version, 'RPM'), '*.rpm')
            if not data['rpm']:
                logger.error('Your path %s%s/RPM does not contain any RPM packages', dirname, version)
                found = False
            results_store.set_build_data(version, data)
        if not found:
            return False
        return True
コード例 #7
0
ファイル: application.py プロジェクト: vcrhonek/rebase-helper
    def build_binary_packages(self):
        """Function calls build class for building packages"""
        try:
            builder = Builder(self.conf.buildtool)
        except NotImplementedError as e:
            raise RebaseHelperError('{}. Supported build tools are {}'.format(
                six.text_type(e), Builder.get_supported_tools()))

        for version in ['old', 'new']:
            successful_builds = 0
            try_build_again = False
            while successful_builds < 1:
                results_dir = '{}-build'.format(os.path.join(self.results_dir, version))
                spec = None
                task_id = None
                koji_build_id = None
                build_dict = {}

                if self.conf.build_tasks is None:
                    spec = self.spec_file if version == 'old' else self.rebase_spec_file
                    package_name = spec.get_package_name()
                    package_version = spec.get_version()
                    package_full_version = spec.get_full_version()

                    if version == 'old' and self.conf.get_old_build_from_koji:
                        if KojiHelper.functional:
                            session = KojiHelper.create_session()
                            koji_version, koji_build_id = KojiHelper.get_latest_build(session, package_name)
                            if koji_version:
                                if koji_version != package_version:
                                    logger.warning('Version of the latest Koji build (%s) with id (%s) '
                                                   'differs from version in SPEC file (%s)!',
                                                   koji_version, koji_build_id, package_version)
                                package_version = package_full_version = koji_version
                            else:
                                logger.warning('Unable to find the latest Koji build!')
                        else:
                            logger.warning('Unable to get the latest Koji build!')

                    build_dict = dict(
                        name=package_name,
                        version=package_version,
                        builds_nowait=self.conf.builds_nowait,
                        build_tasks=self.conf.build_tasks,
                        builder_options=self.conf.builder_options,
                        srpm=results_store.get_build(version).get('srpm'),
                        srpm_logs=results_store.get_build(version).get('logs'))

                    # prepare for building
                    builder.prepare(spec, self.conf)

                    logger.info('Building binary packages for %s version %s', package_name, package_full_version)
                else:
                    task_id = self.conf.build_tasks[0] if version == 'old' else self.conf.build_tasks[1]

                try:
                    if self.conf.build_tasks is None:
                        if koji_build_id:
                            session = KojiHelper.create_session()
                            build_dict['rpm'], build_dict['logs'] = KojiHelper.download_build(session,
                                                                                              koji_build_id,
                                                                                              results_dir)
                        else:
                            build_dict.update(builder.build(spec, results_dir, **build_dict))
                    if builder.creates_tasks() and task_id and not koji_build_id:
                        if not self.conf.builds_nowait:
                            build_dict['rpm'], build_dict['logs'] = builder.wait_for_task(build_dict,
                                                                                          task_id,
                                                                                          results_dir)
                        elif self.conf.build_tasks:
                            build_dict['rpm'], build_dict['logs'] = builder.get_detached_task(task_id, results_dir)
                    build_dict = self._sanitize_build_dict(build_dict)
                    results_store.set_build_data(version, build_dict)
                    successful_builds += 1
                except RebaseHelperError:
                    # Proper RebaseHelperError instance was created already. Re-raise it.
                    raise
                except BinaryPackageBuildError as e:
                    build_dict.update(builder.get_logs())
                    build_dict['binary_package_build_error'] = six.text_type(e)
                    build_dict = self._sanitize_build_dict(build_dict)
                    results_store.set_build_data(version, build_dict)

                    if e.logfile is None:
                        msg = 'Building {} RPM packages failed; see logs in {} for more information'.format(
                            version, os.path.join(results_dir, 'RPM')
                        )
                    else:
                        msg = 'Building {} RPM packages failed; see {} for more information'.format(version, e.logfile)

                    logger.info(msg)
                    if self.rebase_spec_file:
                        # Save current rebase spec file content
                        self.rebase_spec_file.save()
                    if not self.conf.non_interactive and \
                            ConsoleHelper.get_message('Do you want to try it one more time'):
                        try_build_again = True
                    else:
                        raise RebaseHelperError(msg, logfiles=builder.get_logs().get('logs'))
                except Exception:
                    raise RebaseHelperError('Building package failed with unknown reason. '
                                            'Check all available log files.')

                if try_build_again:
                    successful_builds = 0
                    try_build_again = False

                    logger.info('Now it is time to make changes to  %s if necessary.', self.rebase_spec_file.path)
                    if not ConsoleHelper.get_message('Do you want to continue with the rebuild now'):
                        raise KeyboardInterrupt
                    # Update rebase spec file content after potential manual modifications
                    self.rebase_spec_file._read_spec_content()  # pylint: disable=protected-access
                    self.rebase_spec_file._update_data()  # pylint: disable=protected-access
                    # clear current version output directories
                    if os.path.exists(os.path.join(results_dir, 'RPM')):
                        shutil.rmtree(os.path.join(results_dir, 'RPM'))

        if self.conf.builds_nowait and not self.conf.build_tasks:
            if builder.creates_tasks():
                self.print_task_info(builder)
コード例 #8
0
ファイル: application.py プロジェクト: FrNecas/rebase-helper
    def build_packages(self):
        """Function calls build class for building packages"""
        try:
            builder = Builder(self.conf.buildtool)
        except NotImplementedError as ni_e:
            raise RebaseHelperError(
                '%s. Supported build tools are %s' % six.text_type(ni_e),
                Builder.get_supported_tools())

        for version in ['old', 'new']:
            spec_object = self.spec_file if version == 'old' else self.rebase_spec_file
            build_dict = {}
            task_id = None
            koji_build_id = None

            if self.conf.build_tasks is None:
                pkg_name = spec_object.get_package_name()
                pkg_version = spec_object.get_version()
                pkg_full_version = spec_object.get_full_version()

                if version == 'old' and self.conf.get_old_build_from_koji:
                    if KojiHelper.functional:
                        koji_version, koji_build_id = KojiHelper.get_latest_build(
                            pkg_name)
                        if koji_version:
                            if koji_version != pkg_version:
                                logger.warning(
                                    'Version of the latest Koji build (%s) with id (%s) '
                                    'differs from version in SPEC file (%s)!',
                                    koji_version, koji_build_id, pkg_version)
                            pkg_version = pkg_full_version = koji_version
                        else:
                            logger.warning(
                                'Unable to find the latest Koji build!')
                    else:
                        logger.warning('Unable to get the latest Koji build!')

                # prepare for building
                builder.prepare(spec_object, self.conf)

                build_dict['name'] = pkg_name
                build_dict['version'] = pkg_version
                patches = [x.get_path() for x in spec_object.get_patches()]
                spec = spec_object.get_path()
                sources = spec_object.get_sources()
                logger.info('Building packages for %s version %s', pkg_name,
                            pkg_full_version)
            else:
                if version == 'old':
                    task_id = self.conf.build_tasks[0]
                else:
                    task_id = self.conf.build_tasks[1]
            results_dir = os.path.join(self.results_dir, version) + '-build'

            files = {}
            number_retries = 0
            while self.conf.build_retries != number_retries:
                try:
                    if self.conf.build_tasks is None:
                        if koji_build_id:
                            build_dict['rpm'], build_dict[
                                'logs'] = KojiHelper.download_build(
                                    koji_build_id, results_dir)
                        else:
                            build_dict.update(
                                builder.build(spec, sources, patches,
                                              results_dir, **build_dict))
                    if builder.creates_tasks() and not koji_build_id:
                        if not self.conf.builds_nowait:
                            build_dict['rpm'], build_dict[
                                'logs'] = builder.wait_for_task(
                                    build_dict, results_dir)
                            if build_dict['rpm'] is None:
                                return False
                        elif self.conf.build_tasks:
                            build_dict['rpm'], build_dict[
                                'logs'] = builder.get_detached_task(
                                    task_id, results_dir)
                            if build_dict['rpm'] is None:
                                return False
                    # Build finishes properly. Go out from while cycle
                    results_store.set_build_data(version, build_dict)
                    break

                except SourcePackageBuildError as e:
                    build_dict.update(builder.get_logs())
                    build_dict['source_package_build_error'] = six.text_type(e)
                    results_store.set_build_data(version, build_dict)
                    #  always fail for original version
                    if version == 'old':
                        raise RebaseHelperError(
                            'Creating old SRPM package failed.')
                    logger.error('Building source package failed.')
                    #  TODO: implement log analyzer for SRPMs and add the checks here!!!
                    raise

                except BinaryPackageBuildError as e:
                    #  always fail for original version
                    rpm_dir = os.path.join(results_dir, 'RPM')
                    build_dict.update(builder.get_logs())
                    build_dict['binary_package_build_error'] = six.text_type(e)
                    results_store.set_build_data(version, build_dict)
                    build_log = 'build.log'
                    build_log_path = os.path.join(rpm_dir, build_log)
                    if version == 'old':
                        error_message = 'Building old RPM package failed. Check logs: {} '.format(
                            builder.get_logs().get('logs', 'N/A'))
                        raise RebaseHelperError(
                            error_message,
                            logfiles=builder.get_logs().get('logs'))
                    logger.error('Building binary packages failed.')
                    msg = 'Building package failed'
                    try:
                        files = BuildLogAnalyzer.parse_log(rpm_dir, build_log)
                    except BuildLogAnalyzerMissingError:
                        raise RebaseHelperError('Build log %s does not exist',
                                                build_log_path)
                    except BuildLogAnalyzerMakeError:
                        raise RebaseHelperError(
                            '%s during build. Check log %s',
                            msg,
                            build_log_path,
                            logfiles=[build_log_path])
                    except BuildLogAnalyzerPatchError:
                        raise RebaseHelperError(
                            '%s during patching. Check log %s',
                            msg,
                            build_log_path,
                            logfiles=[build_log_path])
                    except RuntimeError:
                        if self.conf.build_retries == number_retries:
                            raise RebaseHelperError(
                                '%s with unknown reason. Check log %s',
                                msg,
                                build_log_path,
                                logfiles=[build_log_path])

                    if 'missing' in files:
                        missing_files = '\n'.join(files['missing'])
                        logger.info('Files not packaged in the SPEC file:\n%s',
                                    missing_files)
                    elif 'deleted' in files:
                        deleted_files = '\n'.join(files['deleted'])
                        logger.warning(
                            'Removed files packaged in SPEC file:\n%s',
                            deleted_files)
                    else:
                        if self.conf.build_retries == number_retries:
                            raise RebaseHelperError(
                                "Build failed, but no issues were found in the build log %s",
                                build_log,
                                logfiles=[build_log])
                    self.rebase_spec_file.modify_spec_files_section(files)

                if not self.conf.non_interactive:
                    msg = 'Do you want rebase-helper to try to build the packages one more time'
                    if not ConsoleHelper.get_message(msg):
                        raise KeyboardInterrupt
                else:
                    logger.warning(
                        'Some patches were not successfully applied')
                # build just failed, otherwise we would break out of the while loop
                logger.debug('Number of retries is %s',
                             self.conf.build_retries)
                number_retries += 1
                if self.conf.build_retries > number_retries:
                    # only remove builds if this retry is not the last one
                    if os.path.exists(os.path.join(results_dir, 'RPM')):
                        shutil.rmtree(os.path.join(results_dir, 'RPM'))
                    if os.path.exists(os.path.join(results_dir, 'SRPM')):
                        shutil.rmtree(os.path.join(results_dir, 'SRPM'))
            if self.conf.build_retries == number_retries:
                raise RebaseHelperError(
                    'Building package failed with unknown reason. Check all available log files.'
                )

        if self.conf.builds_nowait and not self.conf.build_tasks:
            if builder.creates_tasks():
                self.print_task_info(builder)

        return True
コード例 #9
0
    def build_binary_packages(self):
        """Function calls build class for building packages"""
        try:
            builder = build_helper.get_tool(self.conf.buildtool)
        except NotImplementedError as e:
            raise RebaseHelperError('{}. Supported build tools are {}'.format(
                six.text_type(e), build_helper.get_supported_tools()))

        for version in ['old', 'new']:
            results_dir = '{}-build'.format(os.path.join(self.results_dir, version))
            spec = None
            task_id = None
            koji_build_id = None
            build_dict = {}

            if self.conf.build_tasks is None:
                spec = self.spec_file if version == 'old' else self.rebase_spec_file
                package_name = spec.get_package_name()
                package_version = spec.get_version()
                package_full_version = spec.get_full_version()

                if version == 'old' and self.conf.get_old_build_from_koji:
                    koji_build_id, package_version, package_full_version = KojiHelper.get_old_build_info(
                                                                               package_name,
                                                                               package_version)

                build_dict = dict(
                    name=package_name,
                    version=package_version,
                    builds_nowait=self.conf.builds_nowait,
                    build_tasks=self.conf.build_tasks,
                    builder_options=self.conf.builder_options,
                    srpm=results_store.get_build(version).get('srpm'),
                    srpm_logs=results_store.get_build(version).get('logs'))

                # prepare for building
                builder.prepare(spec, self.conf)

                logger.info('Building binary packages for %s version %s', package_name, package_full_version)
            else:
                task_id = self.conf.build_tasks[0] if version == 'old' else self.conf.build_tasks[1]

            try:
                if self.conf.build_tasks is None:
                    if koji_build_id:
                        session = KojiHelper.create_session()
                        build_dict['rpm'], build_dict['logs'] = KojiHelper.download_build(session,
                                                                                          koji_build_id,
                                                                                          os.path.join(
                                                                                              results_dir,
                                                                                              'RPM',
                                                                                          ),
                                                                                          arches=['noarch', 'x86_64'])
                    else:
                        build_dict.update(builder.build(spec, results_dir, **build_dict))
                if builder.CREATES_TASKS and task_id and not koji_build_id:
                    if not self.conf.builds_nowait:
                        build_dict['rpm'], build_dict['logs'] = builder.wait_for_task(build_dict,
                                                                                      task_id,
                                                                                      results_dir)
                    elif self.conf.build_tasks:
                        build_dict['rpm'], build_dict['logs'] = builder.get_detached_task(task_id, results_dir)
                build_dict = self._sanitize_build_dict(build_dict)
                results_store.set_build_data(version, build_dict)
            except RebaseHelperError:  # pylint: disable=try-except-raise
                # Proper RebaseHelperError instance was created already. Re-raise it.
                raise
            except BinaryPackageBuildError as e:
                build_dict.update(builder.get_logs())
                build_dict['binary_package_build_error'] = six.text_type(e)
                build_dict = self._sanitize_build_dict(build_dict)
                results_store.set_build_data(version, build_dict)

                if e.logfile is None:
                    msg = 'Building {} RPM packages failed; see logs in {} for more information'.format(
                        version, os.path.join(results_dir, 'RPM')
                    )
                else:
                    msg = 'Building {} RPM packages failed; see {} for more information'.format(version, e.logfile)

                logger.info(msg)
                raise RebaseHelperError(msg, logfiles=builder.get_logs().get('logs'))
            except Exception:
                raise RebaseHelperError('Building package failed with unknown reason. '
                                        'Check all available log files.')

        if self.conf.builds_nowait and not self.conf.build_tasks:
            if builder.CREATES_TASKS:
                self.print_task_info(builder)
コード例 #10
0
    def build_binary_packages(self):
        """Function calls build class for building packages"""
        try:
            builder = plugin_manager.build_tools.get_plugin(self.conf.buildtool)
        except NotImplementedError as e:
            raise RebaseHelperError('{}. Supported build tools are {}'.format(
                str(e), plugin_manager.build_tools.get_supported_tools()))

        for version in ['old', 'new']:
            results_dir = '{}-build'.format(os.path.join(self.results_dir, version))
            spec = None
            task_id = None
            koji_build_id = None
            build_dict = {}

            if self.conf.build_tasks is None:
                spec = self.spec_file if version == 'old' else self.rebase_spec_file
                package_name = spec.get_package_name()
                package_version = spec.get_version()
                package_full_version = spec.get_full_version()

                if version == 'old' and self.conf.get_old_build_from_koji:
                    koji_build_id, package_version, package_full_version = KojiHelper.get_old_build_info(
                                                                               package_name,
                                                                               package_version)

                build_dict = dict(
                    name=package_name,
                    version=package_version,
                    builds_nowait=self.conf.builds_nowait,
                    build_tasks=self.conf.build_tasks,
                    builder_options=self.conf.builder_options,
                    srpm=results_store.get_build(version).get('srpm'),
                    srpm_logs=results_store.get_build(version).get('logs'))

                # prepare for building
                builder.prepare(spec, self.conf)

                logger.info('Building binary packages for %s version %s', package_name, package_full_version)
            else:
                task_id = self.conf.build_tasks[0] if version == 'old' else self.conf.build_tasks[1]

            try:
                if self.conf.build_tasks is None:
                    if koji_build_id:
                        session = KojiHelper.create_session()
                        build_dict['rpm'], build_dict['logs'] = KojiHelper.download_build(session,
                                                                                          koji_build_id,
                                                                                          os.path.join(
                                                                                              results_dir,
                                                                                              'RPM',
                                                                                          ),
                                                                                          arches=['noarch', 'x86_64'])
                    else:
                        build_dict.update(builder.build(spec, results_dir, **build_dict))
                if builder.CREATES_TASKS and task_id and not koji_build_id:
                    if not self.conf.builds_nowait:
                        build_dict['rpm'], build_dict['logs'] = builder.wait_for_task(build_dict,
                                                                                      task_id,
                                                                                      results_dir)
                    elif self.conf.build_tasks:
                        build_dict['rpm'], build_dict['logs'] = builder.get_detached_task(task_id, results_dir)
                build_dict = self._sanitize_build_dict(build_dict)
                results_store.set_build_data(version, build_dict)
            except RebaseHelperError:  # pylint: disable=try-except-raise
                # Proper RebaseHelperError instance was created already. Re-raise it.
                raise
            except BinaryPackageBuildError as e:
                build_dict.update(builder.get_logs())
                build_dict['binary_package_build_error'] = str(e)
                build_dict = self._sanitize_build_dict(build_dict)
                results_store.set_build_data(version, build_dict)

                if e.logfile is None:
                    msg = 'Building {} RPM packages failed; see logs in {} for more information'.format(
                        version, os.path.join(results_dir, 'RPM')
                    )
                else:
                    msg = 'Building {} RPM packages failed; see {} for more information'.format(version, e.logfile)

                logger.error(msg)
                raise RebaseHelperError(msg, logfiles=builder.get_logs().get('logs'))
            except Exception:
                raise RebaseHelperError('Building package failed with unknown reason. '
                                        'Check all available log files.')

        if self.conf.builds_nowait and not self.conf.build_tasks:
            if builder.CREATES_TASKS:
                self.print_task_info(builder)
コード例 #11
0
ファイル: application.py プロジェクト: dhodovsk/rebase-helper
    def build_source_packages(self):
        try:
            builder = plugin_manager.srpm_build_tools.get_plugin(
                self.conf.srpm_buildtool)
        except NotImplementedError as e:
            raise RebaseHelperError(
                '{}. Supported SRPM build tools are {}'.format(
                    str(e), ', '.join(plugin_manager.srpm_build_tools.
                                      get_supported_plugins()))) from e

        for version in ['old', 'new']:
            koji_build_id = None
            results_dir = os.path.join(self.results_dir,
                                       '{}-build'.format(version), 'SRPM')
            spec = self.spec_file if version == 'old' else self.rebase_spec_file
            package_name = spec.header.name
            package_version = spec.header.version
            logger.info('Building source package for %s version %s',
                        package_name, package_version)

            if version == 'old' and self.conf.get_old_build_from_koji:
                koji_build_id, ver = KojiHelper.get_old_build_info(
                    package_name, package_version)
                if ver:
                    package_version = ver

            build_dict = dict(
                name=package_name,
                version=package_version,
                srpm_buildtool=self.conf.srpm_buildtool,
                srpm_builder_options=self.conf.srpm_builder_options,
                app_kwargs=self.kwargs)
            try:
                os.makedirs(results_dir)
                if koji_build_id:
                    session = KojiHelper.create_session()
                    build_dict['srpm'], build_dict[
                        'logs'] = KojiHelper.download_build(session,
                                                            koji_build_id,
                                                            results_dir,
                                                            arches=['src'])

                else:
                    build_dict.update(
                        builder.build(spec, results_dir, **build_dict))
                build_dict = self._sanitize_build_dict(build_dict)
                results_store.set_build_data(version, build_dict)
            except RebaseHelperError:  # pylint: disable=try-except-raise
                raise
            except SourcePackageBuildError as e:
                build_dict['logs'] = e.logs
                build_dict['source_package_build_error'] = str(e)
                build_dict = self._sanitize_build_dict(build_dict)
                results_store.set_build_data(version, build_dict)
                if e.logfile:
                    msg = 'Building {} SRPM packages failed; see {} for more information'.format(
                        version, e.logfile)
                else:
                    msg = 'Building {} SRPM packages failed; see logs in {} for more information'.format(
                        version, results_dir)
                raise RebaseHelperError(msg, logfiles=e.logs) from e
            except Exception as e:
                raise RebaseHelperError(
                    'Building package failed with unknown reason. '
                    'Check all available log files.') from e
コード例 #12
0
    def build_packages(self):
        """Function calls build class for building packages"""
        if self.conf.buildtool == KojiBuildTool.CMD and not koji_builder:
            logger.info('Importing module koji failed. Switching to mock builder.')
            self.conf.buildtool = MockBuildTool.CMD
        try:
            builder = Builder(self.conf.buildtool)
        except NotImplementedError as ni_e:
            raise RebaseHelperError('%s. Supported build tools are %s' % six.text_type(ni_e),
                                    Builder.get_supported_tools())

        for version in ['old', 'new']:
            spec_object = self.spec_file if version == 'old' else self.rebase_spec_file
            build_dict = {}
            task_id = None
            if self.conf.build_tasks is None:
                build_dict['name'] = spec_object.get_package_name()
                build_dict['version'] = spec_object.get_version()
                patches = [x.get_path() for x in spec_object.get_patches()]
                spec = spec_object.get_path()
                sources = spec_object.get_sources()
                logger.info('Building packages for %s version %s',
                            spec_object.get_package_name(),
                            spec_object.get_full_version())
            else:
                if version == 'old':
                    task_id = self.conf.build_tasks[0]
                else:
                    task_id = self.conf.build_tasks[1]
            results_dir = os.path.join(self.results_dir, version)
            build_dict['builds_nowait'] = self.conf.builds_nowait
            build_dict['build_tasks'] = self.conf.build_tasks
            build_dict['builder_options'] = self.conf.builder_options

            files = {}
            number_retries = 0
            while self.conf.build_retries != number_retries:
                try:
                    if self.conf.build_tasks is None:
                        build_dict.update(builder.build(spec, sources, patches, results_dir, **build_dict))
                    if not self.conf.builds_nowait:
                        if self.conf.buildtool == KojiBuildTool.CMD:
                            while not build_dict['rpm']:
                                kh = KojiHelper()
                                build_dict['rpm'], build_dict['logs'] = kh.get_koji_tasks(
                                    build_dict['koji_task_id'],
                                    results_dir
                                )
                    else:
                        if self.conf.build_tasks:
                            if self.conf.buildtool == KojiBuildTool.CMD:
                                kh = KojiHelper()
                                try:
                                    build_dict['rpm'], build_dict['logs'] = kh.get_koji_tasks(task_id, results_dir)
                                    results_store.set_build_data(version, build_dict)
                                    if not build_dict['rpm']:
                                        return False
                                except TypeError:
                                    logger.info('Koji tasks are not finished yet. Try again later')
                                    return False
                            elif self.conf.buildtool == CoprBuildTool.CMD:
                                copr_helper = CoprHelper()
                                client = copr_helper.get_client()
                                build_id = int(task_id)
                                status = copr_helper.get_build_status(client, build_id)
                                if status in ['importing', 'pending', 'starting', 'running']:
                                    logger.info('Copr build is not finished yet. Try again later')
                                    return False
                                else:
                                    build_dict['rpm'], build_dict['logs'] = copr_helper.download_build(
                                        client,
                                        build_id,
                                        results_dir
                                    )
                                    if status not in ['succeeded', 'skipped']:
                                        logger.info('Copr build {} did not complete successfully'.format(build_id))
                                        return False
                    # Build finishes properly. Go out from while cycle
                    results_store.set_build_data(version, build_dict)
                    break

                except SourcePackageBuildError:
                    build_dict.update(builder.get_logs())
                    results_store.set_build_data(version, build_dict)
                    #  always fail for original version
                    if version == 'old':
                        raise RebaseHelperError('Creating old SRPM package failed.')
                    logger.error('Building source package failed.')
                    #  TODO: implement log analyzer for SRPMs and add the checks here!!!
                    raise

                except BinaryPackageBuildError:
                    #  always fail for original version
                    rpm_dir = os.path.join(results_dir, 'RPM')
                    build_dict.update(builder.get_logs())
                    results_store.set_build_data(version, build_dict)
                    build_log = 'build.log'
                    build_log_path = os.path.join(rpm_dir, build_log)
                    if version == 'old':
                        error_message = 'Building old RPM package failed. Check logs: {} '.format(
                            builder.get_logs().get('logs', 'N/A')
                        )
                        raise RebaseHelperError(error_message)
                    logger.error('Building binary packages failed.')
                    msg = 'Building package failed'
                    try:
                        files = BuildLogAnalyzer.parse_log(rpm_dir, build_log)
                    except BuildLogAnalyzerMissingError:
                        raise RebaseHelperError('Build log %s does not exist', build_log_path)
                    except BuildLogAnalyzerMakeError:
                        raise RebaseHelperError('%s during build. Check log %s', msg, build_log_path)
                    except BuildLogAnalyzerPatchError:
                        raise RebaseHelperError('%s during patching. Check log %s', msg, build_log_path)
                    except RuntimeError:
                        if self.conf.build_retries == number_retries:
                            raise RebaseHelperError('%s with unknown reason. Check log %s', msg, build_log_path)

                    if 'missing' in files:
                        missing_files = '\n'.join(files['missing'])
                        logger.info('Files not packaged in the SPEC file:\n%s', missing_files)
                    elif 'deleted' in files:
                        deleted_files = '\n'.join(files['deleted'])
                        logger.warning('Removed files packaged in SPEC file:\n%s', deleted_files)
                    else:
                        if self.conf.build_retries == number_retries:
                            raise RebaseHelperError("Build failed, but no issues were found in the build log %s",
                                                    build_log)
                    self.rebase_spec_file.modify_spec_files_section(files)

                if not self.conf.non_interactive:
                        msg = 'Do you want rebase-helper to try build the packages one more time'
                        if not ConsoleHelper.get_message(msg):
                            raise KeyboardInterrupt
                else:
                    logger.warning('Some patches were not successfully applied')
                #  build just failed, otherwise we would break out of the while loop
                logger.debug('Number of retries is %s', self.conf.build_retries)
                if os.path.exists(os.path.join(results_dir, 'RPM')):
                    shutil.rmtree(os.path.join(results_dir, 'RPM'))
                if os.path.exists(os.path.join(results_dir, 'SRPM')):
                    shutil.rmtree(os.path.join(results_dir, 'SRPM'))
                number_retries += 1
            if self.conf.build_retries == number_retries:
                raise RebaseHelperError('Building package failed with unknown reason. Check all available log files.')

        return True
コード例 #13
0
    def build_packages(self):
        """Function calls build class for building packages"""
        if self.conf.buildtool == KojiBuildTool.CMD and not koji_builder:
            logger.info(
                'Importing module koji failed. Switching to mock builder.')
            self.conf.buildtool = MockBuildTool.CMD
        try:
            builder = Builder(self.conf.buildtool)
        except NotImplementedError as ni_e:
            raise RebaseHelperError(
                '%s. Supported build tools are %s' % six.text_type(ni_e),
                Builder.get_supported_tools())

        for version in ['old', 'new']:
            spec_object = self.spec_file if version == 'old' else self.rebase_spec_file
            build_dict = {}
            task_id = None
            if self.conf.build_tasks is None:
                build_dict['name'] = spec_object.get_package_name()
                build_dict['version'] = spec_object.get_version()
                patches = [x.get_path() for x in spec_object.get_patches()]
                spec = spec_object.get_path()
                sources = spec_object.get_sources()
                logger.info('Building packages for %s version %s',
                            spec_object.get_package_name(),
                            spec_object.get_full_version())
            else:
                if version == 'old':
                    task_id = self.conf.build_tasks[0]
                else:
                    task_id = self.conf.build_tasks[1]
            results_dir = os.path.join(self.results_dir, version)
            build_dict['builds_nowait'] = self.conf.builds_nowait
            build_dict['build_tasks'] = self.conf.build_tasks
            build_dict['builder_options'] = self.conf.builder_options

            files = {}
            number_retries = 0
            while self.conf.build_retries != number_retries:
                try:
                    if self.conf.build_tasks is None:
                        build_dict.update(
                            builder.build(spec, sources, patches, results_dir,
                                          **build_dict))
                    if not self.conf.builds_nowait:
                        if self.conf.buildtool == KojiBuildTool.CMD:
                            while not build_dict['rpm']:
                                kh = KojiHelper()
                                build_dict['rpm'], build_dict[
                                    'logs'] = kh.get_koji_tasks(
                                        build_dict['koji_task_id'],
                                        results_dir)
                    else:
                        if self.conf.build_tasks:
                            if self.conf.buildtool == KojiBuildTool.CMD:
                                kh = KojiHelper()
                                try:
                                    build_dict['rpm'], build_dict[
                                        'logs'] = kh.get_koji_tasks(
                                            task_id, results_dir)
                                    results_store.set_build_data(
                                        version, build_dict)
                                    if not build_dict['rpm']:
                                        return False
                                except TypeError:
                                    logger.info(
                                        'Koji tasks are not finished yet. Try again later'
                                    )
                                    return False
                            elif self.conf.buildtool == CoprBuildTool.CMD:
                                copr_helper = CoprHelper()
                                client = copr_helper.get_client()
                                build_id = int(task_id)
                                status = copr_helper.get_build_status(
                                    client, build_id)
                                if status in [
                                        'importing', 'pending', 'starting',
                                        'running'
                                ]:
                                    logger.info(
                                        'Copr build is not finished yet. Try again later'
                                    )
                                    return False
                                else:
                                    build_dict['rpm'], build_dict[
                                        'logs'] = copr_helper.download_build(
                                            client, build_id, results_dir)
                                    if status not in ['succeeded', 'skipped']:
                                        logger.info(
                                            'Copr build {} did not complete successfully'
                                            .format(build_id))
                                        return False
                    # Build finishes properly. Go out from while cycle
                    results_store.set_build_data(version, build_dict)
                    break

                except SourcePackageBuildError:
                    build_dict.update(builder.get_logs())
                    results_store.set_build_data(version, build_dict)
                    #  always fail for original version
                    if version == 'old':
                        raise RebaseHelperError(
                            'Creating old SRPM package failed.')
                    logger.error('Building source package failed.')
                    #  TODO: implement log analyzer for SRPMs and add the checks here!!!
                    raise

                except BinaryPackageBuildError:
                    #  always fail for original version
                    rpm_dir = os.path.join(results_dir, 'RPM')
                    build_dict.update(builder.get_logs())
                    results_store.set_build_data(version, build_dict)
                    build_log = 'build.log'
                    build_log_path = os.path.join(rpm_dir, build_log)
                    if version == 'old':
                        error_message = 'Building old RPM package failed. Check logs: {} '.format(
                            builder.get_logs().get('logs', 'N/A'))
                        raise RebaseHelperError(error_message)
                    logger.error('Building binary packages failed.')
                    msg = 'Building package failed'
                    try:
                        files = BuildLogAnalyzer.parse_log(rpm_dir, build_log)
                    except BuildLogAnalyzerMissingError:
                        raise RebaseHelperError('Build log %s does not exist',
                                                build_log_path)
                    except BuildLogAnalyzerMakeError:
                        raise RebaseHelperError(
                            '%s during build. Check log %s', msg,
                            build_log_path)
                    except BuildLogAnalyzerPatchError:
                        raise RebaseHelperError(
                            '%s during patching. Check log %s', msg,
                            build_log_path)
                    except RuntimeError:
                        if self.conf.build_retries == number_retries:
                            raise RebaseHelperError(
                                '%s with unknown reason. Check log %s', msg,
                                build_log_path)

                    if 'missing' in files:
                        missing_files = '\n'.join(files['missing'])
                        logger.info('Files not packaged in the SPEC file:\n%s',
                                    missing_files)
                    elif 'deleted' in files:
                        deleted_files = '\n'.join(files['deleted'])
                        logger.warning(
                            'Removed files packaged in SPEC file:\n%s',
                            deleted_files)
                    else:
                        if self.conf.build_retries == number_retries:
                            raise RebaseHelperError(
                                "Build failed, but no issues were found in the build log %s",
                                build_log)
                    self.rebase_spec_file.modify_spec_files_section(files)

                if not self.conf.non_interactive:
                    msg = 'Do you want rebase-helper to try build the packages one more time'
                    if not ConsoleHelper.get_message(msg):
                        raise KeyboardInterrupt
                else:
                    logger.warning(
                        'Some patches were not successfully applied')
                #  build just failed, otherwise we would break out of the while loop
                logger.debug('Number of retries is %s',
                             self.conf.build_retries)
                if os.path.exists(os.path.join(results_dir, 'RPM')):
                    shutil.rmtree(os.path.join(results_dir, 'RPM'))
                if os.path.exists(os.path.join(results_dir, 'SRPM')):
                    shutil.rmtree(os.path.join(results_dir, 'SRPM'))
                number_retries += 1
            if self.conf.build_retries == number_retries:
                raise RebaseHelperError(
                    'Building package failed with unknown reason. Check all available log files.'
                )

        return True