Пример #1
0
    def run(self):
        """
        Steps through the entire workflow of a repo sync.

        :return:    A SyncReport detailing how the sync went
        :rtype:     pulp.plugins.model.SyncReport
        """
        # using this tmp dir ensures that cleanup leaves nothing behind, since
        # we delete below
        self.tmp_dir = tempfile.mkdtemp(dir=self.working_dir)
        try:
            self.progress_status['metadata']['state'] = constants.STATE_RUNNING
            self.set_progress()

            # Verify that we have a feed url.  if there is no feed url then we have nothing to sync
            if self.sync_feed is None:
                raise FailedException(_('Unable to sync a repository that has no feed'))

            metadata_files = self.get_metadata()
            # Save the default checksum from the metadata
            self.save_default_metadata_checksum_on_repo(metadata_files)

            if self.progress_status['metadata']['state'] == constants.STATE_RUNNING:
                self.progress_status['metadata']['state'] = constants.STATE_COMPLETE
            self.set_progress()

            self.content_report['state'] = constants.STATE_RUNNING
            self.set_progress()
            self.update_content(metadata_files)
            if self.content_report['state'] == constants.STATE_RUNNING:
                self.content_report['state'] = constants.STATE_COMPLETE
            self.set_progress()

            _logger.info(_('Downloading additional units.'))
            if models.Distribution.TYPE in self.call_config.get(constants.CONFIG_SKIP, []):
                self.distribution_report['state'] = constants.STATE_SKIPPED
            else:
                self.distribution_report['state'] = constants.STATE_RUNNING
                self.set_progress()
                treeinfo.sync(self.sync_conduit, self.sync_feed, self.tmp_dir,
                              self.nectar_config, self.distribution_report, self.set_progress)
            self.set_progress()

            if models.Errata.TYPE in self.call_config.get(constants.CONFIG_SKIP, []):
                self.progress_status['errata']['state'] = constants.STATE_SKIPPED
            else:
                self.progress_status['errata']['state'] = constants.STATE_RUNNING
                self.set_progress()
                self.get_errata(metadata_files)
                self.progress_status['errata']['state'] = constants.STATE_COMPLETE
            self.set_progress()

            self.progress_status['comps']['state'] = constants.STATE_RUNNING
            self.set_progress()
            self.get_comps_file_units(metadata_files, group.process_group_element, group.GROUP_TAG)
            self.get_comps_file_units(metadata_files, group.process_category_element,
                                      group.CATEGORY_TAG)
            self.get_comps_file_units(metadata_files, group.process_environment_element,
                                      group.ENVIRONMENT_TAG)
            self.progress_status['comps']['state'] = constants.STATE_COMPLETE
            self.set_progress()

        except CancelException:
            report = self.sync_conduit.build_cancel_report(self._progress_summary, self.progress_status)
            report.canceled_flag = True
            return report

        except Exception, e:
            _logger.exception('sync failed')
            for step, value in self.progress_status.iteritems():
                if value.get('state') == constants.STATE_RUNNING:
                    value['state'] = constants.STATE_FAILED
                    value['error'] = str(e)
            self.set_progress()
            report = self.sync_conduit.build_failure_report(self._progress_summary, self.progress_status)
            return report
Пример #2
0
    def run(self):
        """
        Steps through the entire workflow of a repo sync.

        :return:    A SyncReport detailing how the sync went
        :rtype:     pulp.plugins.model.SyncReport
        """
        # Empty list could be returned in case _parse_as_mirrorlist()
        # was not able to find any valid url
        if self.sync_feed == []:
            raise PulpCodedException(error_code=error_codes.RPM1004,
                                     reason='Not found')
        url_count = 0
        for url in self.sync_feed:
            # Verify that we have a feed url.
            # if there is no feed url, then we have nothing to sync
            if url is None:
                raise PulpCodedException(error_code=error_codes.RPM1005)
            # using this tmp dir ensures that cleanup leaves nothing behind, since
            # we delete below
            self.tmp_dir = tempfile.mkdtemp(dir=self.working_dir)
            url_count += 1
            try:
                with self.update_state(self.progress_status['metadata']):
                    metadata_files = self.check_metadata(url)
                    metadata_files = self.get_metadata(metadata_files)

                    # Save the default checksum from the metadata
                    self.save_default_metadata_checksum_on_repo(metadata_files)

                with self.update_state(self.content_report) as skip:
                    if not (skip or self.skip_repomd_steps):
                        self.update_content(metadata_files, url)

                _logger.info(_('Downloading additional units.'))

                with self.update_state(self.distribution_report,
                                       models.Distribution.TYPE) as skip:
                    if not skip:
                        treeinfo.sync(self.sync_conduit, url, self.tmp_dir,
                                      self.nectar_config,
                                      self.distribution_report,
                                      self.set_progress)

                with self.update_state(self.progress_status['errata'],
                                       models.Errata.TYPE) as skip:
                    if not (skip or self.skip_repomd_steps):
                        self.get_errata(metadata_files)

                with self.update_state(self.progress_status['comps']) as skip:
                    if not (skip or self.skip_repomd_steps):
                        self.get_comps_file_units(metadata_files,
                                                  group.process_group_element,
                                                  group.GROUP_TAG)
                        self.get_comps_file_units(
                            metadata_files, group.process_category_element,
                            group.CATEGORY_TAG)
                        self.get_comps_file_units(
                            metadata_files, group.process_environment_element,
                            group.ENVIRONMENT_TAG)

            except CancelException:
                report = self.sync_conduit.build_cancel_report(
                    self._progress_summary, self.progress_status)
                report.canceled_flag = True
                return report

            except PulpCodedException, e:
                # Check if the caught exception indicates that the mirror is bad.
                # Try next mirror in the list without raising the exception.
                # In case it was the last mirror in the list, raise the exception.
                bad_mirror_exceptions = [
                    error_codes.RPM1004, error_codes.RPM1006
                ]
                if (e.error_code in bad_mirror_exceptions) and \
                        url_count != len(self.sync_feed):
                    continue
                else:
                    self._set_failed_state(e)
                    raise

            except Exception, e:
                # In case other exceptions were caught that are not related to the state of the
                # mirror, raise the exception immediately and do not iterate throught the rest
                # of the mirrors.
                self._set_failed_state(e)
                report = self.sync_conduit.build_failure_report(
                    self._progress_summary, self.progress_status)
                return report
Пример #3
0
    def run(self):
        """
        Steps through the entire workflow of a repo sync.

        :return:    A SyncReport detailing how the sync went
        :rtype:     pulp.plugins.model.SyncReport
        """
        # using this tmp dir ensures that cleanup leaves nothing behind, since
        # we delete below
        self.tmp_dir = tempfile.mkdtemp(dir=self.working_dir)
        try:
            self.progress_status['metadata']['state'] = constants.STATE_RUNNING
            self.set_progress()

            # Verify that we have a feed url.  if there is no feed url then we have nothing to sync
            if self.sync_feed is None:
                raise PulpCodedException(error_code=error_codes.RPM1005)

            metadata_files = self.get_metadata()
            # Save the default checksum from the metadata
            self.save_default_metadata_checksum_on_repo(metadata_files)

            if self.progress_status['metadata'][
                    'state'] == constants.STATE_RUNNING:
                self.progress_status['metadata'][
                    'state'] = constants.STATE_COMPLETE
            self.set_progress()

            self.content_report['state'] = constants.STATE_RUNNING
            self.set_progress()
            self.update_content(metadata_files)
            if self.content_report['state'] == constants.STATE_RUNNING:
                self.content_report['state'] = constants.STATE_COMPLETE
            self.set_progress()

            _logger.info(_('Downloading additional units.'))
            if models.Distribution.TYPE in self.call_config.get(
                    constants.CONFIG_SKIP, []):
                self.distribution_report['state'] = constants.STATE_SKIPPED
            else:
                self.distribution_report['state'] = constants.STATE_RUNNING
                self.set_progress()
                treeinfo.sync(self.sync_conduit, self.sync_feed, self.tmp_dir,
                              self.nectar_config, self.distribution_report,
                              self.set_progress)
            self.set_progress()

            if models.Errata.TYPE in self.call_config.get(
                    constants.CONFIG_SKIP, []):
                self.progress_status['errata'][
                    'state'] = constants.STATE_SKIPPED
            else:
                self.progress_status['errata'][
                    'state'] = constants.STATE_RUNNING
                self.set_progress()
                self.get_errata(metadata_files)
                self.progress_status['errata'][
                    'state'] = constants.STATE_COMPLETE
            self.set_progress()

            self.progress_status['comps']['state'] = constants.STATE_RUNNING
            self.set_progress()
            self.get_comps_file_units(metadata_files,
                                      group.process_group_element,
                                      group.GROUP_TAG)
            self.get_comps_file_units(metadata_files,
                                      group.process_category_element,
                                      group.CATEGORY_TAG)
            self.get_comps_file_units(metadata_files,
                                      group.process_environment_element,
                                      group.ENVIRONMENT_TAG)
            self.progress_status['comps']['state'] = constants.STATE_COMPLETE
            self.set_progress()

        except CancelException:
            report = self.sync_conduit.build_cancel_report(
                self._progress_summary, self.progress_status)
            report.canceled_flag = True
            return report

        except Exception, e:
            for step, value in self.progress_status.iteritems():
                if value.get('state') == constants.STATE_RUNNING:
                    value['state'] = constants.STATE_FAILED
                    value['error'] = str(e)
            self.set_progress()
            if isinstance(e, PulpCodedException):
                raise
            report = self.sync_conduit.build_failure_report(
                self._progress_summary, self.progress_status)
            return report
Пример #4
0
    def run(self):
        """
        Steps through the entire workflow of a repo sync.

        :return:    A SyncReport detailing how the sync went
        :rtype:     pulp.plugins.model.SyncReport
        """
        # Empty list could be returned in case _parse_as_mirrorlist()
        # was not able to find any valid url
        if self.sync_feed == []:
            raise PulpCodedException(error_code=error_codes.RPM1004, reason='Not found')
        url_count = 0
        for url in self.sync_feed:
            # Verify that we have a feed url.
            # if there is no feed url, then we have nothing to sync
            if url is None:
                raise PulpCodedException(error_code=error_codes.RPM1005)
            # using this tmp dir ensures that cleanup leaves nothing behind, since
            # we delete below
            self.tmp_dir = tempfile.mkdtemp(dir=self.working_dir)
            url_count += 1
            try:
                with self.update_state(self.progress_status['metadata']):
                    metadata_files = self.check_metadata(url)
                    metadata_files = self.get_metadata(metadata_files)

                    # Save the default checksum from the metadata
                    self.save_default_metadata_checksum_on_repo(metadata_files)

                with self.update_state(self.content_report) as skip:
                    if not (skip or self.skip_repomd_steps):
                        self.update_content(metadata_files, url)

                _logger.info(_('Downloading additional units.'))

                with self.update_state(self.distribution_report, models.Distribution.TYPE) as skip:
                    if not skip:
                        treeinfo.sync(self.sync_conduit, url, self.tmp_dir,
                                      self.nectar_config, self.distribution_report,
                                      self.set_progress)

                with self.update_state(self.progress_status['errata'], models.Errata.TYPE) as skip:
                    if not (skip or self.skip_repomd_steps):
                        self.get_errata(metadata_files)

                with self.update_state(self.progress_status['comps']) as skip:
                    if not (skip or self.skip_repomd_steps):
                        self.get_comps_file_units(metadata_files, group.process_group_element,
                                                  group.GROUP_TAG)
                        self.get_comps_file_units(metadata_files, group.process_category_element,
                                                  group.CATEGORY_TAG)
                        self.get_comps_file_units(metadata_files, group.process_environment_element,
                                                  group.ENVIRONMENT_TAG)

            except CancelException:
                report = self.sync_conduit.build_cancel_report(self._progress_summary,
                                                               self.progress_status)
                report.canceled_flag = True
                return report

            except PulpCodedException, e:
                # Check if the caught exception indicates that the mirror is bad.
                # Try next mirror in the list without raising the exception.
                # In case it was the last mirror in the list, raise the exception.
                bad_mirror_exceptions = [error_codes.RPM1004, error_codes.RPM1006]
                if (e.error_code in bad_mirror_exceptions) and \
                        url_count != len(self.sync_feed):
                            continue
                else:
                    self._set_failed_state(e)
                    raise

            except Exception, e:
                # In case other exceptions were caught that are not related to the state of the
                # mirror, raise the exception immediately and do not iterate throught the rest
                # of the mirrors.
                self._set_failed_state(e)
                report = self.sync_conduit.build_failure_report(self._progress_summary,
                                                                self.progress_status)
                return report
Пример #5
0
    def run(self):
        """
        Steps through the entire workflow of a repo sync.

        :return:    A SyncReport detailing how the sync went
        :rtype:     pulp.plugins.model.SyncReport
        """
        # using this tmp dir ensures that cleanup leaves nothing behind, since
        # we delete below
        self.tmp_dir = tempfile.mkdtemp(dir=self.working_dir)
        try:
            self.progress_status['metadata']['state'] = constants.STATE_RUNNING
            self.set_progress()
            metadata_files = self.get_metadata()
            if self.progress_status['metadata']['state'] == constants.STATE_RUNNING:
                self.progress_status['metadata']['state'] = constants.STATE_COMPLETE
            self.set_progress()

            self.content_report['state'] = constants.STATE_RUNNING
            self.set_progress()
            self.update_content(metadata_files)
            if self.content_report['state'] == constants.STATE_RUNNING:
                self.content_report['state'] = constants.STATE_COMPLETE
            self.set_progress()

            if models.Distribution.TYPE in self.call_config.get(constants.CONFIG_SKIP, []):
                self.distribution_report['state'] = constants.STATE_SKIPPED
            else:
                self.distribution_report['state'] = constants.STATE_RUNNING
                self.set_progress()
                treeinfo.sync(self.sync_conduit, self.sync_feed, self.tmp_dir,
                              self.nectar_config, self.distribution_report, self.set_progress)
            self.set_progress()

            if models.Errata.TYPE in self.call_config.get(constants.CONFIG_SKIP, []):
                self.progress_status['errata']['state'] = constants.STATE_SKIPPED
            else:
                self.progress_status['errata']['state'] = constants.STATE_RUNNING
                self.set_progress()
                self.get_errata(metadata_files)
                self.progress_status['errata']['state'] = constants.STATE_COMPLETE
            self.set_progress()

            self.progress_status['comps']['state'] = constants.STATE_RUNNING
            self.set_progress()
            self.get_groups(metadata_files)
            self.get_categories(metadata_files)
            self.progress_status['comps']['state'] = constants.STATE_COMPLETE
            self.set_progress()

        except CancelException:
            report = self.sync_conduit.build_cancel_report(self._progress_summary, self.progress_status)
            report.canceled_flag = True
            return report

        except Exception, e:
            _LOGGER.exception('sync failed')
            for step, value in self.progress_status.iteritems():
                if value.get('state') == constants.STATE_RUNNING:
                    value['state'] = constants.STATE_FAILED
                    value['error'] = str(e)
            self.set_progress()
            report = self.sync_conduit.build_failure_report(self._progress_summary, self.progress_status)
            return report