def run(self): """ Steps through the entire workflow of a repo sync. :return: A SyncReport detailing how the sync went :rtype: pulp.plugins.model.SyncReport """ # using this tmp dir ensures that cleanup leaves nothing behind, since # we delete below self.tmp_dir = tempfile.mkdtemp(dir=self.working_dir) try: self.progress_status['metadata']['state'] = constants.STATE_RUNNING self.set_progress() # Verify that we have a feed url. if there is no feed url then we have nothing to sync if self.sync_feed is None: raise FailedException(_('Unable to sync a repository that has no feed')) metadata_files = self.get_metadata() # Save the default checksum from the metadata self.save_default_metadata_checksum_on_repo(metadata_files) if self.progress_status['metadata']['state'] == constants.STATE_RUNNING: self.progress_status['metadata']['state'] = constants.STATE_COMPLETE self.set_progress() self.content_report['state'] = constants.STATE_RUNNING self.set_progress() self.update_content(metadata_files) if self.content_report['state'] == constants.STATE_RUNNING: self.content_report['state'] = constants.STATE_COMPLETE self.set_progress() _logger.info(_('Downloading additional units.')) if models.Distribution.TYPE in self.call_config.get(constants.CONFIG_SKIP, []): self.distribution_report['state'] = constants.STATE_SKIPPED else: self.distribution_report['state'] = constants.STATE_RUNNING self.set_progress() treeinfo.sync(self.sync_conduit, self.sync_feed, self.tmp_dir, self.nectar_config, self.distribution_report, self.set_progress) self.set_progress() if models.Errata.TYPE in self.call_config.get(constants.CONFIG_SKIP, []): self.progress_status['errata']['state'] = constants.STATE_SKIPPED else: self.progress_status['errata']['state'] = constants.STATE_RUNNING self.set_progress() self.get_errata(metadata_files) self.progress_status['errata']['state'] = constants.STATE_COMPLETE self.set_progress() self.progress_status['comps']['state'] = constants.STATE_RUNNING self.set_progress() self.get_comps_file_units(metadata_files, group.process_group_element, group.GROUP_TAG) self.get_comps_file_units(metadata_files, group.process_category_element, group.CATEGORY_TAG) self.get_comps_file_units(metadata_files, group.process_environment_element, group.ENVIRONMENT_TAG) self.progress_status['comps']['state'] = constants.STATE_COMPLETE self.set_progress() except CancelException: report = self.sync_conduit.build_cancel_report(self._progress_summary, self.progress_status) report.canceled_flag = True return report except Exception, e: _logger.exception('sync failed') for step, value in self.progress_status.iteritems(): if value.get('state') == constants.STATE_RUNNING: value['state'] = constants.STATE_FAILED value['error'] = str(e) self.set_progress() report = self.sync_conduit.build_failure_report(self._progress_summary, self.progress_status) return report
def run(self): """ Steps through the entire workflow of a repo sync. :return: A SyncReport detailing how the sync went :rtype: pulp.plugins.model.SyncReport """ # Empty list could be returned in case _parse_as_mirrorlist() # was not able to find any valid url if self.sync_feed == []: raise PulpCodedException(error_code=error_codes.RPM1004, reason='Not found') url_count = 0 for url in self.sync_feed: # Verify that we have a feed url. # if there is no feed url, then we have nothing to sync if url is None: raise PulpCodedException(error_code=error_codes.RPM1005) # using this tmp dir ensures that cleanup leaves nothing behind, since # we delete below self.tmp_dir = tempfile.mkdtemp(dir=self.working_dir) url_count += 1 try: with self.update_state(self.progress_status['metadata']): metadata_files = self.check_metadata(url) metadata_files = self.get_metadata(metadata_files) # Save the default checksum from the metadata self.save_default_metadata_checksum_on_repo(metadata_files) with self.update_state(self.content_report) as skip: if not (skip or self.skip_repomd_steps): self.update_content(metadata_files, url) _logger.info(_('Downloading additional units.')) with self.update_state(self.distribution_report, models.Distribution.TYPE) as skip: if not skip: treeinfo.sync(self.sync_conduit, url, self.tmp_dir, self.nectar_config, self.distribution_report, self.set_progress) with self.update_state(self.progress_status['errata'], models.Errata.TYPE) as skip: if not (skip or self.skip_repomd_steps): self.get_errata(metadata_files) with self.update_state(self.progress_status['comps']) as skip: if not (skip or self.skip_repomd_steps): self.get_comps_file_units(metadata_files, group.process_group_element, group.GROUP_TAG) self.get_comps_file_units( metadata_files, group.process_category_element, group.CATEGORY_TAG) self.get_comps_file_units( metadata_files, group.process_environment_element, group.ENVIRONMENT_TAG) except CancelException: report = self.sync_conduit.build_cancel_report( self._progress_summary, self.progress_status) report.canceled_flag = True return report except PulpCodedException, e: # Check if the caught exception indicates that the mirror is bad. # Try next mirror in the list without raising the exception. # In case it was the last mirror in the list, raise the exception. bad_mirror_exceptions = [ error_codes.RPM1004, error_codes.RPM1006 ] if (e.error_code in bad_mirror_exceptions) and \ url_count != len(self.sync_feed): continue else: self._set_failed_state(e) raise except Exception, e: # In case other exceptions were caught that are not related to the state of the # mirror, raise the exception immediately and do not iterate throught the rest # of the mirrors. self._set_failed_state(e) report = self.sync_conduit.build_failure_report( self._progress_summary, self.progress_status) return report
def run(self): """ Steps through the entire workflow of a repo sync. :return: A SyncReport detailing how the sync went :rtype: pulp.plugins.model.SyncReport """ # using this tmp dir ensures that cleanup leaves nothing behind, since # we delete below self.tmp_dir = tempfile.mkdtemp(dir=self.working_dir) try: self.progress_status['metadata']['state'] = constants.STATE_RUNNING self.set_progress() # Verify that we have a feed url. if there is no feed url then we have nothing to sync if self.sync_feed is None: raise PulpCodedException(error_code=error_codes.RPM1005) metadata_files = self.get_metadata() # Save the default checksum from the metadata self.save_default_metadata_checksum_on_repo(metadata_files) if self.progress_status['metadata'][ 'state'] == constants.STATE_RUNNING: self.progress_status['metadata'][ 'state'] = constants.STATE_COMPLETE self.set_progress() self.content_report['state'] = constants.STATE_RUNNING self.set_progress() self.update_content(metadata_files) if self.content_report['state'] == constants.STATE_RUNNING: self.content_report['state'] = constants.STATE_COMPLETE self.set_progress() _logger.info(_('Downloading additional units.')) if models.Distribution.TYPE in self.call_config.get( constants.CONFIG_SKIP, []): self.distribution_report['state'] = constants.STATE_SKIPPED else: self.distribution_report['state'] = constants.STATE_RUNNING self.set_progress() treeinfo.sync(self.sync_conduit, self.sync_feed, self.tmp_dir, self.nectar_config, self.distribution_report, self.set_progress) self.set_progress() if models.Errata.TYPE in self.call_config.get( constants.CONFIG_SKIP, []): self.progress_status['errata'][ 'state'] = constants.STATE_SKIPPED else: self.progress_status['errata'][ 'state'] = constants.STATE_RUNNING self.set_progress() self.get_errata(metadata_files) self.progress_status['errata'][ 'state'] = constants.STATE_COMPLETE self.set_progress() self.progress_status['comps']['state'] = constants.STATE_RUNNING self.set_progress() self.get_comps_file_units(metadata_files, group.process_group_element, group.GROUP_TAG) self.get_comps_file_units(metadata_files, group.process_category_element, group.CATEGORY_TAG) self.get_comps_file_units(metadata_files, group.process_environment_element, group.ENVIRONMENT_TAG) self.progress_status['comps']['state'] = constants.STATE_COMPLETE self.set_progress() except CancelException: report = self.sync_conduit.build_cancel_report( self._progress_summary, self.progress_status) report.canceled_flag = True return report except Exception, e: for step, value in self.progress_status.iteritems(): if value.get('state') == constants.STATE_RUNNING: value['state'] = constants.STATE_FAILED value['error'] = str(e) self.set_progress() if isinstance(e, PulpCodedException): raise report = self.sync_conduit.build_failure_report( self._progress_summary, self.progress_status) return report
def run(self): """ Steps through the entire workflow of a repo sync. :return: A SyncReport detailing how the sync went :rtype: pulp.plugins.model.SyncReport """ # Empty list could be returned in case _parse_as_mirrorlist() # was not able to find any valid url if self.sync_feed == []: raise PulpCodedException(error_code=error_codes.RPM1004, reason='Not found') url_count = 0 for url in self.sync_feed: # Verify that we have a feed url. # if there is no feed url, then we have nothing to sync if url is None: raise PulpCodedException(error_code=error_codes.RPM1005) # using this tmp dir ensures that cleanup leaves nothing behind, since # we delete below self.tmp_dir = tempfile.mkdtemp(dir=self.working_dir) url_count += 1 try: with self.update_state(self.progress_status['metadata']): metadata_files = self.check_metadata(url) metadata_files = self.get_metadata(metadata_files) # Save the default checksum from the metadata self.save_default_metadata_checksum_on_repo(metadata_files) with self.update_state(self.content_report) as skip: if not (skip or self.skip_repomd_steps): self.update_content(metadata_files, url) _logger.info(_('Downloading additional units.')) with self.update_state(self.distribution_report, models.Distribution.TYPE) as skip: if not skip: treeinfo.sync(self.sync_conduit, url, self.tmp_dir, self.nectar_config, self.distribution_report, self.set_progress) with self.update_state(self.progress_status['errata'], models.Errata.TYPE) as skip: if not (skip or self.skip_repomd_steps): self.get_errata(metadata_files) with self.update_state(self.progress_status['comps']) as skip: if not (skip or self.skip_repomd_steps): self.get_comps_file_units(metadata_files, group.process_group_element, group.GROUP_TAG) self.get_comps_file_units(metadata_files, group.process_category_element, group.CATEGORY_TAG) self.get_comps_file_units(metadata_files, group.process_environment_element, group.ENVIRONMENT_TAG) except CancelException: report = self.sync_conduit.build_cancel_report(self._progress_summary, self.progress_status) report.canceled_flag = True return report except PulpCodedException, e: # Check if the caught exception indicates that the mirror is bad. # Try next mirror in the list without raising the exception. # In case it was the last mirror in the list, raise the exception. bad_mirror_exceptions = [error_codes.RPM1004, error_codes.RPM1006] if (e.error_code in bad_mirror_exceptions) and \ url_count != len(self.sync_feed): continue else: self._set_failed_state(e) raise except Exception, e: # In case other exceptions were caught that are not related to the state of the # mirror, raise the exception immediately and do not iterate throught the rest # of the mirrors. self._set_failed_state(e) report = self.sync_conduit.build_failure_report(self._progress_summary, self.progress_status) return report
def run(self): """ Steps through the entire workflow of a repo sync. :return: A SyncReport detailing how the sync went :rtype: pulp.plugins.model.SyncReport """ # using this tmp dir ensures that cleanup leaves nothing behind, since # we delete below self.tmp_dir = tempfile.mkdtemp(dir=self.working_dir) try: self.progress_status['metadata']['state'] = constants.STATE_RUNNING self.set_progress() metadata_files = self.get_metadata() if self.progress_status['metadata']['state'] == constants.STATE_RUNNING: self.progress_status['metadata']['state'] = constants.STATE_COMPLETE self.set_progress() self.content_report['state'] = constants.STATE_RUNNING self.set_progress() self.update_content(metadata_files) if self.content_report['state'] == constants.STATE_RUNNING: self.content_report['state'] = constants.STATE_COMPLETE self.set_progress() if models.Distribution.TYPE in self.call_config.get(constants.CONFIG_SKIP, []): self.distribution_report['state'] = constants.STATE_SKIPPED else: self.distribution_report['state'] = constants.STATE_RUNNING self.set_progress() treeinfo.sync(self.sync_conduit, self.sync_feed, self.tmp_dir, self.nectar_config, self.distribution_report, self.set_progress) self.set_progress() if models.Errata.TYPE in self.call_config.get(constants.CONFIG_SKIP, []): self.progress_status['errata']['state'] = constants.STATE_SKIPPED else: self.progress_status['errata']['state'] = constants.STATE_RUNNING self.set_progress() self.get_errata(metadata_files) self.progress_status['errata']['state'] = constants.STATE_COMPLETE self.set_progress() self.progress_status['comps']['state'] = constants.STATE_RUNNING self.set_progress() self.get_groups(metadata_files) self.get_categories(metadata_files) self.progress_status['comps']['state'] = constants.STATE_COMPLETE self.set_progress() except CancelException: report = self.sync_conduit.build_cancel_report(self._progress_summary, self.progress_status) report.canceled_flag = True return report except Exception, e: _LOGGER.exception('sync failed') for step, value in self.progress_status.iteritems(): if value.get('state') == constants.STATE_RUNNING: value['state'] = constants.STATE_FAILED value['error'] = str(e) self.set_progress() report = self.sync_conduit.build_failure_report(self._progress_summary, self.progress_status) return report