def get(self, request, repo_id): """ Looks for query parameters 'importers' and 'distributors', and will add the corresponding fields to the repository returned. Query parameter 'details' is equivalent to passing both 'importers' and 'distributors'. :param request: WSGI request object :type request: django.core.handlers.wsgi.WSGIRequest :param repo_id: id of requested repository :type repo_id: str :return: Response containing a serialized dict for the requested repo. :rtype : django.http.HttpResponse :raises exceptions.MissingResource: if repo cannot be found """ repo_obj = model.Repository.objects.get_repo_or_missing_resource(repo_id) repo = serializers.Repository(repo_obj).data # Add importers and distributors to the dicts if requested. details = request.GET.get('details', 'false').lower() == 'true' if request.GET.get('importers', 'false').lower() == 'true' or details: _merge_related_objects('importers', model.Importer, (repo,)) if request.GET.get('distributors', 'false').lower() == 'true' or details: _merge_related_objects('distributors', model.Distributor, (repo,)) if details: repo['total_repository_units'] = sum(repo['content_unit_counts'].itervalues()) total_missing = repo_controller.missing_unit_count(repo_obj.repo_id) repo['locally_stored_units'] = repo['total_repository_units'] - total_missing return generate_json_response_with_pulp_encoder(repo)
def get_metadata(self, metadata_files): """ :param metadata_files: instance of MetadataFiles :type: pulp_rpm.plugins.importers.yum.repomd.metadata.MetadataFiles :return: instance of MetadataFiles where each relevant file has been identified and downloaded. :rtype: pulp_rpm.plugins.importers.yum.repomd.metadata.MetadataFiles """ self.downloader = metadata_files.downloader scratchpad = self.conduit.get_scratchpad() or {} previous_revision = scratchpad.get(constants.REPOMD_REVISION_KEY, 0) previous_skip_set = set(scratchpad.get(constants.PREVIOUS_SKIP_LIST, [])) current_skip_set = set(self.config.get(constants.CONFIG_SKIP, [])) self.current_revision = metadata_files.revision # if the revision is positive, hasn't increased and the skip list doesn't include # new types that weren't present on the last run... # determine missing units missing_units = repo_controller.missing_unit_count(self.repo.repo_id) if 0 < metadata_files.revision <= previous_revision \ and previous_skip_set - current_skip_set == set() \ and (self.download_deferred or not missing_units): _logger.info(_('upstream repo metadata has not changed. Skipping steps.')) self.skip_repomd_steps = True return metadata_files else: _logger.info(_('Downloading metadata files.')) metadata_files.download_metadata_files() self.downloader = None _logger.info(_('Generating metadata databases.')) metadata_files.generate_dbs() self.import_unknown_metadata_files(metadata_files) return metadata_files
def get_metadata(self, metadata_files): """ Get metadata and decide whether to sync the repository or not. :param metadata_files: instance of MetadataFiles :type: pulp_rpm.plugins.importers.yum.repomd.metadata.MetadataFiles :return: instance of MetadataFiles where each relevant file has been identified and downloaded. :rtype: pulp_rpm.plugins.importers.yum.repomd.metadata.MetadataFiles """ self.downloader = metadata_files.downloader scratchpad = self.conduit.get_scratchpad() or {} previous_revision = scratchpad.get(constants.REPOMD_REVISION_KEY, 0) previous_skip_set = set(scratchpad.get(constants.PREVIOUS_SKIP_LIST, [])) current_skip_set = set(self.config.get(constants.CONFIG_SKIP, [])) self.current_revision = metadata_files.revision last_sync = self.conduit.last_sync() sync_due_to_unit_removal = False if last_sync is not None: last_sync = dateutils.parse_iso8601_datetime(last_sync) last_removed = self.repo.last_unit_removed sync_due_to_unit_removal = last_removed is not None and last_sync < last_removed # determine missing units missing_units = repo_controller.missing_unit_count(self.repo.repo_id) # if the current MD revision is not newer than the old one # and we aren't using an override URL # and the skip list doesn't have any new types # and there are no missing units, or we have deferred download enabled # and no units were removed after last sync # then skip fetching the repo MD :) if 0 < metadata_files.revision <= previous_revision \ and not self.config.override_config.get(importer_constants.KEY_FEED) \ and previous_skip_set - current_skip_set == set() \ and (self.download_deferred or not missing_units) \ and not sync_due_to_unit_removal: _logger.info(_('upstream repo metadata has not changed. Skipping steps.')) self.skip_repomd_steps = True return metadata_files else: _logger.info(_('Downloading metadata files.')) metadata_files.download_metadata_files() self.downloader = None _logger.info(_('Generating metadata databases.')) metadata_files.generate_dbs() self.import_unknown_metadata_files(metadata_files) return metadata_files
def get_metadata(self, metadata_files): """ Get metadata and decide whether to sync the repository or not. :param metadata_files: instance of MetadataFiles :type: pulp_rpm.plugins.importers.yum.repomd.metadata.MetadataFiles :return: instance of MetadataFiles where each relevant file has been identified and downloaded. :rtype: pulp_rpm.plugins.importers.yum.repomd.metadata.MetadataFiles """ self.downloader = metadata_files.downloader scratchpad = self.conduit.get_scratchpad() or {} previous_revision = scratchpad.get(constants.REPOMD_REVISION_KEY, 0) self.current_revision = metadata_files.revision # determine missing units missing_units = repo_controller.missing_unit_count(self.repo.repo_id) force_full_sync = repo_controller.check_perform_full_sync(self.repo.repo_id, self.conduit, self.config) # if the platform does not prescribe forcing a full sync # (due to removed unit, force_full flag, config change, etc.) # the current MD revision is not newer than the old one # and there are no missing units, or we have deferred download enabled # then skip fetching the repo MD :) skip_sync_steps = not force_full_sync and \ 0 < self.current_revision <= previous_revision and \ (self.download_deferred or not missing_units) if skip_sync_steps: _logger.info(_('upstream repo metadata has not changed. Skipping steps.')) self.skip_repomd_steps = True return metadata_files else: _logger.info(_('Downloading metadata files.')) metadata_files.download_metadata_files() self.downloader = None _logger.info(_('Generating metadata databases.')) metadata_files.generate_dbs() self.import_unknown_metadata_files(metadata_files) return metadata_files
def get_metadata(self, metadata_files): """ :param metadata_files: instance of MetadataFiles :type: pulp_rpm.plugins.importers.yum.repomd.metadata.MetadataFiles :return: instance of MetadataFiles where each relevant file has been identified and downloaded. :rtype: pulp_rpm.plugins.importers.yum.repomd.metadata.MetadataFiles """ self.downloader = metadata_files.downloader scratchpad = self.conduit.get_scratchpad() or {} previous_revision = scratchpad.get(constants.REPOMD_REVISION_KEY, 0) previous_skip_set = set( scratchpad.get(constants.PREVIOUS_SKIP_LIST, [])) current_skip_set = set(self.config.get(constants.CONFIG_SKIP, [])) self.current_revision = metadata_files.revision # if the revision is positive, hasn't increased and the skip list doesn't include # new types that weren't present on the last run... # determine missing units missing_units = repo_controller.missing_unit_count(self.repo.repo_id) if 0 < metadata_files.revision <= previous_revision \ and previous_skip_set - current_skip_set == set() \ and (self.download_deferred or not missing_units): _logger.info( _('upstream repo metadata has not changed. Skipping steps.')) self.skip_repomd_steps = True return metadata_files else: _logger.info(_('Downloading metadata files.')) metadata_files.download_metadata_files() self.downloader = None _logger.info(_('Generating metadata databases.')) metadata_files.generate_dbs() self.import_unknown_metadata_files(metadata_files) return metadata_files
def _run(self, tmp_dir): """ Look for a distribution in the target repo and sync it if found :param tmp_dir: The absolute path to the temporary directory :type tmp_dir: str """ treeinfo_path = self.get_treefile(tmp_dir) if not treeinfo_path: _logger.debug(_('No treeinfo found')) return try: unit, files = self.parse_treeinfo_file(treeinfo_path) except ValueError: _logger.error(_('could not parse treeinfo')) self.progress_report['state'] = constants.STATE_FAILED return existing_units = repo_controller.find_repo_content_units( self.repo, repo_content_unit_q=Q(unit_type_id=ids.TYPE_ID_DISTRO), yield_content_unit=True) existing_units = list(existing_units) # determine missing units missing_units = repo_controller.missing_unit_count(self.repo.repo_id) # Continue only when the distribution has changed. if len(existing_units) == 1 and \ self.existing_distribution_is_current(existing_units[0], unit) and \ (self.download_deferred or not missing_units): _logger.info(_('upstream distribution unchanged; skipping')) return # Process the distribution dist_files, pulp_dist_xml_path = self.process_distribution(tmp_dir) files.extend(dist_files) self.update_unit_files(unit, files) # Download distribution files if not self.download_deferred: try: downloaded = self.download_files(tmp_dir, files) except DownloadFailed: # All files must be downloaded to continue. return else: unit.downloaded = False downloaded = [] # Save the unit. unit.save() # Update deferred downloading catalog self.update_catalog_entries(unit, files) # The treeinfo and PULP_DISTRIBTION.xml files are always imported into platform # # storage regardless of the download policy unit.safe_import_content(treeinfo_path, os.path.basename(treeinfo_path)) if pulp_dist_xml_path is not None: unit.safe_import_content(pulp_dist_xml_path, os.path.basename(pulp_dist_xml_path)) # The downloaded files are imported into platform storage. if downloaded: for destination, location in downloaded: unit.safe_import_content(destination, location) else: if not unit.downloaded: unit.downloaded = True unit.save() # Associate the unit. repo_controller.associate_single_unit(self.repo, unit) # find any old distribution units and remove them. See BZ #1150714 for existing_unit in existing_units: if existing_unit == unit: continue msg = _('Removing out-of-date distribution unit {k} for repo {r}') _logger.info( msg.format(k=existing_unit.unit_key, r=self.repo.repo_id)) qs = RepositoryContentUnit.objects.filter( repo_id=self.repo.repo_id, unit_id=existing_unit.id) qs.delete()
def _run(self, tmp_dir): """ Look for a distribution in the target repo and sync it if found :param tmp_dir: The absolute path to the temporary directory :type tmp_dir: str """ treeinfo_path = self.get_treefile(tmp_dir) if not treeinfo_path: _logger.debug(_('No treeinfo found')) return self.metadata_found = True try: unit, files = self.parse_treeinfo_file(treeinfo_path) except ValueError: _logger.error(_('could not parse treeinfo')) self.progress_report['state'] = constants.STATE_FAILED return existing_units = repo_controller.find_repo_content_units( self.repo, repo_content_unit_q=Q(unit_type_id=ids.TYPE_ID_DISTRO), yield_content_unit=True) existing_units = list(existing_units) # determine missing units missing_units = repo_controller.missing_unit_count(self.repo.repo_id) # Continue only when the distribution has changed. if len(existing_units) == 1 and \ self.existing_distribution_is_current(existing_units[0], unit) and \ (self.download_deferred or not missing_units): _logger.info(_('upstream distribution unchanged; skipping')) return # Process the distribution dist_files, pulp_dist_xml_path = self.process_distribution(tmp_dir) files.extend(dist_files) self.update_unit_files(unit, files) # Download distribution files if not self.download_deferred: try: downloaded = self.download_files(tmp_dir, files) except DownloadFailed: # All files must be downloaded to continue. return else: unit.downloaded = False downloaded = [] # Save the unit. unit.save() # Update deferred downloading catalog self.update_catalog_entries(unit, files) # The treeinfo and PULP_DISTRIBTION.xml files are always imported into platform # # storage regardless of the download policy unit.safe_import_content(treeinfo_path, os.path.basename(treeinfo_path)) if pulp_dist_xml_path is not None: unit.safe_import_content(pulp_dist_xml_path, os.path.basename(pulp_dist_xml_path)) # The downloaded files are imported into platform storage. if downloaded: for destination, location in downloaded: unit.safe_import_content(destination, location) else: if not unit.downloaded: unit.downloaded = True unit.save() # Associate the unit. repo_controller.associate_single_unit(self.repo, unit) # find any old distribution units and remove them. See BZ #1150714 for existing_unit in existing_units: if existing_unit == unit: continue msg = _('Removing out-of-date distribution unit {k} for repo {r}') _logger.info(msg.format(k=existing_unit.unit_key, r=self.repo.repo_id)) qs = RepositoryContentUnit.objects.filter( repo_id=self.repo.repo_id, unit_id=existing_unit.id) qs.delete()