def setUp(self): super(Migration0004Tests, self).setUp() # Special way to import modules that start with a number self.migration = _import_all_the_way( 'pulp_rpm.plugins.migrations.0004_pkg_group_category_repoid') factory.initialize() types_db.update_database([TYPE_DEF_GROUP, TYPE_DEF_CATEGORY]) # Create the repositories necessary for the tests self.source_repo_id = 'source-repo' # where units were copied from with the bad code self.dest_repo_id = 'dest-repo' # where bad units were copied to source_repo = Repo(self.source_repo_id, '') Repo.get_collection().insert(source_repo, safe=True) dest_repo = Repo(self.dest_repo_id, '') Repo.get_collection().insert(dest_repo, safe=True) source_importer = RepoImporter(self.source_repo_id, 'yum_importer', 'yum_importer', {}) RepoImporter.get_collection().insert(source_importer, safe=True) dest_importer = RepoImporter(self.dest_repo_id, 'yum_importer', 'yum_importer', {}) RepoImporter.get_collection().insert(dest_importer, safe=True)
def set_importer(repo_id, importer_type_id, repo_plugin_config): """ Configures an importer to be used for the given repository. Keep in mind this method is written assuming single importer for a repo. The domain model technically supports multiple importers, but this call is what enforces the single importer behavior. :param repo_id: identifies the repo :type repo_id: str :param importer_type_id: identifies the type of importer being added; must correspond to an importer loaded at server startup :type importer_type_id: str :param repo_plugin_config: configuration values for the importer; may be None :type repo_plugin_config: dict :raise MissingResource: if repo_id does not represent a valid repo :raise InvalidImporterConfiguration: if the importer cannot be initialized for the given repo """ RepoImporterManager.validate_importer_config(repo_id, importer_type_id, repo_plugin_config) importer_coll = RepoImporter.get_collection() repo_obj = model.Repository.objects.get_repo_or_missing_resource(repo_id) importer_instance, plugin_config = plugin_api.get_importer_by_id(importer_type_id) # Convention is that a value of None means unset. Remove any keys that # are explicitly set to None so the plugin will default them. if repo_plugin_config is not None: clean_config = dict([(k, v) for k, v in repo_plugin_config.items() if v is not None]) else: clean_config = None # Let the importer plugin verify the configuration call_config = PluginCallConfiguration(plugin_config, clean_config) transfer_repo = repo_obj.to_transfer_repo() # Remove old importer if one exists try: RepoImporterManager.remove_importer(repo_id) except MissingResource: pass # it didn't exist, so no harm done # Let the importer plugin initialize the repository try: importer_instance.importer_added(transfer_repo, call_config) except Exception: _logger.exception( 'Error initializing importer [%s] for repo [%s]' % (importer_type_id, repo_id)) raise PulpExecutionException(), None, sys.exc_info()[2] # Database Update importer_id = importer_type_id # use the importer name as its repo ID importer = RepoImporter(repo_id, importer_id, importer_type_id, clean_config) importer_coll.save(importer) return importer
class RepoImporterManager(object): def get_importer(self, repo_id): """ Returns metadata about an importer associated with the given repo. @return: key-value pairs describing the importer in use @rtype: dict @raise MissingResource: if the repo does not exist or has no importer associated """ importer = RepoImporter.get_collection().find_one({'repo_id': repo_id}) if importer is None: raise MissingResource(repository=repo_id) return importer def get_importers(self, repo_id): """ Returns a list of all importers associated with the given repo. @return: list of key-value pairs describing the importers in use; empty list if the repo has no importers @rtype: list of dict @raise MissingResource: if the given repo doesn't exist """ repo = Repo.get_collection().find_one({'id': repo_id}) if repo is None: raise MissingResource(repo_id) importers = list(RepoImporter.get_collection().find( {'repo_id': repo_id})) return importers @staticmethod def find_by_repo_list(repo_id_list): """ Returns serialized versions of all importers for given repos. Any IDs that do not refer to valid repos are ignored and will not raise an error. @param repo_id_list: list of importer IDs to fetch @type repo_id_list: list of str @return: list of serialized importers @rtype: list of dict """ spec = {'repo_id': {'$in': repo_id_list}} projection = {'scratchpad': 0} importers = list(RepoImporter.get_collection().find(spec, projection)) # Process any scheduled syncs and get schedule details using schedule id for importer in importers: scheduled_sync_ids = importer.get('scheduled_syncs', None) if scheduled_sync_ids is not None: scheduled_sync_details = list( ScheduledCall.get_collection().find( {"id": { "$in": scheduled_sync_ids }})) importer['scheduled_syncs'] = [ s["schedule"] for s in scheduled_sync_details ] return importers def set_importer(self, repo_id, importer_type_id, repo_plugin_config): """ Configures an importer to be used for the given repository. Keep in mind this method is written assuming single importer for a repo. The domain model technically supports multiple importers, but this call is what enforces the single importer behavior. @param repo_id: identifies the repo @type repo_id; str @param importer_type_id: identifies the type of importer being added; must correspond to an importer loaded at server startup @type importer_type_id: str @param repo_plugin_config: configuration values for the importer; may be None @type repo_plugin_config: dict @raise MissingResource: if repo_id does not represent a valid repo @raise InvalidImporterConfiguration: if the importer cannot be initialized for the given repo """ repo_coll = Repo.get_collection() importer_coll = RepoImporter.get_collection() # Validation repo = repo_coll.find_one({'id': repo_id}) if repo is None: raise MissingResource(repo_id) if not plugin_api.is_valid_importer(importer_type_id): raise InvalidValue(['importer_type_id']) importer_instance, plugin_config = plugin_api.get_importer_by_id( importer_type_id) # Convention is that a value of None means unset. Remove any keys that # are explicitly set to None so the plugin will default them. if repo_plugin_config is not None: clean_config = dict([(k, v) for k, v in repo_plugin_config.items() if v is not None]) else: clean_config = None # Let the importer plugin verify the configuration call_config = PluginCallConfiguration(plugin_config, clean_config) transfer_repo = common_utils.to_transfer_repo(repo) transfer_repo.working_dir = common_utils.importer_working_dir( importer_type_id, repo_id) query_manager = manager_factory.repo_query_manager() related_repos = query_manager.find_with_importer_type(importer_type_id) transfer_related_repos = [] for r in related_repos: all_configs = [d['config'] for d in r['importers']] trr = common_utils.to_related_repo(r, all_configs) transfer_related_repos.append(trr) try: result = importer_instance.validate_config(transfer_repo, call_config, transfer_related_repos) # For backward compatibility with plugins that don't yet return the tuple if isinstance(result, bool): valid_config = result message = None else: valid_config, message = result except Exception, e: _LOG.exception( 'Exception received from importer [%s] while validating config' % importer_type_id) raise PulpDataException(e.args), None, sys.exc_info()[2] if not valid_config: raise PulpDataException(message) # Remove old importer if one exists try: self.remove_importer(repo_id) except MissingResource: pass # it didn't exist, so no harm done # Let the importer plugin initialize the repository try: importer_instance.importer_added(transfer_repo, call_config) except Exception: _LOG.exception('Error initializing importer [%s] for repo [%s]' % (importer_type_id, repo_id)) raise PulpExecutionException(), None, sys.exc_info()[2] # Database Update importer_id = importer_type_id # use the importer name as its repo ID importer = RepoImporter(repo_id, importer_id, importer_type_id, clean_config) importer_coll.save(importer, safe=True) return importer