def update_importer_config(repo_id, importer_config): """ Attempts to update the saved configuration for the given repo's importer. The importer will be asked if the new configuration is valid. If not, this method will raise an error and the existing configuration will remain unchanged. :param repo_id: identifies the repo :type repo_id: str :param importer_config: new configuration values to use for this repo :type importer_config: dict """ repo_importer = model.Importer.objects.get_or_404(repo_id=repo_id) importer_instance, plugin_config = plugin_api.get_importer_by_id(repo_importer.importer_type_id) validate_importer_config(repo_id, repo_importer.importer_type_id, plugin_config) # The convention is that None in an update removes the value and sets it to the default. unset_property_names = [k for k in importer_config if importer_config[k] is None] for key in unset_property_names: repo_importer.config.pop(key, None) importer_config.pop(key, None) # Whatever is left over are the changed/added values, so merge them in. repo_importer.config.update(importer_config) try: repo_importer.save() except ValidationError, e: raise exceptions.InvalidValue(e.to_dict().keys())
def remove_importer(repo_id): """ Removes an importer from a repository. :param repo_id: identifies the repo :type repo_id: str :raise MissingResource: if the given repo does not exist :raise MissingResource: if the given repo does not have an importer """ importer_coll = RepoImporter.get_collection() # Validation repo_obj = model.Repository.objects.get_repo_or_missing_resource(repo_id) repo_importer = importer_coll.find_one({"repo_id": repo_id}) if repo_importer is None: raise MissingResource(repo_id) # remove schedules RepoSyncScheduleManager().delete_by_importer_id(repo_id, repo_importer["id"]) # Call the importer's cleanup method importer_type_id = repo_importer["importer_type_id"] importer_instance, plugin_config = plugin_api.get_importer_by_id(importer_type_id) call_config = PluginCallConfiguration(plugin_config, repo_importer["config"]) transfer_repo = repo_obj.to_transfer_repo() importer_instance.importer_removed(transfer_repo, call_config) # Update the database to reflect the removal importer_coll.remove({"repo_id": repo_id})
def remove_from_importer(repo_id, removed_units): # Retrieve the repo from the database and convert to the transfer repo repo_query_manager = manager_factory.repo_query_manager() repo = repo_query_manager.get_repository(repo_id) importer_manager = manager_factory.repo_importer_manager() repo_importer = importer_manager.get_importer(repo_id) transfer_repo = common_utils.to_transfer_repo(repo) transfer_repo.working_dir = common_utils.importer_working_dir(repo_importer['importer_type_id'], repo_id, mkdir=True) # Convert the units into transfer units unit_type_ids = calculate_associated_type_ids(repo_id, removed_units) transfer_units = create_transfer_units(removed_units, unit_type_ids) # Retrieve the plugin instance to invoke importer_instance, plugin_config = plugin_api.get_importer_by_id(repo_importer['importer_type_id']) call_config = PluginCallConfiguration(plugin_config, repo_importer['config']) # Invoke the importer's remove method try: importer_instance.remove_units(transfer_repo, transfer_units, call_config) except Exception: _LOG.exception('Exception from importer [%s] while removing units from repo [%s]' % (repo_importer['id'], repo_id))
def sync(repo_id, sync_config_override=None): """ Performs a synchronize operation on the given repository and triggers publishs for distributors with autopublish enabled. The given repo must have an importer configured. This method is intentionally limited to synchronizing a single repo. Performing multiple repository syncs concurrently will require a more global view of the server and must be handled outside the scope of this class. :param repo_id: identifies the repo to sync :type repo_id: str :param sync_config_override: optional config containing values to use for this sync only :type sync_config_override: dict :return: TaskResult containing sync results and a list of spawned tasks :rtype: pulp.server.async.tasks.TaskResult :raise pulp_exceptions.MissingResource: if specified repo does not exist, or it does not have an importer and associated plugin :raise pulp_exceptions.PulpExecutionException: if the task fails. """ repo_obj = model.Repository.objects.get_repo_or_missing_resource(repo_id) transfer_repo = repo_obj.to_transfer_repo() importer_collection = RepoImporter.get_collection() repo_importer = importer_collection.find_one({'repo_id': repo_obj.repo_id}) if repo_importer is None: raise pulp_exceptions.MissingResource(repository=repo_id) try: importer, imp_config = plugin_api.get_importer_by_id(repo_importer['importer_type_id']) except plugin_exceptions.PluginNotFound: raise pulp_exceptions.MissingResource(repository=repo_id) call_config = PluginCallConfiguration(imp_config, repo_importer['config'], sync_config_override) transfer_repo.working_dir = common_utils.get_working_directory() conduit = RepoSyncConduit(repo_id, repo_importer['id']) sync_result_collection = RepoSyncResult.get_collection() # Fire an events around the call fire_manager = manager_factory.event_fire_manager() fire_manager.fire_repo_sync_started(repo_id) # Perform the sync sync_start_timestamp = _now_timestamp() sync_result = None try: # Replace the Importer's sync_repo() method with our register_sigterm_handler decorator, # which will set up cancel_sync_repo() as the target for the signal handler sync_repo = register_sigterm_handler(importer.sync_repo, importer.cancel_sync_repo) sync_report = sync_repo(transfer_repo, conduit, call_config) except Exception, e: sync_end_timestamp = _now_timestamp() sync_result = RepoSyncResult.error_result( repo_obj.repo_id, repo_importer['id'], repo_importer['importer_type_id'], sync_start_timestamp, sync_end_timestamp, e, sys.exc_info()[2]) raise
def remove_from_importer(repo_id, transfer_units): # Retrieve the repo from the database and convert to the transfer repo repo_query_manager = manager_factory.repo_query_manager() repo = repo_query_manager.get_repository(repo_id) importer_manager = manager_factory.repo_importer_manager() repo_importer = importer_manager.get_importer(repo_id) transfer_repo = common_utils.to_transfer_repo(repo) transfer_repo.working_dir = common_utils.importer_working_dir( repo_importer['importer_type_id'], repo_id, mkdir=True) # Retrieve the plugin instance to invoke importer_instance, plugin_config = plugin_api.get_importer_by_id( repo_importer['importer_type_id']) call_config = PluginCallConfiguration(plugin_config, repo_importer['config']) # Invoke the importer's remove method try: importer_instance.remove_units(transfer_repo, transfer_units, call_config) except Exception: msg = _( 'Exception from importer [%(i)s] while removing units from repo [%(r)s]' ) msg = msg % {'i': repo_importer['id'], 'r': repo_id} logger.exception(msg)
def remove_from_importer(repo_id, transfer_units): # Retrieve the repo from the database and convert to the transfer repo repo_query_manager = manager_factory.repo_query_manager() repo = repo_query_manager.get_repository(repo_id) importer_manager = manager_factory.repo_importer_manager() repo_importer = importer_manager.get_importer(repo_id) transfer_repo = common_utils.to_transfer_repo(repo) transfer_repo.working_dir = common_utils.importer_working_dir(repo_importer['importer_type_id'], repo_id, mkdir=True) # Retrieve the plugin instance to invoke importer_instance, plugin_config = plugin_api.get_importer_by_id( repo_importer['importer_type_id']) call_config = PluginCallConfiguration(plugin_config, repo_importer['config']) # Invoke the importer's remove method try: importer_instance.remove_units(transfer_repo, transfer_units, call_config) except Exception: msg = _('Exception from importer [%(i)s] while removing units from repo [%(r)s]') msg = msg % {'i': repo_importer['id'], 'r': repo_id} logger.exception(msg)
def validate_importer_config(repo_obj, importer_type_id, config): """ Validates the importer configuration. :param repo_obj: repository object :type repo_obj: pulp.server.db.model.Repository :param importer_type_id: type of importer, must correspond to a plugin loaded at server startup :type importer_type_id: str :param config: configuration values for the importer :type config: dict :raises PulpCodedValidationException: if importer_type_id is invalid :raises exceptions.PulpDataException: if config is invalid. """ if not plugin_api.is_valid_importer(importer_type_id): raise exceptions.PulpCodedValidationException(error_code=error_codes.PLP1008, importer_type_id=importer_type_id) importer_instance, plugin_config = plugin_api.get_importer_by_id(importer_type_id) call_config = PluginCallConfiguration(plugin_config, config) transfer_repo = repo_obj.to_transfer_repo() result = importer_instance.validate_config(transfer_repo, call_config) # For backward compatibility with plugins that don't yet return the tuple if isinstance(result, bool): valid_config = result message = None else: valid_config, message = result if not valid_config: raise exceptions.PulpDataException(message)
def remove_importer(repo_id): """ Removes an importer from a repository. :param repo_id: identifies the repo :type repo_id: str :raise MissingResource: if the given repo does not exist :raise MissingResource: if the given repo does not have an importer """ importer_coll = RepoImporter.get_collection() # Validation repo_obj = model.Repository.objects.get_repo_or_missing_resource(repo_id) repo_importer = importer_coll.find_one({'repo_id': repo_id}) if repo_importer is None: raise MissingResource(repo_id) # remove schedules RepoSyncScheduleManager().delete_by_importer_id(repo_id, repo_importer['id']) # Call the importer's cleanup method importer_type_id = repo_importer['importer_type_id'] importer_instance, plugin_config = plugin_api.get_importer_by_id(importer_type_id) call_config = PluginCallConfiguration(plugin_config, repo_importer['config']) transfer_repo = repo_obj.to_transfer_repo() importer_instance.importer_removed(transfer_repo, call_config) # Update the database to reflect the removal importer_coll.remove({'repo_id': repo_id})
def validate_importer_config(repo_id, importer_type_id, importer_config): """ This validates that the repository and importer type exist as these are both required to validate the configuration. :param repo_id: identifies the repo :type repo_id: str :param importer_type_id: type of importer, must correspond to a plugin loaded at server startup :type importer_type_id: str :param importer_config: configuration values for the importer; may be None :type importer_config: dict :raises exceptions.PulpCodedValidationException: if config is invalid. """ repo_obj = model.Repository.objects.get_repo_or_missing_resource(repo_id) if not plugin_api.is_valid_importer(importer_type_id): raise exceptions.PulpCodedValidationException(error_code=error_codes.PLP1008, importer_type_id=importer_type_id) importer_instance, plugin_config = plugin_api.get_importer_by_id(importer_type_id) clean_config = clean_config_dict(importer_config) call_config = PluginCallConfiguration(plugin_config, clean_config) transfer_repo = repo_obj.to_transfer_repo() result = importer_instance.validate_config(transfer_repo, call_config) # For backward compatibility with plugins that don't yet return the tuple if isinstance(result, bool): valid_config = result message = None else: valid_config, message = result if not valid_config: raise exceptions.PulpCodedValidationException(validation_errors=message)
def set_importer(repo_id, importer_type_id, repo_plugin_config): """ Configures an importer to be used for the given repository. Keep in mind this method is written assuming single importer for a repo. The domain model technically supports multiple importers, but this call is what enforces the single importer behavior. :param repo_id: identifies the repo :type repo_id: str :param importer_type_id: identifies the type of importer being added; must correspond to an importer loaded at server startup :type importer_type_id: str :param repo_plugin_config: configuration values for the importer; may be None :type repo_plugin_config: dict :raise MissingResource: if repo_id does not represent a valid repo :raise InvalidImporterConfiguration: if the importer cannot be initialized for the given repo """ RepoImporterManager.validate_importer_config(repo_id, importer_type_id, repo_plugin_config) repo_coll = Repo.get_collection() importer_coll = RepoImporter.get_collection() repo = repo_coll.find_one({'id': repo_id}) importer_instance, plugin_config = plugin_api.get_importer_by_id(importer_type_id) # Convention is that a value of None means unset. Remove any keys that # are explicitly set to None so the plugin will default them. if repo_plugin_config is not None: clean_config = dict([(k, v) for k, v in repo_plugin_config.items() if v is not None]) else: clean_config = None # Let the importer plugin verify the configuration call_config = PluginCallConfiguration(plugin_config, clean_config) transfer_repo = common_utils.to_transfer_repo(repo) transfer_repo.working_dir = common_utils.importer_working_dir(importer_type_id, repo_id) # Remove old importer if one exists try: RepoImporterManager.remove_importer(repo_id) except MissingResource: pass # it didn't exist, so no harm done # Let the importer plugin initialize the repository try: importer_instance.importer_added(transfer_repo, call_config) except Exception: _logger.exception( 'Error initializing importer [%s] for repo [%s]' % (importer_type_id, repo_id)) raise PulpExecutionException(), None, sys.exc_info()[2] # Database Update importer_id = importer_type_id # use the importer name as its repo ID importer = RepoImporter(repo_id, importer_id, importer_type_id, clean_config) importer_coll.save(importer, safe=True) return importer
def resolve_dependencies_by_units(repo_id, units, options): """ Calculates dependencies for the given set of units in the given repository. :param repo_id: identifies the repository :type repo_id: str :param units: list of database representations of units to resolve dependencies for :type units: list :param options: dict of options to pass the importer to drive the resolution :type options: dict or None :return: report from the plugin :rtype: object :raise MissingResource: if the repo does not exist or does not have an importer """ # Validation repo_query_manager = manager_factory.repo_query_manager() importer_manager = manager_factory.repo_importer_manager() # The following will raise MissingResource as appropriate repo = repo_query_manager.get_repository(repo_id) repo_importer = importer_manager.get_importer(repo_id) try: importer_instance, plugin_config = plugin_api.get_importer_by_id( repo_importer['importer_type_id']) except plugin_exceptions.PluginNotFound: raise MissingResource(repo_id), None, sys.exc_info()[2] # Package for the importer call call_config = PluginCallConfiguration(plugin_config, repo_importer['config'], options) transfer_repo = common_utils.to_transfer_repo(repo) transfer_repo.working_dir = common_utils.importer_working_dir( repo_importer['importer_type_id'], repo_id, mkdir=True) conduit = DependencyResolutionConduit(repo_id, repo_importer['id']) # Convert all of the units into the plugin standard representation transfer_units = [] # Preload all the type defs so we don't hammer the database unnecessarily type_defs = {} all_type_def_ids = set([u['unit_type_id'] for u in units]) for def_id in all_type_def_ids: type_def = types_db.type_definition(def_id) type_defs[def_id] = type_def for unit in units: type_id = unit['unit_type_id'] u = conduit_common_utils.to_plugin_associated_unit(unit, type_defs[type_id]) transfer_units.append(u) # Invoke the importer try: dep_report = importer_instance.resolve_dependencies(transfer_repo, transfer_units, conduit, call_config) except Exception, e: raise PulpExecutionException(), None, sys.exc_info()[2]
def set_importer(repo_id, importer_type_id, repo_plugin_config): """ Configures an importer to be used for the given repository. Keep in mind this method is written assuming single importer for a repo. The domain model technically supports multiple importers, but this call is what enforces the single importer behavior. :param repo_id: identifies the repo :type repo_id: str :param importer_type_id: identifies the type of importer being added; must correspond to an importer loaded at server startup :type importer_type_id: str :param repo_plugin_config: configuration values for the importer; may be None :type repo_plugin_config: dict :raise MissingResource: if repo_id does not represent a valid repo :raise InvalidImporterConfiguration: if the importer cannot be initialized for the given repo """ repo_coll = Repo.get_collection() importer_coll = RepoImporter.get_collection() # Validation repo = repo_coll.find_one({'id' : repo_id}) if repo is None: raise MissingResource(repo_id) if not plugin_api.is_valid_importer(importer_type_id): raise InvalidValue(['importer_type_id']) importer_instance, plugin_config = plugin_api.get_importer_by_id(importer_type_id) # Convention is that a value of None means unset. Remove any keys that # are explicitly set to None so the plugin will default them. if repo_plugin_config is not None: clean_config = dict([(k, v) for k, v in repo_plugin_config.items() if v is not None]) else: clean_config = None # Let the importer plugin verify the configuration call_config = PluginCallConfiguration(plugin_config, clean_config) transfer_repo = common_utils.to_transfer_repo(repo) transfer_repo.working_dir = common_utils.importer_working_dir(importer_type_id, repo_id) try: result = importer_instance.validate_config(transfer_repo, call_config) # For backward compatibility with plugins that don't yet return the tuple if isinstance(result, bool): valid_config = result message = None else: valid_config, message = result except Exception, e: logger.exception( 'Exception received from importer [%s] while validating config' % importer_type_id) raise PulpDataException(e.args), None, sys.exc_info()[2]
def resolve_dependencies_by_units(repo_id, units, options): """ Calculates dependencies for the given set of units in the given repository. :param repo_id: identifies the repository :type repo_id: str :param units: list of database representations of units to resolve dependencies for :type units: list :param options: dict of options to pass the importer to drive the resolution :type options: dict or None :return: report from the plugin :rtype: object :raise MissingResource: if the repo does not exist or does not have an importer """ # Validation repo_query_manager = manager_factory.repo_query_manager() importer_manager = manager_factory.repo_importer_manager() # The following will raise MissingResource as appropriate repo = repo_query_manager.get_repository(repo_id) repo_importer = importer_manager.get_importer(repo_id) try: importer_instance, plugin_config = plugin_api.get_importer_by_id( repo_importer['importer_type_id']) except plugin_exceptions.PluginNotFound: raise MissingResource(repo_id), None, sys.exc_info()[2] # Package for the importer call call_config = PluginCallConfiguration(plugin_config, repo_importer['config'], options) transfer_repo = common_utils.to_transfer_repo(repo) conduit = DependencyResolutionConduit(repo_id, repo_importer['id']) # Convert all of the units into the plugin standard representation transfer_units = [] # Preload all the type defs so we don't hammer the database unnecessarily type_defs = {} all_type_def_ids = set([u['unit_type_id'] for u in units]) for def_id in all_type_def_ids: type_def = types_db.type_definition(def_id) type_defs[def_id] = type_def for unit in units: type_id = unit['unit_type_id'] u = conduit_common_utils.to_plugin_associated_unit(unit, type_defs[type_id]) transfer_units.append(u) # Invoke the importer try: dep_report = importer_instance.resolve_dependencies(transfer_repo, transfer_units, conduit, call_config) except Exception: raise PulpExecutionException(), None, sys.exc_info()[2] return dep_report
def sync(repo_id, sync_config_override=None, scheduled_call_id=None): """ Performs a synchronize operation on the given repository and triggers publishes for distributors with auto-publish enabled. The given repo must have an importer configured. This method is intentionally limited to synchronizing a single repo. Performing multiple repository syncs concurrently will require a more global view of the server and must be handled outside the scope of this class. :param repo_id: identifies the repo to sync :type repo_id: str :param sync_config_override: optional config containing values to use for this sync only :type sync_config_override: dict :param scheduled_call_id: id of scheduled call that dispatched this task :type scheduled_call_id: str :return: TaskResult containing sync results and a list of spawned tasks :rtype: pulp.server.async.tasks.TaskResult :raise pulp_exceptions.MissingResource: if specified repo does not exist, or it does not have an importer and associated plugin :raise pulp_exceptions.PulpExecutionException: if the task fails. """ repo_obj = model.Repository.objects.get_repo_or_missing_resource(repo_id) transfer_repo = repo_obj.to_transfer_repo() repo_importer = model.Importer.objects.get_or_404(repo_id=repo_id) try: importer, imp_config = plugin_api.get_importer_by_id(repo_importer.importer_type_id) except plugin_exceptions.PluginNotFound: raise pulp_exceptions.MissingResource(repository=repo_id) call_config = PluginCallConfiguration(imp_config, repo_importer.config, sync_config_override) transfer_repo.working_dir = common_utils.get_working_directory() conduit = RepoSyncConduit(repo_id, repo_importer.importer_type_id, repo_importer.id) sync_result_collection = RepoSyncResult.get_collection() # Fire an events around the call fire_manager = manager_factory.event_fire_manager() fire_manager.fire_repo_sync_started(repo_id) # Perform the sync sync_start_timestamp = _now_timestamp() sync_result = None try: # Replace the Importer's sync_repo() method with our register_sigterm_handler decorator, # which will set up cancel_sync_repo() as the target for the signal handler sync_repo = register_sigterm_handler(importer.sync_repo, importer.cancel_sync_repo) sync_report = sync_repo(transfer_repo, conduit, call_config) except Exception, e: sync_end_timestamp = _now_timestamp() sync_result = RepoSyncResult.error_result( repo_obj.repo_id, repo_importer['id'], repo_importer['importer_type_id'], sync_start_timestamp, sync_end_timestamp, e, sys.exc_info()[2]) raise
def import_uploaded_unit(self, repo_id, unit_type_id, unit_key, unit_metadata, upload_id): """ Called to trigger the importer's handling of an uploaded unit. This should not be called until the bits have finished uploading. The importer is then responsible for moving the file to the correct location, adding it to the Pulp server's inventory, and associating it with the repository. This call will first call is_valid_upload to check the integrity of the destination repository. See that method's documentation for exception possibilities. @param repo_id: identifies the repository into which the unit is uploaded @type repo_id: str @param unit_type_id: type of unit being uploaded @type unit_type_id: str @param unit_key: unique identifier for the unit (user-specified) @type unit_key: dict @param unit_metadata: any user-specified information about the unit @type unit_metadata: dict @param upload_id: upload being imported @type upload_id: str """ # If it doesn't raise an exception, it's good to go self.is_valid_upload(repo_id, unit_type_id) repo_query_manager = manager_factory.repo_query_manager() importer_manager = manager_factory.repo_importer_manager() repo = repo_query_manager.find_by_id(repo_id) repo_importer = importer_manager.get_importer(repo_id) try: importer_instance, plugin_config = plugin_api.get_importer_by_id(repo_importer['importer_type_id']) except plugin_exceptions.PluginNotFound: raise MissingResource(repo_id), None, sys.exc_info()[2] # Assemble the data needed for the import conduit = UploadConduit(repo_id, repo_importer['id'], RepoContentUnit.OWNER_TYPE_USER, pulp_principal.get_principal()['login']) call_config = PluginCallConfiguration(plugin_config, repo_importer['config'], None) transfer_repo = repo_common_utils.to_transfer_repo(repo) transfer_repo.working_dir = repo_common_utils.importer_working_dir(repo_importer['importer_type_id'], repo_id, mkdir=True) file_path = self._upload_file_path(upload_id) # Invoke the importer try: # def upload_unit(self, type_id, unit_key, metadata, file_path, conduit, config): report = importer_instance.upload_unit(transfer_repo, unit_type_id, unit_key, unit_metadata, file_path, conduit, call_config) except Exception, e: _LOG.exception('Error from the importer while importing uploaded unit to repository [%s]' % repo_id) raise PulpExecutionException(e), None, sys.exc_info()[2]
def set_importer(repo_id, importer_type_id, repo_plugin_config): """ Configures an importer to be used for the given repository. Keep in mind this method is written assuming single importer for a repo. The domain model technically supports multiple importers, but this call is what enforces the single importer behavior. :param repo_id: identifies the repo :type repo_id: str :param importer_type_id: identifies the type of importer being added; must correspond to an importer loaded at server startup :type importer_type_id: str :param repo_plugin_config: configuration values for the importer; may be None :type repo_plugin_config: dict :raise MissingResource: if repo_id does not represent a valid repo :raise InvalidImporterConfiguration: if the importer cannot be initialized for the given repo """ RepoImporterManager.validate_importer_config(repo_id, importer_type_id, repo_plugin_config) importer_coll = RepoImporter.get_collection() repo_obj = model.Repository.objects.get_repo_or_missing_resource(repo_id) importer_instance, plugin_config = plugin_api.get_importer_by_id(importer_type_id) # Convention is that a value of None means unset. Remove any keys that # are explicitly set to None so the plugin will default them. if repo_plugin_config is not None: clean_config = dict([(k, v) for k, v in repo_plugin_config.items() if v is not None]) else: clean_config = None # Let the importer plugin verify the configuration call_config = PluginCallConfiguration(plugin_config, clean_config) transfer_repo = repo_obj.to_transfer_repo() # Remove old importer if one exists try: RepoImporterManager.remove_importer(repo_id) except MissingResource: pass # it didn't exist, so no harm done # Let the importer plugin initialize the repository try: importer_instance.importer_added(transfer_repo, call_config) except Exception: _logger.exception( 'Error initializing importer [%s] for repo [%s]' % (importer_type_id, repo_id)) raise PulpExecutionException(), None, sys.exc_info()[2] # Database Update importer_id = importer_type_id # use the importer name as its repo ID importer = RepoImporter(repo_id, importer_id, importer_type_id, clean_config) importer_coll.save(importer) return importer
def _get_importer_instance_and_config(repo_id): importer_manager = manager_factory.repo_importer_manager() try: repo_importer = importer_manager.get_importer(repo_id) importer, config = plugin_api.get_importer_by_id(repo_importer['importer_type_id']) except (MissingResource, plugin_exceptions.PluginNotFound): importer = None config = None return importer, config
def _get_importer_instance_and_config(self, repo_id): importer_manager = manager_factory.repo_importer_manager() try: repo_importer = importer_manager.get_importer(repo_id) importer, config = plugin_api.get_importer_by_id(repo_importer['importer_type_id']) except (MissingResource, plugin_exceptions.PluginNotFound): importer = None config = None return importer, config
def set_importer(repo_id, importer_type_id, repo_plugin_config): """ Configures an importer to be used for the given repository. :param repo: repository object that the importer should be associated with :type repo: pulp.server.db.model.Repository :param importer_type_id: type of importer, must correspond to a plugin loaded at server startup :type importer_type_id: str :param repo_plugin_config: configuration values for the importer; may be None :type repo_plugin_config: dict or None :return: key-value pairs describing the importer that was set :rtype: dict :raises PulpExecutionException: if something goes wrong in the plugin :raises exceptions.InvalidValue: if the values passed to create the importer are invalid """ repo = model.Repository.objects.get_repo_or_missing_resource(repo_id) if not plugin_api.is_valid_importer(importer_type_id): raise exceptions.PulpCodedValidationException(error_code=error_codes.PLP1008, importer_type_id=importer_type_id) importer_instance, plugin_config = plugin_api.get_importer_by_id(importer_type_id) clean_config = clean_config_dict(repo_plugin_config) # Let the importer plugin verify the configuration call_config = PluginCallConfiguration(plugin_config, clean_config) transfer_repo = repo.to_transfer_repo() validate_importer_config(repo, importer_type_id, clean_config) try: remove_importer(repo_id) except exceptions.MissingResource: pass # it didn't exist, so no harm done # Let the importer plugin initialize the importer try: importer_instance.importer_added(transfer_repo, call_config) except Exception: _logger.exception( 'Error initializing importer [%s] for repo [%s]' % (importer_type_id, repo.repo_id)) raise exceptions.PulpExecutionException(), None, sys.exc_info()[2] importer = model.Importer(repo_id, importer_type_id, clean_config) try: importer.save() except ValidationError, e: raise exceptions.InvalidValue(e.to_dict().keys())
def test_importers(self): # listing importers = api.list_importers() self.assertEqual(len(importers), 1) self.assertEqual(importers, {IMPORTER_ID: METADATA}) # list types self.assertEqual(api.list_importer_types(IMPORTER_ID), METADATA) # by id importer = api.get_importer_by_id(IMPORTER_ID) self.assertFalse(importer is None) self.assertTrue(isinstance(importer[0], MockImporter)) self.assertRaises(PluginNotFound, api.get_importer_by_id, 'not-valid') # is_valid self.assertTrue(api.is_valid_importer(IMPORTER_ID)) self.assertFalse(api.is_valid_importer('not-valid'))
def remove_from_importer(repo_id, transfer_units): repo_obj = model.Repository.objects.get_repo_or_missing_resource(repo_id) transfer_repo = repo_obj.to_transfer_repo() repo_importer = model.Importer.objects(repo_id=repo_id).first() # Retrieve the plugin instance to invoke importer_instance, plugin_config = plugin_api.get_importer_by_id(repo_importer.importer_type_id) call_config = PluginCallConfiguration(plugin_config, repo_importer.config) # Invoke the importer's remove method try: importer_instance.remove_units(transfer_repo, transfer_units, call_config) except Exception: msg = _("Exception from importer [%(i)s] while removing units from repo [%(r)s]") msg = msg % {"i": repo_importer.importer_type_id, "r": repo_id} logger.exception(msg)
def validate_importer_config(repo_id, importer_type_id, importer_config): """ Validate an importer configuration. This validates that the repository and importer type exist as these are both required to validate the configuration. :param repo_id: identifies the repo :type repo_id: str :param importer_type_id: identifies the type of importer being added; must correspond to an importer loaded at server startup :type importer_type_id: str :param importer_config: configuration values for the importer; may be None :type importer_config: dict """ repo_coll = Repo.get_collection() repo = repo_coll.find_one({'id': repo_id}) if repo is None: raise MissingResource(repo_id) if not plugin_api.is_valid_importer(importer_type_id): raise PulpCodedValidationException(error_code=error_codes.PLP1008, importer_type_id=importer_type_id) importer_instance, plugin_config = plugin_api.get_importer_by_id(importer_type_id) # Convention is that a value of None means unset. Remove any keys that # are explicitly set to None so the plugin will default them. if importer_config is not None: clean_config = dict([(k, v) for k, v in importer_config.items() if v is not None]) else: clean_config = None # Let the importer plugin verify the configuration call_config = PluginCallConfiguration(plugin_config, clean_config) transfer_repo = common_utils.to_transfer_repo(repo) transfer_repo.working_dir = common_utils.importer_working_dir(importer_type_id, repo_id) result = importer_instance.validate_config(transfer_repo, call_config) # For backward compatibility with plugins that don't yet return the tuple if isinstance(result, bool): valid_config = result message = None else: valid_config, message = result if not valid_config: raise PulpCodedValidationException(validation_errors=message)
def remove_from_importer(repo_id, transfer_units): repo_obj = model.Repository.objects.get_repo_or_missing_resource(repo_id) transfer_repo = repo_obj.to_transfer_repo() importer_manager = manager_factory.repo_importer_manager() repo_importer = importer_manager.get_importer(repo_id) # Retrieve the plugin instance to invoke importer_instance, plugin_config = plugin_api.get_importer_by_id( repo_importer['importer_type_id']) call_config = PluginCallConfiguration(plugin_config, repo_importer['config']) # Invoke the importer's remove method try: importer_instance.remove_units(transfer_repo, transfer_units, call_config) except Exception: msg = _('Exception from importer [%(i)s] while removing units from repo [%(r)s]') msg = msg % {'i': repo_importer['id'], 'r': repo_id} logger.exception(msg)
def remove_importer(repo_id): """ Removes an importer from a repository. :param repo_id: identifies the repo :type repo_id: str """ repo_obj = model.Repository.objects.get_repo_or_missing_resource(repo_id) repo_importer = model.Importer.objects.get_or_404(repo_id=repo_id) # remove schedules sync_manager = manager_factory.repo_sync_schedule_manager() sync_manager.delete_by_importer_id(repo_id, repo_importer.importer_type_id) # Call the importer's cleanup method importer_instance, plugin_config = plugin_api.get_importer_by_id(repo_importer.importer_type_id) call_config = PluginCallConfiguration(plugin_config, repo_importer.config) transfer_repo = repo_obj.to_transfer_repo() importer_instance.importer_removed(transfer_repo, call_config) repo_importer.delete()
def remove_importer(repo_id): """ Removes an importer from a repository. :param repo_id: identifies the repo :type repo_id: str :raise MissingResource: if the given repo does not exist :raise MissingResource: if the given repo does not have an importer """ repo_coll = Repo.get_collection() importer_coll = RepoImporter.get_collection() # Validation repo = repo_coll.find_one({'id': repo_id}) if repo is None: raise MissingResource(repo_id) repo_importer = importer_coll.find_one({'repo_id': repo_id}) if repo_importer is None: raise MissingResource(repo_id) # remove schedules RepoSyncScheduleManager().delete_by_importer_id(repo_id, repo_importer['id']) # Call the importer's cleanup method importer_type_id = repo_importer['importer_type_id'] importer_instance, plugin_config = plugin_api.get_importer_by_id(importer_type_id) call_config = PluginCallConfiguration(plugin_config, repo_importer['config']) transfer_repo = common_utils.to_transfer_repo(repo) transfer_repo.working_dir = common_utils.importer_working_dir(importer_type_id, repo_id) importer_instance.importer_removed(transfer_repo, call_config) # Update the database to reflect the removal importer_coll.remove({'repo_id': repo_id}, safe=True)
def get_importer_by_id(object_id): """ Get a plugin and call configuration using the document ID of the repository-importer association document. :param object_id: The document ID. :type object_id: str :return: A tuple of: (pulp.plugins.importer.Importer, pulp.plugins.config.PluginCallConfiguration) :rtype: tuple :raise pulp.plugins.loader.exceptions.PluginNotFound: not found. """ try: object_id = ObjectId(object_id) except InvalidId: raise plugin_exceptions.PluginNotFound() try: document = model.Importer.objects.get(id=object_id) except DoesNotExist: raise plugin_exceptions.PluginNotFound() plugin, cfg = plugin_api.get_importer_by_id(document.importer_type_id) call_conf = PluginCallConfiguration(cfg, document.config) return plugin, call_conf
def associate_from_repo(source_repo_id, dest_repo_id, criteria=None, import_config_override=None): """ Creates associations in a repository based on the contents of a source repository. Units from the source repository can be filtered by specifying a criteria object. The destination repository must have an importer that can support the types of units being associated. This is done by analyzing the unit list and the importer metadata and takes place before the destination repository is called. Pulp does not actually perform the associations as part of this call. The unit list is determined and passed to the destination repository's importer. It is the job of the importer to make the associate calls back into Pulp where applicable. If criteria is None, the effect of this call is to copy the source repository's associations into the destination repository. :param source_repo_id: identifies the source repository :type source_repo_id: str :param dest_repo_id: identifies the destination repository :type dest_repo_id: str :param criteria: optional; if specified, will filter the units retrieved from the source repository :type criteria: pulp.server.db.model.criteria.UnitAssociationCriteria :param import_config_override: optional config containing values to use for this import only :type import_config_override: dict :return: dict with key 'units_successful' whose value is a list of unit keys that were copied. units that were associated by this operation :rtype: dict :raise MissingResource: if either of the specified repositories don't exist """ importer_manager = manager_factory.repo_importer_manager() source_repo = model.Repository.objects.get_repo_or_missing_resource(source_repo_id) dest_repo = model.Repository.objects.get_repo_or_missing_resource(dest_repo_id) # This will raise MissingResource if there isn't one, which is the # behavior we want this method to exhibit, so just let it bubble up. dest_repo_importer = importer_manager.get_importer(dest_repo_id) source_repo_importer = importer_manager.get_importer(source_repo_id) # The docs are incorrect on the list_importer_types call; it actually # returns a dict with the types under key "types" for some reason. supported_type_ids = set(plugin_api.list_importer_types( dest_repo_importer['importer_type_id'])['types']) # Get the unit types from the repo source repo source_repo_unit_types = set(source_repo.content_unit_counts.keys()) # Now we can make sure the destination repository's importer is capable # of importing either the selected units or all of the units if not source_repo_unit_types.issubset(supported_type_ids): raise exceptions.PulpCodedException( error_code=error_codes.PLP0000, message='The the target importer does not support the types from the source') transfer_units = None # If criteria is specified, retrieve the list of units now if criteria is not None: # if all source types have been converted to mongo - search via new style if source_repo_unit_types.issubset(set(plugin_api.list_unit_models())): association_q = mongoengine.Q(__raw__=criteria.association_spec) unit_q = mongoengine.Q(__raw__=criteria.unit_spec) transfer_units = repo_controller.find_repo_content_units( repository=source_repo, repo_content_unit_q=association_q, units_q=unit_q, yield_content_unit=True) else: # else, search via old style associate_us = load_associated_units(source_repo_id, criteria) # If units were supposed to be filtered but none matched, we're done if len(associate_us) == 0: # Return an empty list to indicate nothing was copied return {'units_successful': []} # Convert all of the units into the plugin standard representation if # a filter was specified transfer_units = None if associate_us is not None: associated_unit_type_ids = calculate_associated_type_ids(source_repo_id, associate_us) transfer_units = create_transfer_units(associate_us, associated_unit_type_ids) # Convert the two repos into the plugin API model transfer_dest_repo = dest_repo.to_transfer_repo() transfer_source_repo = source_repo.to_transfer_repo() # Invoke the importer importer_instance, plugin_config = plugin_api.get_importer_by_id( dest_repo_importer['importer_type_id']) call_config = PluginCallConfiguration(plugin_config, dest_repo_importer['config'], import_config_override) conduit = ImportUnitConduit( source_repo_id, dest_repo_id, source_repo_importer['id'], dest_repo_importer['id']) try: copied_units = importer_instance.import_units( transfer_source_repo, transfer_dest_repo, conduit, call_config, units=transfer_units) unit_ids = [u.to_id_dict() for u in copied_units] return {'units_successful': unit_ids} except Exception: msg = _('Exception from importer [%(i)s] while importing units into repository [%(r)s]') msg_dict = {'i': dest_repo_importer['importer_type_id'], 'r': dest_repo_id} logger.exception(msg % msg_dict) raise exceptions.PulpExecutionException(), None, sys.exc_info()[2]
def associate_from_repo(self, source_repo_id, dest_repo_id, criteria=None): """ Creates associations in a repository based on the contents of a source repository. Units from the source repository can be filtered by specifying a criteria object. The destination repository must have an importer that can support the types of units being associated. This is done by analyzing the unit list and the importer metadata and takes place before the destination repository is called. Pulp does not actually perform the associations as part of this call. The unit list is determined and passed to the destination repository's importer. It is the job of the importer to make the associate calls back into Pulp where applicable. If criteria is None, the effect of this call is to copy the source repository's associations into the destination repository. @param source_repo_id: identifies the source repository @type source_repo_id: str @param dest_repo_id: identifies the destination repository @type dest_repo_id: str @param criteria: optional; if specified, will filter the units retrieved from the source repository @type criteria: L{Criteria} @raise MissingResource: if either of the specified repositories don't exist """ # Validation repo_query_manager = manager_factory.repo_query_manager() importer_manager = manager_factory.repo_importer_manager() dependency_manager = manager_factory.dependency_manager() association_query_manager = manager_factory.repo_unit_association_query_manager() source_repo = repo_query_manager.get_repository(source_repo_id) dest_repo = repo_query_manager.get_repository(dest_repo_id) # This will raise MissingResource if there isn't one, which is the # behavior we want this method to exhibit, so just let it bubble up. dest_repo_importer = importer_manager.get_importer(dest_repo_id) source_repo_importer = importer_manager.get_importer(source_repo_id) # The docs are incorrect on the list_importer_types call; it actually # returns a dict with the types under key "types" for some reason. supported_type_ids = plugin_api.list_importer_types(dest_repo_importer['importer_type_id'])['types'] # If criteria is specified, retrieve the list of units now associate_us = None if criteria is not None: criteria.association_fields = None criteria.unit_fields = None # Retrieve the units to be associated associate_us = association_query_manager.get_units(source_repo_id, criteria=criteria) # If units were supposed to be filtered but none matched, we're done if len(associate_us) is 0: return # Now we can make sure the destination repository's importer is capable # of importing either the selected units or all of the units if associate_us is not None: associated_unit_type_ids = set([u['unit_type_id'] for u in associate_us]) else: association_query_manager = manager_factory.repo_unit_association_query_manager() # We may want to make a call here that only retrieves the unique # type IDs instead of all of the units, but for now it doesn't exist # and I'm not entirely sure this will be a huge problem. all_units = association_query_manager.get_units(source_repo_id) associated_unit_type_ids = set(u['unit_type_id'] for u in all_units) unsupported_types = [t for t in associated_unit_type_ids if t not in supported_type_ids] if len(unsupported_types) > 0: raise exceptions.InvalidValue(['types']) # Convert all of the units into the plugin standard representation if # a filter was specified transfer_units = None if associate_us is not None: type_defs = {} for def_id in associated_unit_type_ids: type_def = types_db.type_definition(def_id) type_defs[def_id] = type_def transfer_units = [] for unit in associate_us: type_id = unit['unit_type_id'] u = conduit_common_utils.to_plugin_unit(unit, type_defs[type_id]) transfer_units.append(u) # Convert the two repos into the plugin API model transfer_dest_repo = common_utils.to_transfer_repo(dest_repo) transfer_dest_repo.working_dir = common_utils.importer_working_dir(dest_repo_importer['importer_type_id'], dest_repo['id'], mkdir=True) transfer_source_repo = common_utils.to_transfer_repo(source_repo) transfer_source_repo.working_dir = common_utils.importer_working_dir(source_repo_importer['importer_type_id'], source_repo['id'], mkdir=True) # Invoke the importer importer_instance, plugin_config = plugin_api.get_importer_by_id(dest_repo_importer['importer_type_id']) call_config = PluginCallConfiguration(plugin_config, dest_repo_importer['config']) conduit = ImportUnitConduit(source_repo_id, dest_repo_id, source_repo_importer['id'], dest_repo_importer['id']) try: importer_instance.import_units(transfer_source_repo, transfer_dest_repo, conduit, call_config, units=transfer_units) except Exception: _LOG.exception('Exception from importer [%s] while importing units into repository [%s]' % (dest_repo_importer['importer_type_id'], dest_repo_id)) raise exceptions.PulpExecutionException(), None, sys.exc_info()[2]
def import_uploaded_unit(repo_id, unit_type_id, unit_key, unit_metadata, upload_id, override_config=None): """ Called to trigger the importer's handling of an uploaded unit. This should not be called until the bits have finished uploading. The importer is then responsible for moving the file to the correct location, adding it to the Pulp server's inventory, and associating it with the repository. This call will first call is_valid_upload to check the integrity of the destination repository. See that method's documentation for exception possibilities. :param repo_id: identifies the repository into which the unit is uploaded :type repo_id: str :param unit_type_id: type of unit being uploaded :type unit_type_id: str :param unit_key: unique identifier for the unit (user-specified) :type unit_key: dict :param unit_metadata: any user-specified information about the unit :type unit_metadata: dict :param upload_id: upload being imported :type upload_id: str :return: A dictionary describing the success or failure of the upload. It must contain the following keys: 'success_flag': bool. Indicates whether the upload was successful 'summary': json-serializable object, providing summary 'details': json-serializable object, providing details :rtype: dict :raises MissingResource: if upload request was for the non-existent repository :raises PulpCodedException: if import was unsuccessful and it was handled by the importer :raises PulpException: if import was unsuccessful and it was not handled by the importer :raises PulpExecutionException: if an unexpected error occured during the upload """ # If it doesn't raise an exception, it's good to go ContentUploadManager.is_valid_upload(repo_id, unit_type_id) repo_obj = model.Repository.objects.get_repo_or_missing_resource(repo_id) repo_importer = model.Importer.objects.get_or_404(repo_id=repo_id) try: importer_instance, plugin_config = plugin_api.get_importer_by_id( repo_importer['importer_type_id']) except plugin_exceptions.PluginNotFound: raise MissingResource(repo_id), None, sys.exc_info()[2] # Assemble the data needed for the import conduit = UploadConduit(repo_id, repo_importer['id']) call_config = PluginCallConfiguration(plugin_config, repo_importer['config'], override_config) transfer_repo = repo_obj.to_transfer_repo() file_path = ContentUploadManager._upload_file_path(upload_id) # Invoke the importer try: result = importer_instance.upload_unit(transfer_repo, unit_type_id, unit_key, unit_metadata, file_path, conduit, call_config) if not result['success_flag']: raise PulpCodedException( error_code=error_codes.PLP0047, repo_id=transfer_repo.id, importer_id=repo_importer['importer_type_id'], unit_type=unit_type_id, summary=result['summary'], details=result['details'] ) repo_controller.rebuild_content_unit_counts(repo_obj) return result except PulpException: msg = _('Error from the importer while importing uploaded unit to repository [%(r)s]') msg = msg % {'r': repo_id} logger.exception(msg) raise except Exception, e: msg = _('Error from the importer while importing uploaded unit to repository [%(r)s]') msg = msg % {'r': repo_id} logger.exception(msg) raise PulpExecutionException(e), None, sys.exc_info()[2]
def associate_from_repo(source_repo_id, dest_repo_id, criteria=None, import_config_override=None): """ Creates associations in a repository based on the contents of a source repository. Units from the source repository can be filtered by specifying a criteria object. The destination repository must have an importer that can support the types of units being associated. This is done by analyzing the unit list and the importer metadata and takes place before the destination repository is called. Pulp does not actually perform the associations as part of this call. The unit list is determined and passed to the destination repository's importer. It is the job of the importer to make the associate calls back into Pulp where applicable. If criteria is None, the effect of this call is to copy the source repository's associations into the destination repository. :param source_repo_id: identifies the source repository :type source_repo_id: str :param dest_repo_id: identifies the destination repository :type dest_repo_id: str :param criteria: optional; if specified, will filter the units retrieved from the source repository :type criteria: UnitAssociationCriteria :param import_config_override: optional config containing values to use for this import only :type import_config_override: dict :return: list of unit IDs (see pulp.plugins.model.Unit.to_id_dict) for units that were associated by this operation :rtype: list :raise MissingResource: if either of the specified repositories don't exist """ # Validation repo_query_manager = manager_factory.repo_query_manager() importer_manager = manager_factory.repo_importer_manager() source_repo = repo_query_manager.get_repository(source_repo_id) dest_repo = repo_query_manager.get_repository(dest_repo_id) # This will raise MissingResource if there isn't one, which is the # behavior we want this method to exhibit, so just let it bubble up. dest_repo_importer = importer_manager.get_importer(dest_repo_id) source_repo_importer = importer_manager.get_importer(source_repo_id) # The docs are incorrect on the list_importer_types call; it actually # returns a dict with the types under key "types" for some reason. supported_type_ids = plugin_api.list_importer_types( dest_repo_importer['importer_type_id'])['types'] # If criteria is specified, retrieve the list of units now associate_us = None if criteria is not None: associate_us = load_associated_units(source_repo_id, criteria) # If units were supposed to be filtered but none matched, we're done if len(associate_us) is 0: # Return an empty list to indicate nothing was copied return [] # Now we can make sure the destination repository's importer is capable # of importing either the selected units or all of the units associated_unit_type_ids = calculate_associated_type_ids(source_repo_id, associate_us) unsupported_types = [t for t in associated_unit_type_ids if t not in supported_type_ids] if len(unsupported_types) > 0: raise exceptions.InvalidValue(['types']) # Convert all of the units into the plugin standard representation if # a filter was specified transfer_units = None if associate_us is not None: transfer_units = create_transfer_units(associate_us, associated_unit_type_ids) # Convert the two repos into the plugin API model transfer_dest_repo = common_utils.to_transfer_repo(dest_repo) transfer_dest_repo.working_dir = common_utils.importer_working_dir( dest_repo_importer['importer_type_id'], dest_repo['id'], mkdir=True) transfer_source_repo = common_utils.to_transfer_repo(source_repo) transfer_source_repo.working_dir = common_utils.importer_working_dir( source_repo_importer['importer_type_id'], source_repo['id'], mkdir=True) # Invoke the importer importer_instance, plugin_config = plugin_api.get_importer_by_id( dest_repo_importer['importer_type_id']) call_config = PluginCallConfiguration(plugin_config, dest_repo_importer['config'], import_config_override) login = manager_factory.principal_manager().get_principal()['login'] conduit = ImportUnitConduit( source_repo_id, dest_repo_id, source_repo_importer['id'], dest_repo_importer['id'], RepoContentUnit.OWNER_TYPE_USER, login) try: copied_units = importer_instance.import_units( transfer_source_repo, transfer_dest_repo, conduit, call_config, units=transfer_units) unit_ids = [u.to_id_dict() for u in copied_units] return {'units_successful': unit_ids} except Exception: msg = _('Exception from importer [%(i)s] while importing units into repository [%(r)s]') msg = msg % {'i': dest_repo_importer['importer_type_id'], 'r': dest_repo_id} logger.exception(msg) raise exceptions.PulpExecutionException(), None, sys.exc_info()[2]
def update_importer_config(repo_id, importer_config): """ Attempts to update the saved configuration for the given repo's importer. The importer will be asked if the new configuration is valid. If not, this method will raise an error and the existing configuration will remain unchanged. :param repo_id: identifies the repo :type repo_id: str :param importer_config: new configuration values to use for this repo :type importer_config: dict :raise MissingResource: if the given repo does not exist :raise MissingResource: if the given repo does not have an importer :raise InvalidConfiguration: if the plugin indicates the given configuration is invalid """ repo_coll = Repo.get_collection() importer_coll = RepoImporter.get_collection() # Input Validation repo = repo_coll.find_one({'id': repo_id}) if repo is None: raise MissingResource(repo_id) repo_importer = importer_coll.find_one({'repo_id': repo_id}) if repo_importer is None: raise MissingResource(repo_id) importer_type_id = repo_importer['importer_type_id'] importer_instance, plugin_config = plugin_api.get_importer_by_id(importer_type_id) # The supplied config is a delta of changes to make to the existing config. # The plugin expects a full configuration, so we apply those changes to # the original config and pass that to the plugin's validate method. merged_config = dict(repo_importer['config']) # The convention is that None in an update is removing the value and # setting it to the default. Find all such properties in this delta and # remove them from the existing config if they are there. unset_property_names = [k for k in importer_config if importer_config[k] is None] for key in unset_property_names: merged_config.pop(key, None) importer_config.pop(key, None) # Whatever is left over are the changed/added values, so merge them in. merged_config.update(importer_config) # Let the importer plugin verify the configuration call_config = PluginCallConfiguration(plugin_config, merged_config) transfer_repo = common_utils.to_transfer_repo(repo) try: result = importer_instance.validate_config(transfer_repo, call_config) # For backward compatibility with plugins that don't yet return the tuple if isinstance(result, bool): valid_config = result message = None else: valid_config, message = result except Exception, e: msg = _('Exception received from importer [%(i)s] while validating config for repo ' '[%(r)s]') msg = msg % {'i': importer_type_id, 'r': repo_id} _logger.exception(msg) raise PulpDataException(e.args), None, sys.exc_info()[2]
def associate_from_repo(cls, source_repo_id, dest_repo_id, criteria, import_config_override=None): """ Creates associations in a repository based on the contents of a source repository. Units from the source repository can be filtered by specifying a criteria object. The destination repository must have an importer that can support the types of units being associated. This is done by analyzing the unit list and the importer metadata and takes place before the destination repository is called. Pulp does not actually perform the associations as part of this call. The unit list is determined and passed to the destination repository's importer. It is the job of the importer to make the associate calls back into Pulp where applicable. If criteria is None, the effect of this call is to copy the source repository's associations into the destination repository. :param source_repo_id: identifies the source repository :type source_repo_id: str :param dest_repo_id: identifies the destination repository :type dest_repo_id: str :param criteria: optional; if specified, will filter the units retrieved from the source repository :type criteria: pulp.server.db.model.criteria.UnitAssociationCriteria :param import_config_override: optional config containing values to use for this import only :type import_config_override: dict :return: dict with key 'units_successful' whose value is a list of unit keys that were copied. units that were associated by this operation :rtype: dict :raise MissingResource: if either of the specified repositories don't exist """ criteria = UnitAssociationCriteria.from_dict(criteria) source_repo = model.Repository.objects.get_repo_or_missing_resource(source_repo_id) dest_repo = model.Repository.objects.get_repo_or_missing_resource(dest_repo_id) dest_repo_importer = model.Importer.objects.get_or_404(repo_id=dest_repo_id) source_repo_importer = model.Importer.objects.get_or_404(repo_id=source_repo_id) # The docs are incorrect on the list_importer_types call; it actually # returns a dict with the types under key "types" for some reason. supported_type_ids = set(plugin_api.list_importer_types( dest_repo_importer.importer_type_id)['types']) # Get the unit types from the repo source repo source_repo_unit_types = set(source_repo.content_unit_counts.keys()) # Now we can make sure the destination repository's importer is capable # of importing either the selected units or all of the units if not source_repo_unit_types.issubset(supported_type_ids): raise exceptions.PulpCodedException(error_code=error_codes.PLP0044) transfer_units = None # if all source types have been converted to mongo - search via new style if source_repo_unit_types.issubset(set(plugin_api.list_unit_models())): transfer_units = cls._units_from_criteria(source_repo, criteria) else: # else, search via old style associate_us = load_associated_units(source_repo_id, criteria) # If units were supposed to be filtered but none matched, we're done if len(associate_us) == 0: # Return an empty list to indicate nothing was copied return {'units_successful': []} # Convert all of the units into the plugin standard representation if # a filter was specified transfer_units = None if associate_us is not None: transfer_units = create_transfer_units(associate_us) # Convert the two repos into the plugin API model transfer_dest_repo = dest_repo.to_transfer_repo() transfer_source_repo = source_repo.to_transfer_repo() # Invoke the importer importer_instance, plugin_config = plugin_api.get_importer_by_id( dest_repo_importer.importer_type_id) call_config = PluginCallConfiguration(plugin_config, dest_repo_importer.config, import_config_override) conduit = ImportUnitConduit( source_repo_id, dest_repo_id, source_repo_importer.importer_type_id, dest_repo_importer.importer_type_id) try: copied_units = importer_instance.import_units( transfer_source_repo, transfer_dest_repo, conduit, call_config, units=transfer_units) if isinstance(copied_units, tuple): suc_units_ids = [u.to_id_dict() for u in copied_units[0] if u is not None] unsuc_units_ids = [u.to_id_dict() for u in copied_units[1]] repo_controller.rebuild_content_unit_counts(dest_repo) return {'units_successful': suc_units_ids, 'units_failed_signature_filter': unsuc_units_ids} unit_ids = [u.to_id_dict() for u in copied_units if u is not None] repo_controller.rebuild_content_unit_counts(dest_repo) return {'units_successful': unit_ids} except Exception: msg = _('Exception from importer [%(i)s] while importing units into repository [%(r)s]') msg_dict = {'i': dest_repo_importer.importer_type_id, 'r': dest_repo_id} logger.exception(msg % msg_dict) raise exceptions.PulpExecutionException(), None, sys.exc_info()[2]
def import_uploaded_unit(self, repo_id, unit_type_id, unit_key, unit_metadata, upload_id): """ Called to trigger the importer's handling of an uploaded unit. This should not be called until the bits have finished uploading. The importer is then responsible for moving the file to the correct location, adding it to the Pulp server's inventory, and associating it with the repository. This call will first call is_valid_upload to check the integrity of the destination repository. See that method's documentation for exception possibilities. @param repo_id: identifies the repository into which the unit is uploaded @type repo_id: str @param unit_type_id: type of unit being uploaded @type unit_type_id: str @param unit_key: unique identifier for the unit (user-specified) @type unit_key: dict @param unit_metadata: any user-specified information about the unit @type unit_metadata: dict @param upload_id: upload being imported @type upload_id: str """ # If it doesn't raise an exception, it's good to go self.is_valid_upload(repo_id, unit_type_id) repo_query_manager = manager_factory.repo_query_manager() importer_manager = manager_factory.repo_importer_manager() repo = repo_query_manager.find_by_id(repo_id) repo_importer = importer_manager.get_importer(repo_id) try: importer_instance, plugin_config = plugin_api.get_importer_by_id( repo_importer['importer_type_id']) except plugin_exceptions.PluginNotFound: raise MissingResource(repo_id), None, sys.exc_info()[2] # Assemble the data needed for the import conduit = UploadConduit( repo_id, repo_importer['id'], RepoContentUnit.OWNER_TYPE_USER, manager_factory.principal_manager().get_principal()['login']) call_config = PluginCallConfiguration(plugin_config, repo_importer['config'], None) transfer_repo = repo_common_utils.to_transfer_repo(repo) transfer_repo.working_dir = repo_common_utils.importer_working_dir( repo_importer['importer_type_id'], repo_id, mkdir=True) file_path = self._upload_file_path(upload_id) # Invoke the importer try: importer_instance.upload_unit(transfer_repo, unit_type_id, unit_key, unit_metadata, file_path, conduit, call_config) except PulpException: _LOG.exception( 'Error from the importer while importing uploaded unit to repository [%s]' % repo_id) raise except Exception, e: _LOG.exception( 'Error from the importer while importing uploaded unit to repository [%s]' % repo_id) raise PulpExecutionException(e), None, sys.exc_info()[2]
def update_importer_config(self, repo_id, importer_config): """ Attempts to update the saved configuration for the given repo's importer. The importer will be asked if the new configuration is valid. If not, this method will raise an error and the existing configuration will remain unchanged. @param repo_id: identifies the repo @type repo_id: str @param importer_config: new configuration values to use for this repo @type importer_config: dict @raise MissingResource: if the given repo does not exist @raise MissingResource: if the given repo does not have an importer @raise InvalidConfiguration: if the plugin indicates the given configuration is invalid """ repo_coll = Repo.get_collection() importer_coll = RepoImporter.get_collection() # Input Validation repo = repo_coll.find_one({'id' : repo_id}) if repo is None: raise MissingResource(repo_id) repo_importer = importer_coll.find_one({'repo_id' : repo_id}) if repo_importer is None: raise MissingResource(repo_id) importer_type_id = repo_importer['importer_type_id'] importer_instance, plugin_config = plugin_api.get_importer_by_id(importer_type_id) # The supplied config is a delta of changes to make to the existing config. # The plugin expects a full configuration, so we apply those changes to # the original config and pass that to the plugin's validate method. merged_config = dict(repo_importer['config']) # The convention is that None in an update is removing the value and # setting it to the default. Find all such properties in this delta and # remove them from the existing config if they are there. unset_property_names = [k for k in importer_config if importer_config[k] is None] for key in unset_property_names: merged_config.pop(key, None) importer_config.pop(key, None) # Whatever is left over are the changed/added values, so merge them in. merged_config.update(importer_config) # Let the importer plugin verify the configuration call_config = PluginCallConfiguration(plugin_config, merged_config) transfer_repo = common_utils.to_transfer_repo(repo) transfer_repo.working_dir = common_utils.importer_working_dir(importer_type_id, repo_id) query_manager = manager_factory.repo_query_manager() related_repos = query_manager.find_with_importer_type(importer_type_id) transfer_related_repos = [] for r in related_repos: # Don't include the repo being updated in this list if r['id'] == repo_id: continue all_configs = [d['config'] for d in r['importers']] trr = common_utils.to_related_repo(r, all_configs) transfer_related_repos.append(trr) try: result = importer_instance.validate_config(transfer_repo, call_config, transfer_related_repos) # For backward compatibility with plugins that don't yet return the tuple if isinstance(result, bool): valid_config = result message = None else: valid_config, message = result except Exception, e: _LOG.exception('Exception received from importer [%s] while validating config for repo [%s]' % (importer_type_id, repo_id)) raise PulpDataException(e.args), None, sys.exc_info()[2]
def import_uploaded_unit(repo_id, unit_type_id, unit_key, unit_metadata, upload_id, override_config=None): """ Called to trigger the importer's handling of an uploaded unit. This should not be called until the bits have finished uploading. The importer is then responsible for moving the file to the correct location, adding it to the Pulp server's inventory, and associating it with the repository. This call will first call is_valid_upload to check the integrity of the destination repository. See that method's documentation for exception possibilities. :param repo_id: identifies the repository into which the unit is uploaded :type repo_id: str :param unit_type_id: type of unit being uploaded :type unit_type_id: str :param unit_key: unique identifier for the unit (user-specified) :type unit_key: dict :param unit_metadata: any user-specified information about the unit :type unit_metadata: dict :param upload_id: upload being imported :type upload_id: str :return: A SyncReport indicating the success or failure of the upload :rtype: pulp.plugins.model.SyncReport """ # If it doesn't raise an exception, it's good to go ContentUploadManager.is_valid_upload(repo_id, unit_type_id) repo_query_manager = manager_factory.repo_query_manager() importer_manager = manager_factory.repo_importer_manager() repo = repo_query_manager.find_by_id(repo_id) repo_importer = importer_manager.get_importer(repo_id) try: importer_instance, plugin_config = plugin_api.get_importer_by_id( repo_importer['importer_type_id']) except plugin_exceptions.PluginNotFound: raise MissingResource(repo_id), None, sys.exc_info()[2] # Assemble the data needed for the import conduit = UploadConduit(repo_id, repo_importer['id']) call_config = PluginCallConfiguration(plugin_config, repo_importer['config'], override_config) transfer_repo = repo_common_utils.to_transfer_repo(repo) file_path = ContentUploadManager._upload_file_path(upload_id) # Invoke the importer try: return importer_instance.upload_unit(transfer_repo, unit_type_id, unit_key, unit_metadata, file_path, conduit, call_config) except PulpException: msg = _('Error from the importer while importing uploaded unit to repository [%(r)s]') msg = msg % {'r': repo_id} logger.exception(msg) raise except Exception, e: msg = _('Error from the importer while importing uploaded unit to repository [%(r)s]') msg = msg % {'r': repo_id} logger.exception(msg) raise PulpExecutionException(e), None, sys.exc_info()[2]