def test_importers(self):
# listing
importers = api.list_importers()
self.assertEqual(len(importers), 1)
self.assertEqual(importers, {IMPORTER_ID: METADATA})
# list types
self.assertEqual(api.list_importer_types(IMPORTER_ID), METADATA)
# by id
importer = api.get_importer_by_id(IMPORTER_ID)
self.assertFalse(importer is None)
self.assertTrue(isinstance(importer[0], MockImporter))
self.assertRaises(PluginNotFound, api.get_importer_by_id, 'not-valid')
# is_valid
self.assertTrue(api.is_valid_importer(IMPORTER_ID))
self.assertFalse(api.is_valid_importer('not-valid'))
def validate_importer_config(repo_obj, importer_type_id, config):
"""
Validates the importer configuration.
:param repo_obj: repository object
:type repo_obj: pulp.server.db.model.Repository
:param importer_type_id: type of importer, must correspond to a plugin loaded at server startup
:type importer_type_id: str
:param config: configuration values for the importer
:type config: dict
:raises PulpCodedValidationException: if importer_type_id is invalid
:raises exceptions.PulpDataException: if config is invalid.
"""
if not plugin_api.is_valid_importer(importer_type_id):
raise exceptions.PulpCodedValidationException(error_code=error_codes.PLP1008,
importer_type_id=importer_type_id)
importer_instance, plugin_config = plugin_api.get_importer_by_id(importer_type_id)
call_config = PluginCallConfiguration(plugin_config, config)
transfer_repo = repo_obj.to_transfer_repo()
result = importer_instance.validate_config(transfer_repo, call_config)
# For backward compatibility with plugins that don't yet return the tuple
if isinstance(result, bool):
valid_config = result
message = None
else:
valid_config, message = result
if not valid_config:
raise exceptions.PulpDataException(message)
def validate_importer_config(repo_id, importer_type_id, importer_config):
"""
This validates that the repository and importer type exist as these are both required to
validate the configuration.
:param repo_id: identifies the repo
:type repo_id: str
:param importer_type_id: type of importer, must correspond to a plugin loaded at server startup
:type importer_type_id: str
:param importer_config: configuration values for the importer; may be None
:type importer_config: dict
:raises exceptions.PulpCodedValidationException: if config is invalid.
"""
repo_obj = model.Repository.objects.get_repo_or_missing_resource(repo_id)
if not plugin_api.is_valid_importer(importer_type_id):
raise exceptions.PulpCodedValidationException(error_code=error_codes.PLP1008,
importer_type_id=importer_type_id)
importer_instance, plugin_config = plugin_api.get_importer_by_id(importer_type_id)
clean_config = clean_config_dict(importer_config)
call_config = PluginCallConfiguration(plugin_config, clean_config)
transfer_repo = repo_obj.to_transfer_repo()
result = importer_instance.validate_config(transfer_repo, call_config)
# For backward compatibility with plugins that don't yet return the tuple
if isinstance(result, bool):
valid_config = result
message = None
else:
valid_config, message = result
if not valid_config:
raise exceptions.PulpCodedValidationException(validation_errors=message)
def validate_importer_config(repo_obj, importer_type_id, config):
"""
Validates the importer configuration.
:param repo_obj: repository object
:type repo_obj: pulp.server.db.model.Repository
:param importer_type_id: type of importer, must correspond to a plugin loaded at server startup
:type importer_type_id: str
:param config: configuration values for the importer
:type config: dict
:raises PulpCodedValidationException: if importer_type_id is invalid
:raises exceptions.PulpDataException: if config is invalid.
"""
if not plugin_api.is_valid_importer(importer_type_id):
raise exceptions.PulpCodedValidationException(
error_code=error_codes.PLP1008, importer_type_id=importer_type_id)
importer_instance, plugin_config = plugin_api.get_importer_by_id(
importer_type_id)
call_config = PluginCallConfiguration(plugin_config, config)
transfer_repo = repo_obj.to_transfer_repo()
result = importer_instance.validate_config(transfer_repo, call_config)
# For backward compatibility with plugins that don't yet return the tuple
if isinstance(result, bool):
valid_config = result
message = None
else:
valid_config, message = result
if not valid_config:
raise exceptions.PulpDataException(message)
def PUT(self, repo_id, importer_id):
# Raise a MissingResource exception if the repo or the importer doesn't exist
importer_manager = manager_factory.repo_importer_manager()
importer = importer_manager.get_importer(repo_id)
if importer['id'] != importer_id:
raise exceptions.MissingResource(importer_id=importer_id)
if not plugin_api.is_valid_importer(importer_id):
raise exceptions.PulpCodedValidationException(
error_code=error_codes.PLP1008)
params = self.params()
importer_config = params.get('importer_config', None)
if importer_config is None:
_logger.error(
'Missing configuration updating importer for repository [%s]' %
repo_id)
raise exceptions.MissingValue(['importer_config'])
task_tags = [
tags.resource_tag(tags.RESOURCE_REPOSITORY_TYPE, repo_id),
tags.resource_tag(tags.RESOURCE_REPOSITORY_IMPORTER_TYPE,
importer_id),
tags.action_tag('update_importer')
]
async_result = update_importer_config.apply_async_with_reservation(
tags.RESOURCE_REPOSITORY_TYPE,
repo_id, [repo_id], {'importer_config': importer_config},
tags=task_tags)
raise exceptions.OperationPostponed(async_result)
def set_importer(repo_id, importer_type_id, repo_plugin_config):
"""
Configures an importer to be used for the given repository.
Keep in mind this method is written assuming single importer for a repo.
The domain model technically supports multiple importers, but this
call is what enforces the single importer behavior.
:param repo_id: identifies the repo
:type repo_id: str
:param importer_type_id: identifies the type of importer being added;
must correspond to an importer loaded at server startup
:type importer_type_id: str
:param repo_plugin_config: configuration values for the importer; may be None
:type repo_plugin_config: dict
:raise MissingResource: if repo_id does not represent a valid repo
:raise InvalidImporterConfiguration: if the importer cannot be initialized for the given
repo
"""
repo_coll = Repo.get_collection()
importer_coll = RepoImporter.get_collection()
# Validation
repo = repo_coll.find_one({'id' : repo_id})
if repo is None:
raise MissingResource(repo_id)
if not plugin_api.is_valid_importer(importer_type_id):
raise InvalidValue(['importer_type_id'])
importer_instance, plugin_config = plugin_api.get_importer_by_id(importer_type_id)
# Convention is that a value of None means unset. Remove any keys that
# are explicitly set to None so the plugin will default them.
if repo_plugin_config is not None:
clean_config = dict([(k, v) for k, v in repo_plugin_config.items() if v is not None])
else:
clean_config = None
# Let the importer plugin verify the configuration
call_config = PluginCallConfiguration(plugin_config, clean_config)
transfer_repo = common_utils.to_transfer_repo(repo)
transfer_repo.working_dir = common_utils.importer_working_dir(importer_type_id, repo_id)
try:
result = importer_instance.validate_config(transfer_repo, call_config)
# For backward compatibility with plugins that don't yet return the tuple
if isinstance(result, bool):
valid_config = result
message = None
else:
valid_config, message = result
except Exception, e:
logger.exception(
'Exception received from importer [%s] while validating config' % importer_type_id)
raise PulpDataException(e.args), None, sys.exc_info()[2]
def validate_importer_config(repo_id, importer_type_id, importer_config):
"""
Validate an importer configuration. This validates that the repository and importer type
exist as these are both required to validate the configuration.
:param repo_id: identifies the repo
:type repo_id: str
:param importer_type_id: identifies the type of importer being added;
must correspond to an importer loaded at server startup
:type importer_type_id: str
:param importer_config: configuration values for the importer; may be None
:type importer_config: dict
"""
repo_coll = Repo.get_collection()
repo = repo_coll.find_one({'id': repo_id})
if repo is None:
raise MissingResource(repo_id)
if not plugin_api.is_valid_importer(importer_type_id):
raise PulpCodedValidationException(
error_code=error_codes.PLP1008,
importer_type_id=importer_type_id)
importer_instance, plugin_config = plugin_api.get_importer_by_id(
importer_type_id)
# Convention is that a value of None means unset. Remove any keys that
# are explicitly set to None so the plugin will default them.
if importer_config is not None:
clean_config = dict([(k, v) for k, v in importer_config.items()
if v is not None])
else:
clean_config = None
# Let the importer plugin verify the configuration
call_config = PluginCallConfiguration(plugin_config, clean_config)
transfer_repo = common_utils.to_transfer_repo(repo)
transfer_repo.working_dir = common_utils.importer_working_dir(
importer_type_id, repo_id)
result = importer_instance.validate_config(transfer_repo, call_config)
# For backward compatibility with plugins that don't yet return the tuple
if isinstance(result, bool):
valid_config = result
message = None
else:
valid_config, message = result
if not valid_config:
raise PulpCodedValidationException(validation_errors=message)
def set_importer(repo_id, importer_type_id, repo_plugin_config):
"""
Configures an importer to be used for the given repository.
:param repo: repository object that the importer should be associated with
:type repo: pulp.server.db.model.Repository
:param importer_type_id: type of importer, must correspond to a plugin loaded at server startup
:type importer_type_id: str
:param repo_plugin_config: configuration values for the importer; may be None
:type repo_plugin_config: dict or None
:return: key-value pairs describing the importer that was set
:rtype: dict
:raises PulpExecutionException: if something goes wrong in the plugin
:raises exceptions.InvalidValue: if the values passed to create the importer are invalid
"""
repo = model.Repository.objects.get_repo_or_missing_resource(repo_id)
if not plugin_api.is_valid_importer(importer_type_id):
raise exceptions.PulpCodedValidationException(
error_code=error_codes.PLP1008, importer_type_id=importer_type_id)
importer_instance, plugin_config = plugin_api.get_importer_by_id(
importer_type_id)
clean_config = clean_config_dict(repo_plugin_config)
# Let the importer plugin verify the configuration
call_config = PluginCallConfiguration(plugin_config, clean_config)
transfer_repo = repo.to_transfer_repo()
validate_importer_config(repo, importer_type_id, clean_config)
try:
remove_importer(repo_id)
except exceptions.MissingResource:
pass # it didn't exist, so no harm done
# Let the importer plugin initialize the importer
try:
importer_instance.importer_added(transfer_repo, call_config)
except Exception:
_logger.exception('Error initializing importer [%s] for repo [%s]' %
(importer_type_id, repo.repo_id))
raise exceptions.PulpExecutionException(), None, sys.exc_info()[2]
importer = model.Importer(repo_id, importer_type_id, clean_config)
try:
importer.save()
except ValidationError, e:
raise exceptions.InvalidValue(e.to_dict().keys())
def set_importer(repo_id, importer_type_id, repo_plugin_config):
"""
Configures an importer to be used for the given repository.
:param repo: repository object that the importer should be associated with
:type repo: pulp.server.db.model.Repository
:param importer_type_id: type of importer, must correspond to a plugin loaded at server startup
:type importer_type_id: str
:param repo_plugin_config: configuration values for the importer; may be None
:type repo_plugin_config: dict or None
:return: key-value pairs describing the importer that was set
:rtype: dict
:raises PulpExecutionException: if something goes wrong in the plugin
:raises exceptions.InvalidValue: if the values passed to create the importer are invalid
"""
repo = model.Repository.objects.get_repo_or_missing_resource(repo_id)
if not plugin_api.is_valid_importer(importer_type_id):
raise exceptions.PulpCodedValidationException(error_code=error_codes.PLP1008,
importer_type_id=importer_type_id)
importer_instance, plugin_config = plugin_api.get_importer_by_id(importer_type_id)
clean_config = clean_config_dict(repo_plugin_config)
# Let the importer plugin verify the configuration
call_config = PluginCallConfiguration(plugin_config, clean_config)
transfer_repo = repo.to_transfer_repo()
validate_importer_config(repo, importer_type_id, clean_config)
try:
remove_importer(repo_id)
except exceptions.MissingResource:
pass # it didn't exist, so no harm done
# Let the importer plugin initialize the importer
try:
importer_instance.importer_added(transfer_repo, call_config)
except Exception:
_logger.exception(
'Error initializing importer [%s] for repo [%s]' % (importer_type_id, repo.repo_id))
raise exceptions.PulpExecutionException(), None, sys.exc_info()[2]
importer = model.Importer(repo_id, importer_type_id, clean_config)
try:
importer.save()
except ValidationError, e:
raise exceptions.InvalidValue(e.to_dict().keys())
def validate_importer_config(repo_id, importer_type_id, importer_config):
"""
Validate an importer configuration. This validates that the repository and importer type
exist as these are both required to validate the configuration.
:param repo_id: identifies the repo
:type repo_id: str
:param importer_type_id: identifies the type of importer being added;
must correspond to an importer loaded at server startup
:type importer_type_id: str
:param importer_config: configuration values for the importer; may be None
:type importer_config: dict
"""
repo_coll = Repo.get_collection()
repo = repo_coll.find_one({'id': repo_id})
if repo is None:
raise MissingResource(repo_id)
if not plugin_api.is_valid_importer(importer_type_id):
raise PulpCodedValidationException(error_code=error_codes.PLP1008,
importer_type_id=importer_type_id)
importer_instance, plugin_config = plugin_api.get_importer_by_id(importer_type_id)
# Convention is that a value of None means unset. Remove any keys that
# are explicitly set to None so the plugin will default them.
if importer_config is not None:
clean_config = dict([(k, v) for k, v in importer_config.items() if v is not None])
else:
clean_config = None
# Let the importer plugin verify the configuration
call_config = PluginCallConfiguration(plugin_config, clean_config)
transfer_repo = common_utils.to_transfer_repo(repo)
transfer_repo.working_dir = common_utils.importer_working_dir(importer_type_id, repo_id)
result = importer_instance.validate_config(transfer_repo, call_config)
# For backward compatibility with plugins that don't yet return the tuple
if isinstance(result, bool):
valid_config = result
message = None
else:
valid_config, message = result
if not valid_config:
raise PulpCodedValidationException(validation_errors=message)
def validate_importer_config(repo_id, importer_type_id, importer_config):
"""
This validates that the repository and importer type exist as these are both required to
validate the configuration.
:param repo_id: identifies the repo
:type repo_id: str
:param importer_type_id: type of importer, must correspond to a plugin loaded at server startup
:type importer_type_id: str
:param importer_config: configuration values for the importer; may be None
:type importer_config: dict
:raises exceptions.PulpCodedValidationException: if config is invalid.
"""
repo_obj = model.Repository.objects.get_repo_or_missing_resource(repo_id)
if not plugin_api.is_valid_importer(importer_type_id):
raise exceptions.PulpCodedValidationException(
error_code=error_codes.PLP1008, importer_type_id=importer_type_id)
importer_instance, plugin_config = plugin_api.get_importer_by_id(
importer_type_id)
clean_config = clean_config_dict(importer_config)
call_config = PluginCallConfiguration(plugin_config, clean_config)
transfer_repo = repo_obj.to_transfer_repo()
result = importer_instance.validate_config(transfer_repo, call_config)
# For backward compatibility with plugins that don't yet return the tuple
if isinstance(result, bool):
valid_config = result
message = None
else:
valid_config, message = result
if not valid_config:
raise exceptions.PulpCodedValidationException(
validation_errors=message)
def PUT(self, repo_id, importer_id):
# Raise a MissingResource exception if the repo or the importer doesn't exist
importer_manager = manager_factory.repo_importer_manager()
importer = importer_manager.get_importer(repo_id)
if importer['id'] != importer_id:
raise exceptions.MissingResource(importer_id=importer_id)
if not plugin_api.is_valid_importer(importer_id):
raise exceptions.PulpCodedValidationException(error_code=error_codes.PLP1008)
params = self.params()
importer_config = params.get('importer_config', None)
if importer_config is None:
_logger.error('Missing configuration updating importer for repository [%s]' % repo_id)
raise exceptions.MissingValue(['importer_config'])
task_tags = [tags.resource_tag(tags.RESOURCE_REPOSITORY_TYPE, repo_id),
tags.resource_tag(tags.RESOURCE_REPOSITORY_IMPORTER_TYPE, importer_id),
tags.action_tag('update_importer')]
async_result = update_importer_config.apply_async_with_reservation(
tags.RESOURCE_REPOSITORY_TYPE,
repo_id, [repo_id], {'importer_config': importer_config}, tags=task_tags)
raise exceptions.OperationPostponed(async_result)