def setUp(self):
super(Migration0004Tests, self).setUp()
# Special way to import modules that start with a number
self.migration = _import_all_the_way(
'pulp_rpm.plugins.migrations.0004_pkg_group_category_repoid')
factory.initialize()
types_db.update_database([TYPE_DEF_GROUP, TYPE_DEF_CATEGORY])
# Create the repositories necessary for the tests
self.source_repo_id = 'source-repo' # where units were copied from with the bad code
self.dest_repo_id = 'dest-repo' # where bad units were copied to
source_repo = Repo(self.source_repo_id, '')
Repo.get_collection().insert(source_repo, safe=True)
dest_repo = Repo(self.dest_repo_id, '')
Repo.get_collection().insert(dest_repo, safe=True)
source_importer = RepoImporter(self.source_repo_id, 'yum_importer',
'yum_importer', {})
RepoImporter.get_collection().insert(source_importer, safe=True)
dest_importer = RepoImporter(self.dest_repo_id, 'yum_importer',
'yum_importer', {})
RepoImporter.get_collection().insert(dest_importer, safe=True)
def set_importer(repo_id, importer_type_id, repo_plugin_config):
"""
Configures an importer to be used for the given repository.
Keep in mind this method is written assuming single importer for a repo.
The domain model technically supports multiple importers, but this
call is what enforces the single importer behavior.
:param repo_id: identifies the repo
:type repo_id: str
:param importer_type_id: identifies the type of importer being added;
must correspond to an importer loaded at server startup
:type importer_type_id: str
:param repo_plugin_config: configuration values for the importer; may be None
:type repo_plugin_config: dict
:raise MissingResource: if repo_id does not represent a valid repo
:raise InvalidImporterConfiguration: if the importer cannot be initialized for the given
repo
"""
RepoImporterManager.validate_importer_config(repo_id, importer_type_id, repo_plugin_config)
importer_coll = RepoImporter.get_collection()
repo_obj = model.Repository.objects.get_repo_or_missing_resource(repo_id)
importer_instance, plugin_config = plugin_api.get_importer_by_id(importer_type_id)
# Convention is that a value of None means unset. Remove any keys that
# are explicitly set to None so the plugin will default them.
if repo_plugin_config is not None:
clean_config = dict([(k, v) for k, v in repo_plugin_config.items() if v is not None])
else:
clean_config = None
# Let the importer plugin verify the configuration
call_config = PluginCallConfiguration(plugin_config, clean_config)
transfer_repo = repo_obj.to_transfer_repo()
# Remove old importer if one exists
try:
RepoImporterManager.remove_importer(repo_id)
except MissingResource:
pass # it didn't exist, so no harm done
# Let the importer plugin initialize the repository
try:
importer_instance.importer_added(transfer_repo, call_config)
except Exception:
_logger.exception(
'Error initializing importer [%s] for repo [%s]' % (importer_type_id, repo_id))
raise PulpExecutionException(), None, sys.exc_info()[2]
# Database Update
importer_id = importer_type_id # use the importer name as its repo ID
importer = RepoImporter(repo_id, importer_id, importer_type_id, clean_config)
importer_coll.save(importer)
return importer
class RepoImporterManager(object):
def get_importer(self, repo_id):
"""
Returns metadata about an importer associated with the given repo.
@return: key-value pairs describing the importer in use
@rtype: dict
@raise MissingResource: if the repo does not exist or has no importer associated
"""
importer = RepoImporter.get_collection().find_one({'repo_id': repo_id})
if importer is None:
raise MissingResource(repository=repo_id)
return importer
def get_importers(self, repo_id):
"""
Returns a list of all importers associated with the given repo.
@return: list of key-value pairs describing the importers in use; empty
list if the repo has no importers
@rtype: list of dict
@raise MissingResource: if the given repo doesn't exist
"""
repo = Repo.get_collection().find_one({'id': repo_id})
if repo is None:
raise MissingResource(repo_id)
importers = list(RepoImporter.get_collection().find(
{'repo_id': repo_id}))
return importers
@staticmethod
def find_by_repo_list(repo_id_list):
"""
Returns serialized versions of all importers for given repos. Any
IDs that do not refer to valid repos are ignored and will not
raise an error.
@param repo_id_list: list of importer IDs to fetch
@type repo_id_list: list of str
@return: list of serialized importers
@rtype: list of dict
"""
spec = {'repo_id': {'$in': repo_id_list}}
projection = {'scratchpad': 0}
importers = list(RepoImporter.get_collection().find(spec, projection))
# Process any scheduled syncs and get schedule details using schedule id
for importer in importers:
scheduled_sync_ids = importer.get('scheduled_syncs', None)
if scheduled_sync_ids is not None:
scheduled_sync_details = list(
ScheduledCall.get_collection().find(
{"id": {
"$in": scheduled_sync_ids
}}))
importer['scheduled_syncs'] = [
s["schedule"] for s in scheduled_sync_details
]
return importers
def set_importer(self, repo_id, importer_type_id, repo_plugin_config):
"""
Configures an importer to be used for the given repository.
Keep in mind this method is written assuming single importer for a repo.
The domain model technically supports multiple importers, but this
call is what enforces the single importer behavior.
@param repo_id: identifies the repo
@type repo_id; str
@param importer_type_id: identifies the type of importer being added;
must correspond to an importer loaded at server startup
@type importer_type_id: str
@param repo_plugin_config: configuration values for the importer; may be None
@type repo_plugin_config: dict
@raise MissingResource: if repo_id does not represent a valid repo
@raise InvalidImporterConfiguration: if the importer cannot be
initialized for the given repo
"""
repo_coll = Repo.get_collection()
importer_coll = RepoImporter.get_collection()
# Validation
repo = repo_coll.find_one({'id': repo_id})
if repo is None:
raise MissingResource(repo_id)
if not plugin_api.is_valid_importer(importer_type_id):
raise InvalidValue(['importer_type_id'])
importer_instance, plugin_config = plugin_api.get_importer_by_id(
importer_type_id)
# Convention is that a value of None means unset. Remove any keys that
# are explicitly set to None so the plugin will default them.
if repo_plugin_config is not None:
clean_config = dict([(k, v) for k, v in repo_plugin_config.items()
if v is not None])
else:
clean_config = None
# Let the importer plugin verify the configuration
call_config = PluginCallConfiguration(plugin_config, clean_config)
transfer_repo = common_utils.to_transfer_repo(repo)
transfer_repo.working_dir = common_utils.importer_working_dir(
importer_type_id, repo_id)
query_manager = manager_factory.repo_query_manager()
related_repos = query_manager.find_with_importer_type(importer_type_id)
transfer_related_repos = []
for r in related_repos:
all_configs = [d['config'] for d in r['importers']]
trr = common_utils.to_related_repo(r, all_configs)
transfer_related_repos.append(trr)
try:
result = importer_instance.validate_config(transfer_repo,
call_config,
transfer_related_repos)
# For backward compatibility with plugins that don't yet return the tuple
if isinstance(result, bool):
valid_config = result
message = None
else:
valid_config, message = result
except Exception, e:
_LOG.exception(
'Exception received from importer [%s] while validating config'
% importer_type_id)
raise PulpDataException(e.args), None, sys.exc_info()[2]
if not valid_config:
raise PulpDataException(message)
# Remove old importer if one exists
try:
self.remove_importer(repo_id)
except MissingResource:
pass # it didn't exist, so no harm done
# Let the importer plugin initialize the repository
try:
importer_instance.importer_added(transfer_repo, call_config)
except Exception:
_LOG.exception('Error initializing importer [%s] for repo [%s]' %
(importer_type_id, repo_id))
raise PulpExecutionException(), None, sys.exc_info()[2]
# Database Update
importer_id = importer_type_id # use the importer name as its repo ID
importer = RepoImporter(repo_id, importer_id, importer_type_id,
clean_config)
importer_coll.save(importer, safe=True)
return importer
