def __unit_ids_to_plugin_unit_keys(self, unit_ids_by_type, repo_ids):
"""
Parse a dictionary of unit ids keyed by content type id and return a dictionary of
corresponding plugin unit keys keyed by content type id.
:param unit_ids_by_type: dictionary of <content type id> : <list of unit ids>
:type unit_ids_by_type: dict
:return: if units are specified, return the corresponding plugin unit_keys. If unit_ids_by_type dict
is empty, return plugin unit keys corresponging to all units in given repo ids.
If unit ids list for a particular unit type is empty, return all plugin unit_keys
in given repo ids with that unit type.
:rtype: dict
"""
repo_unit_association_query_manager = managers.repo_unit_association_query_manager()
content_query_manager = managers.content_query_manager()
result_unit_keys = {}
if unit_ids_by_type is not None:
for unit_type_id, unit_ids in unit_ids_by_type.items():
# Get unit type specific collection
collection = content_query_manager.get_content_unit_collection(type_id=unit_type_id)
type_def = content_types_db.type_definition(unit_type_id)
if not unit_ids:
# If unit_list is empty for a unit_type, consider all units of specific type
criteria = UnitAssociationCriteria(unit_fields = ['unit_id'])
for repo_id in repo_ids:
repo_units = repo_unit_association_query_manager.get_units_by_type(repo_id, unit_type_id, criteria)
# Get metadata for each unit from type specific collection
pulp_units = [collection.find_one({'_id': u['unit_id']}) for u in repo_units]
# Convert pulp units to plugin unit keys
plugin_unit_keys = [common_utils.to_plugin_unit(u, type_def).unit_key for u in pulp_units]
result_unit_keys.setdefault(unit_type_id, []).extend(plugin_unit_keys)
else:
# Get metadata for each unit from type specific collection
pulp_units = [collection.find_one({'_id': unit_id}) for unit_id in unit_ids]
# Convert pulp units to plugin unit keys
plugin_unit_keys = [common_utils.to_plugin_unit(u, type_def).unit_key for u in pulp_units]
result_unit_keys.setdefault(unit_type_id, []).extend(plugin_unit_keys)
else:
# If units are not specified, consider all units in given repos.
for repo_id in repo_ids:
all_unit_type_ids = content_types_db.all_type_ids()
for unit_type_id in all_unit_type_ids:
criteria = UnitAssociationCriteria(type_ids=[unit_type_id], unit_fields = ['unit_id', 'unit_type_id'])
repo_units = repo_unit_association_query_manager.get_units(repo_id, criteria)
# Get unit metadata for each unit from type specific collection
collection = content_query_manager.get_content_unit_collection(type_id=unit_type_id)
pulp_units = [collection.find_one({'_id': u['unit_id']}) for u in repo_units]
# Convert pulp units to plugin unit keys
type_def = content_types_db.type_definition(unit_type_id)
plugin_unit_keys = [common_utils.to_plugin_unit(u, type_def).unit_key for u in pulp_units]
result_unit_keys.setdefault(unit_type_id, []).extend(plugin_unit_keys)
return result_unit_keys
def __parse_units(self, user_units, repo_ids):
"""
Parse units specified by user and return a dictionary of all plugin unit_keys
to be considered for applicability keyed by unit_type_id.
:param user_units: dictionary of unit metadata filters keyed by unit-type-id specified by user
:type user_units: dict
:return: if specific units are specified, return the corresponding plugin unit_keys. If units dict is empty,
return all plugin unit_keys corresponging to units in given repo ids keyed by unit_type_id.
If units list for a particular unit type in units is empty, return all plugin unit_keys
in given repo ids with that unit type keyed by unit_type_id.
:rtype: dict
"""
repo_unit_association_query_manager = managers.repo_unit_association_query_manager()
content_query_manager = managers.content_query_manager()
result_unit_keys = {}
if user_units is not None:
for unit_type_id, unit_list in user_units.items():
# Get unit type specific collection
collection = content_query_manager.get_content_unit_collection(type_id=unit_type_id)
type_def = content_types_db.type_definition(unit_type_id)
if not unit_list:
# If unit_list is empty for a unit_type, consider all units of specific type
criteria = UnitAssociationCriteria(unit_fields = ['unit_id'])
for repo_id in repo_ids:
repo_units = repo_unit_association_query_manager.get_units_by_type(repo_id, unit_type_id, criteria)
# Get unit metadata for each unit from type specific collection
pulp_units = [collection.find_one({'_id': u['unit_id']}) for u in repo_units]
plugin_units = [common_utils.to_plugin_unit(u, type_def) for u in pulp_units]
plugin_unit_keys = [u.unit_key for u in plugin_units]
result_unit_keys.setdefault(unit_type_id, []).extend(plugin_unit_keys)
else:
for unit in unit_list:
criteria = UnitAssociationCriteria(unit_filters=unit, unit_fields = ['unit_id'])
for repo_id in repo_ids:
repo_units = repo_unit_association_query_manager.get_units_by_type(repo_id, unit_type_id, criteria)
# Get unit metadata for each unit from type specific collection
pulp_units = [collection.find_one({'_id': u['unit_id']}) for u in repo_units]
plugin_units = [common_utils.to_plugin_unit(u, type_def) for u in pulp_units]
plugin_unit_keys = [u.unit_key for u in plugin_units]
result_unit_keys.setdefault(unit_type_id, []).extend(plugin_unit_keys)
else:
# If units are not specified, consider all units in repo_ids list.
for repo_id in repo_ids:
criteria = UnitAssociationCriteria(unit_fields = ['unit_id','unit_type_id'])
repo_units = repo_unit_association_query_manager.get_units(repo_id, criteria)
# Get unit metadata for each unit from type specific collection
for repo_unit in repo_units:
collection = content_query_manager.get_content_unit_collection(type_id=repo_unit['unit_type_id'])
type_def = content_types_db.type_definition(repo_unit['unit_type_id'])
pulp_unit = collection.find_one({'_id': repo_unit['unit_id']})
plugin_unit = common_utils.to_plugin_unit(pulp_unit, type_def)
result_unit_keys.setdefault(repo_unit['unit_type_id'], []).append(plugin_unit.unit_key)
return result_unit_keys
def search_all_units(self, type_id, criteria):
"""
Searches for units of a given type in the server, regardless of their
associations to any repositories.
@param type_id: indicates the type of units being retrieved
@type type_id: str
@param criteria: used to query which units are returned
@type criteria: pulp.server.db.model.criteria.Criteria
@return: list of unit instances
@rtype: list of L{Unit}
"""
try:
query_manager = manager_factory.content_query_manager()
units = query_manager.find_by_criteria(type_id, criteria)
type_def = types_db.type_definition(type_id)
transfer_units = []
for pulp_unit in units:
u = common_utils.to_plugin_unit(pulp_unit, type_def)
transfer_units.append(u)
return transfer_units
except Exception, e:
logger.exception(
'Exception from server requesting all units of type [%s]' %
type_id)
raise self.exception_class(e), None, sys.exc_info()[2]
def search_all_units(self, type_id, criteria):
"""
Searches for units of a given type in the server, regardless of their
associations to any repositories.
@param type_id: indicates the type of units being retrieved
@type type_id: str
@param criteria: used to query which units are returned
@type criteria: pulp.server.db.model.criteria.Criteria
@return: list of unit instances
@rtype: list of L{Unit}
"""
try:
query_manager = manager_factory.content_query_manager()
units = query_manager.find_by_criteria(type_id, criteria)
type_def = types_db.type_definition(type_id)
transfer_units = []
for pulp_unit in units:
u = common_utils.to_plugin_unit(pulp_unit, type_def)
transfer_units.append(u)
return transfer_units
except Exception, e:
_LOG.exception('Exception from server requesting all units of type [%s]' % type_id)
raise self.exception_class(e), None, sys.exc_info()[2]
def do_get_repo_units(repo_id, criteria, exception_class):
"""
Performs a repo unit association query. This is split apart so we can have
custom mixins with different signatures.
"""
try:
association_query_manager = manager_factory.repo_unit_association_query_manager()
units = association_query_manager.get_units(repo_id, criteria=criteria)
all_units = []
# Load all type definitions in use so we don't hammer the database
unique_type_defs = set([u['unit_type_id'] for u in units])
type_defs = {}
for def_id in unique_type_defs:
type_def = types_db.type_definition(def_id)
type_defs[def_id] = type_def
# Convert to transfer object
for unit in units:
type_id = unit['unit_type_id']
u = common_utils.to_plugin_unit(unit, type_defs[type_id])
all_units.append(u)
return all_units
except Exception, e:
_LOG.exception('Exception from server requesting all content units for repository [%s]' % repo_id)
raise exception_class(e), None, sys.exc_info()[2]
def find_unit_by_unit_key(self, type_id, unit_key):
"""
Finds a unit based on its unit key. If more than one unit comes back,
an exception will be raised.
@param type_id: indicates the type of units being retrieved
@type type_id: str
@param unit_key: the unit key for the unit
@type unit_key: dict
@return: a single unit
@rtype: L{Unit}
"""
content_query_manager = manager_factory.content_query_manager()
try:
# this call returns a unit or raises MissingResource
existing_unit = content_query_manager.get_content_unit_by_keys_dict(
type_id, unit_key)
unit_key_fields = units_controller.get_unit_key_fields_for_type(
type_id)
plugin_unit = common_utils.to_plugin_unit(existing_unit, type_id,
unit_key_fields)
return plugin_unit
except pulp_exceptions.MissingResource:
return None
def create_transfer_units(associate_units, associated_unit_type_ids):
type_defs = {}
for def_id in associated_unit_type_ids:
type_def = types_db.type_definition(def_id)
type_defs[def_id] = type_def
transfer_units = []
for unit in associate_units:
type_id = unit['unit_type_id']
u = conduit_common_utils.to_plugin_unit(unit, type_defs[type_id])
transfer_units.append(u)
return transfer_units
def get_source_units(self, criteria=None):
"""
Returns the collection of content units associated with the source
repository for a unit import.
Units returned from this call will have the id field populated and are
useable in any calls in this conduit that require the id field.
@param criteria: used to scope the returned results or the data within;
the Criteria class can be imported from this module
@type criteria: L{Criteria}
@return: list of unit instances
@rtype: list of L{AssociatedUnit}
"""
try:
units = self.__association_query_manager.get_units_across_types(self.source_repo_id, criteria=criteria)
all_units = []
# Load all type definitions in use so we don't hammer the database
unique_type_defs = set([u['unit_type_id'] for u in units])
type_defs = {}
for def_id in unique_type_defs:
type_def = types_db.type_definition(def_id)
type_defs[def_id] = type_def
# Convert to transfer object
for unit in units:
type_id = unit['unit_type_id']
u = common_utils.to_plugin_unit(unit, type_defs[type_id])
all_units.append(u)
return all_units
except Exception, e:
_LOG.exception('Exception from server requesting all content units for repository [%s]' % self.repo_id)
raise UnitImportConduitException(e), None, sys.exc_info()[2]
def find_unit_by_unit_key(self, type_id, unit_key):
"""
Finds a unit based on its unit key. If more than one unit comes back,
an exception will be raised.
@param type_id: indicates the type of units being retrieved
@type type_id: str
@param unit_key: the unit key for the unit
@type unit_key: dict
@return: a single unit
@rtype: L{Unit}
"""
content_query_manager = manager_factory.content_query_manager()
try:
# this call returns a unit or raises MissingResource
existing_unit = content_query_manager.get_content_unit_by_keys_dict(type_id, unit_key)
unit_key_fields = units_controller.get_unit_key_fields_for_type(type_id)
plugin_unit = common_utils.to_plugin_unit(existing_unit, type_id, unit_key_fields)
return plugin_unit
except pulp_exceptions.MissingResource:
return None
def resolve_dependencies_by_units(self, repo_id, units, options):
"""
Calculates dependencies for the given set of units in the given
repository.
@param repo_id: identifies the repository
@type repo_id: str
@param units: list of database representations of units to resolve
dependencies for
@type units: list
@param options: dict of options to pass the importer to drive the resolution
@type options: dict or None
@return: report from the plugin
@rtype: object
@raise MissingResource: if the repo does not exist or does not have
an importer
"""
# Validation
repo_query_manager = manager_factory.repo_query_manager()
importer_manager = manager_factory.repo_importer_manager()
# The following will raise MissingResource as appropriate
repo = repo_query_manager.get_repository(repo_id)
repo_importer = importer_manager.get_importer(repo_id)
try:
importer_instance, plugin_config = plugin_api.get_importer_by_id(repo_importer['importer_type_id'])
except plugin_exceptions.PluginNotFound:
raise MissingResource(repo_id), None, sys.exc_info()[2]
# Package for the importer call
call_config = PluginCallConfiguration(plugin_config, repo_importer['config'], options)
transfer_repo = common_utils.to_transfer_repo(repo)
transfer_repo.working_dir = common_utils.importer_working_dir(repo_importer['importer_type_id'], repo_id, mkdir=True)
conduit = DependencyResolutionConduit(repo_id, repo_importer['id'])
# Convert all of the units into the plugin standard representation
transfer_units = []
# Preload all the type defs so we don't hammer the database unnecessarily
type_defs = {}
all_type_def_ids = set([u['unit_type_id'] for u in units])
for def_id in all_type_def_ids:
type_def = types_db.type_definition(def_id)
type_defs[def_id] = type_def
for unit in units:
type_id = unit['unit_type_id']
u = conduit_common_utils.to_plugin_unit(unit, type_defs[type_id])
transfer_units.append(u)
# Invoke the importer
try:
dep_report = importer_instance.resolve_dependencies(transfer_repo, transfer_units, conduit, call_config)
except Exception, e:
raise PulpExecutionException(), None, sys.exc_info()[2]
def associate_from_repo(self, source_repo_id, dest_repo_id, criteria=None):
"""
Creates associations in a repository based on the contents of a source
repository. Units from the source repository can be filtered by
specifying a criteria object.
The destination repository must have an importer that can support
the types of units being associated. This is done by analyzing the
unit list and the importer metadata and takes place before the
destination repository is called.
Pulp does not actually perform the associations as part of this call.
The unit list is determined and passed to the destination repository's
importer. It is the job of the importer to make the associate calls
back into Pulp where applicable.
If criteria is None, the effect of this call is to copy the source
repository's associations into the destination repository.
@param source_repo_id: identifies the source repository
@type source_repo_id: str
@param dest_repo_id: identifies the destination repository
@type dest_repo_id: str
@param criteria: optional; if specified, will filter the units retrieved
from the source repository
@type criteria: L{Criteria}
@raise MissingResource: if either of the specified repositories don't exist
"""
# Validation
repo_query_manager = manager_factory.repo_query_manager()
importer_manager = manager_factory.repo_importer_manager()
dependency_manager = manager_factory.dependency_manager()
association_query_manager = manager_factory.repo_unit_association_query_manager()
source_repo = repo_query_manager.get_repository(source_repo_id)
dest_repo = repo_query_manager.get_repository(dest_repo_id)
# This will raise MissingResource if there isn't one, which is the
# behavior we want this method to exhibit, so just let it bubble up.
dest_repo_importer = importer_manager.get_importer(dest_repo_id)
source_repo_importer = importer_manager.get_importer(source_repo_id)
# The docs are incorrect on the list_importer_types call; it actually
# returns a dict with the types under key "types" for some reason.
supported_type_ids = plugin_api.list_importer_types(dest_repo_importer['importer_type_id'])['types']
# If criteria is specified, retrieve the list of units now
associate_us = None
if criteria is not None:
criteria.association_fields = None
criteria.unit_fields = None
# Retrieve the units to be associated
associate_us = association_query_manager.get_units(source_repo_id, criteria=criteria)
# If units were supposed to be filtered but none matched, we're done
if len(associate_us) is 0:
return
# Now we can make sure the destination repository's importer is capable
# of importing either the selected units or all of the units
if associate_us is not None:
associated_unit_type_ids = set([u['unit_type_id'] for u in associate_us])
else:
association_query_manager = manager_factory.repo_unit_association_query_manager()
# We may want to make a call here that only retrieves the unique
# type IDs instead of all of the units, but for now it doesn't exist
# and I'm not entirely sure this will be a huge problem.
all_units = association_query_manager.get_units(source_repo_id)
associated_unit_type_ids = set(u['unit_type_id'] for u in all_units)
unsupported_types = [t for t in associated_unit_type_ids if t not in supported_type_ids]
if len(unsupported_types) > 0:
raise exceptions.InvalidValue(['types'])
# Convert all of the units into the plugin standard representation if
# a filter was specified
transfer_units = None
if associate_us is not None:
type_defs = {}
for def_id in associated_unit_type_ids:
type_def = types_db.type_definition(def_id)
type_defs[def_id] = type_def
transfer_units = []
for unit in associate_us:
type_id = unit['unit_type_id']
u = conduit_common_utils.to_plugin_unit(unit, type_defs[type_id])
transfer_units.append(u)
# Convert the two repos into the plugin API model
transfer_dest_repo = common_utils.to_transfer_repo(dest_repo)
transfer_dest_repo.working_dir = common_utils.importer_working_dir(dest_repo_importer['importer_type_id'], dest_repo['id'], mkdir=True)
transfer_source_repo = common_utils.to_transfer_repo(source_repo)
transfer_source_repo.working_dir = common_utils.importer_working_dir(source_repo_importer['importer_type_id'], source_repo['id'], mkdir=True)
# Invoke the importer
importer_instance, plugin_config = plugin_api.get_importer_by_id(dest_repo_importer['importer_type_id'])
call_config = PluginCallConfiguration(plugin_config, dest_repo_importer['config'])
conduit = ImportUnitConduit(source_repo_id, dest_repo_id, source_repo_importer['id'], dest_repo_importer['id'])
try:
importer_instance.import_units(transfer_source_repo, transfer_dest_repo, conduit, call_config, units=transfer_units)
except Exception:
_LOG.exception('Exception from importer [%s] while importing units into repository [%s]' % (dest_repo_importer['importer_type_id'], dest_repo_id))
raise exceptions.PulpExecutionException(), None, sys.exc_info()[2]
