def unassociate_by_criteria(cls, repo_id, criteria, notify_plugins=True):
"""
Unassociate units that are matched by the given criteria.
:param repo_id: identifies the repo
:type repo_id: str
:param criteria:
:param notify_plugins: if true, relevant plugins will be informed of the removal
:type notify_plugins: bool
"""
criteria = UnitAssociationCriteria.from_dict(criteria)
repo = model.Repository.objects.get_repo_or_missing_resource(repo_id)
unassociate_units = load_associated_units(repo_id, criteria)
if len(unassociate_units) == 0:
return {}
# Convert the units into transfer units. This happens regardless of whether or not
# the plugin will be notified as it's used to generate the return result.
# If all source types have been converted to mongo, search via new style.
repo_unit_types = set(repo.content_unit_counts.keys())
if repo_unit_types.issubset(set(plugin_api.list_unit_models())):
transfer_units = list(cls._units_from_criteria(repo, criteria))
else:
transfer_units = None
if unassociate_units is not None:
transfer_units = list(create_transfer_units(unassociate_units))
if notify_plugins:
remove_from_importer(repo_id, transfer_units)
unit_map = {} # maps unit_type_id to a list of unit_ids
for unit in unassociate_units:
id_list = unit_map.setdefault(unit["unit_type_id"], [])
id_list.append(unit["unit_id"])
collection = RepoContentUnit.get_collection()
for unit_type_id, unit_ids in unit_map.items():
spec = {"repo_id": repo_id, "unit_type_id": unit_type_id, "unit_id": {"$in": unit_ids}}
collection.remove(spec)
unique_count = sum(
1
for unit_id in unit_ids
if not RepoUnitAssociationManager.association_exists(repo_id, unit_id, unit_type_id)
)
if not unique_count:
continue
repo_controller.update_unit_count(repo_id, unit_type_id, -unique_count)
repo_controller.update_last_unit_removed(repo_id)
# Match the return type/format as copy
serializable_units = [u.to_id_dict() for u in transfer_units]
return {"units_successful": serializable_units}
def unassociate_by_criteria(repo_id, criteria, notify_plugins=True):
"""
Unassociate units that are matched by the given criteria.
:param repo_id: identifies the repo
:type repo_id: str
:param criteria:
:param notify_plugins: if true, relevant plugins will be informed of the removal
:type notify_plugins: bool
"""
criteria = UnitAssociationCriteria.from_dict(criteria)
repo = model.Repository.objects.get_repo_or_missing_resource(repo_id)
unassociate_units = load_associated_units(repo_id, criteria)
if len(unassociate_units) == 0:
return {}
# Convert the units into transfer units. This happens regardless of whether or not
# the plugin will be notified as it's used to generate the return result.
# If all source types have been converted to mongo, search via new style.
repo_unit_types = set(repo.content_unit_counts.keys())
if repo_unit_types.issubset(set(plugin_api.list_unit_models())):
transfer_units = list(
RepoUnitAssociationManager._units_from_criteria(
repo, criteria))
else:
transfer_units = None
if unassociate_units is not None:
transfer_units = list(create_transfer_units(unassociate_units))
if notify_plugins:
remove_from_importer(repo_id, transfer_units)
unit_map = {} # maps unit_type_id to a list of unit_ids
for unit in unassociate_units:
id_list = unit_map.setdefault(unit['unit_type_id'], [])
id_list.append(unit['unit_id'])
collection = RepoContentUnit.get_collection()
for unit_type_id, unit_ids in unit_map.items():
spec = {
'repo_id': repo_id,
'unit_type_id': unit_type_id,
'unit_id': {
'$in': unit_ids
}
}
collection.remove(spec)
repo_controller.update_last_unit_removed(repo_id)
repo_controller.rebuild_content_unit_counts(repo)
# Match the return type/format as copy
serializable_units = [u.to_id_dict() for u in transfer_units]
return {'units_successful': serializable_units}
def delete_all_orphans():
"""
Delete all orphaned content units.
"""
for content_type_id in content_types_db.all_type_ids():
OrphanManager.delete_orphans_by_type(content_type_id)
for content_type_id in plugin_api.list_unit_models():
OrphanManager.delete_orphan_content_units_by_type(content_type_id)
def delete_all_orphans():
"""
Delete all orphaned content units.
"""
for content_type_id in content_types_db.all_type_ids():
OrphanManager.delete_orphans_by_type(content_type_id)
for content_type_id in plugin_api.list_unit_models():
OrphanManager.delete_orphan_content_units_by_type(content_type_id)
def orphans_summary(self):
"""
Return a summary of the orphaned units as a dictionary of
content type -> number of orphaned units
:return: summary of orphaned units
:rtype: dict
"""
summary = {}
for content_type_id in content_types_db.all_type_ids():
summary[content_type_id] = self.orphans_count_by_type(content_type_id)
for content_type_id in plugin_api.list_unit_models():
summary[content_type_id] = self.orphans_count_by_type(content_type_id)
return summary
def orphans_summary(self):
"""
Return a summary of the orphaned units as a dictionary of
content type -> number of orphaned units
:return: summary of orphaned units
:rtype: dict
"""
summary = {}
for content_type_id in content_types_db.all_type_ids():
summary[content_type_id] = self.orphans_count_by_type(content_type_id)
for content_type_id in plugin_api.list_unit_models():
summary[content_type_id] = self.orphans_count_by_type(content_type_id)
return summary
def delete_all_orphans():
"""
Delete all orphaned content units.
:return: count of units deleted indexed by content_type_id
:rtype: dict
"""
ret = {}
for content_type_id in content_types_db.all_type_ids():
count = OrphanManager.delete_orphans_by_type(content_type_id)
if count > 0:
ret[content_type_id] = count
for content_type_id in plugin_api.list_unit_models():
count = OrphanManager.delete_orphan_content_units_by_type(content_type_id)
if count > 0:
ret[content_type_id] = count
return ret
def delete_all_orphans():
"""
Delete all orphaned content units.
:return: count of units deleted indexed by content_type_id
:rtype: dict
"""
ret = {}
for content_type_id in content_types_db.all_type_ids():
count = OrphanManager.delete_orphans_by_type(content_type_id)
if count > 0:
ret[content_type_id] = count
for content_type_id in plugin_api.list_unit_models():
count = OrphanManager.delete_orphan_content_units_by_type(content_type_id)
if count > 0:
ret[content_type_id] = count
return ret
def associate_from_repo(source_repo_id,
dest_repo_id,
criteria=None,
import_config_override=None):
"""
Creates associations in a repository based on the contents of a source
repository. Units from the source repository can be filtered by
specifying a criteria object.
The destination repository must have an importer that can support
the types of units being associated. This is done by analyzing the
unit list and the importer metadata and takes place before the
destination repository is called.
Pulp does not actually perform the associations as part of this call.
The unit list is determined and passed to the destination repository's
importer. It is the job of the importer to make the associate calls
back into Pulp where applicable.
If criteria is None, the effect of this call is to copy the source
repository's associations into the destination repository.
:param source_repo_id: identifies the source repository
:type source_repo_id: str
:param dest_repo_id: identifies the destination repository
:type dest_repo_id: str
:param criteria: optional; if specified, will filter the units retrieved from
the source repository
:type criteria: pulp.server.db.model.criteria.UnitAssociationCriteria
:param import_config_override: optional config containing values to use for this import only
:type import_config_override: dict
:return: dict with key 'units_successful' whose
value is a list of unit keys that were copied.
units that were associated by this operation
:rtype: dict
:raise MissingResource: if either of the specified repositories don't exist
"""
importer_manager = manager_factory.repo_importer_manager()
source_repo = model.Repository.objects.get_repo_or_missing_resource(
source_repo_id)
dest_repo = model.Repository.objects.get_repo_or_missing_resource(
dest_repo_id)
# This will raise MissingResource if there isn't one, which is the
# behavior we want this method to exhibit, so just let it bubble up.
dest_repo_importer = importer_manager.get_importer(dest_repo_id)
source_repo_importer = importer_manager.get_importer(source_repo_id)
# The docs are incorrect on the list_importer_types call; it actually
# returns a dict with the types under key "types" for some reason.
supported_type_ids = set(
plugin_api.list_importer_types(
dest_repo_importer['importer_type_id'])['types'])
# Get the unit types from the repo source repo
source_repo_unit_types = set(source_repo.content_unit_counts.keys())
# Now we can make sure the destination repository's importer is capable
# of importing either the selected units or all of the units
if not source_repo_unit_types.issubset(supported_type_ids):
raise exceptions.PulpCodedException(
error_code=error_codes.PLP0000,
message=
'The the target importer does not support the types from the source'
)
transfer_units = None
# If criteria is specified, retrieve the list of units now
if criteria is not None:
# if all source types have been converted to mongo - search via new style
if source_repo_unit_types.issubset(
set(plugin_api.list_unit_models())):
association_q = mongoengine.Q(
__raw__=criteria.association_spec)
unit_q = mongoengine.Q(__raw__=criteria.unit_spec)
transfer_units = repo_controller.find_repo_content_units(
repository=source_repo,
repo_content_unit_q=association_q,
units_q=unit_q,
yield_content_unit=True)
else:
# else, search via old style
associate_us = load_associated_units(source_repo_id, criteria)
# If units were supposed to be filtered but none matched, we're done
if len(associate_us) == 0:
# Return an empty list to indicate nothing was copied
return {'units_successful': []}
# Convert all of the units into the plugin standard representation if
# a filter was specified
transfer_units = None
if associate_us is not None:
transfer_units = create_transfer_units(associate_us)
# Convert the two repos into the plugin API model
transfer_dest_repo = dest_repo.to_transfer_repo()
transfer_source_repo = source_repo.to_transfer_repo()
# Invoke the importer
importer_instance, plugin_config = plugin_api.get_importer_by_id(
dest_repo_importer['importer_type_id'])
call_config = PluginCallConfiguration(plugin_config,
dest_repo_importer['config'],
import_config_override)
conduit = ImportUnitConduit(source_repo_id, dest_repo_id,
source_repo_importer['id'],
dest_repo_importer['id'])
try:
copied_units = importer_instance.import_units(transfer_source_repo,
transfer_dest_repo,
conduit,
call_config,
units=transfer_units)
unit_ids = [u.to_id_dict() for u in copied_units]
return {'units_successful': unit_ids}
except Exception:
msg = _(
'Exception from importer [%(i)s] while importing units into repository [%(r)s]'
)
msg_dict = {
'i': dest_repo_importer['importer_type_id'],
'r': dest_repo_id
}
logger.exception(msg % msg_dict)
raise exceptions.PulpExecutionException(), None, sys.exc_info()[2]
def associate_from_repo(source_repo_id, dest_repo_id, criteria,
import_config_override=None):
"""
Creates associations in a repository based on the contents of a source
repository. Units from the source repository can be filtered by
specifying a criteria object.
The destination repository must have an importer that can support
the types of units being associated. This is done by analyzing the
unit list and the importer metadata and takes place before the
destination repository is called.
Pulp does not actually perform the associations as part of this call.
The unit list is determined and passed to the destination repository's
importer. It is the job of the importer to make the associate calls
back into Pulp where applicable.
If criteria is None, the effect of this call is to copy the source
repository's associations into the destination repository.
:param source_repo_id: identifies the source repository
:type source_repo_id: str
:param dest_repo_id: identifies the destination repository
:type dest_repo_id: str
:param criteria: optional; if specified, will filter the units retrieved from
the source repository
:type criteria: pulp.server.db.model.criteria.UnitAssociationCriteria
:param import_config_override: optional config containing values to use for this import only
:type import_config_override: dict
:return: dict with key 'units_successful' whose
value is a list of unit keys that were copied.
units that were associated by this operation
:rtype: dict
:raise MissingResource: if either of the specified repositories don't exist
"""
criteria = UnitAssociationCriteria.from_dict(criteria)
source_repo = model.Repository.objects.get_repo_or_missing_resource(source_repo_id)
dest_repo = model.Repository.objects.get_repo_or_missing_resource(dest_repo_id)
dest_repo_importer = model.Importer.objects.get_or_404(repo_id=dest_repo_id)
source_repo_importer = model.Importer.objects.get_or_404(repo_id=source_repo_id)
# The docs are incorrect on the list_importer_types call; it actually
# returns a dict with the types under key "types" for some reason.
supported_type_ids = set(plugin_api.list_importer_types(
dest_repo_importer.importer_type_id)['types'])
# Get the unit types from the repo source repo
source_repo_unit_types = set(source_repo.content_unit_counts.keys())
# Now we can make sure the destination repository's importer is capable
# of importing either the selected units or all of the units
if not source_repo_unit_types.issubset(supported_type_ids):
raise exceptions.PulpCodedException(error_code=error_codes.PLP0044)
transfer_units = None
# if all source types have been converted to mongo - search via new style
if source_repo_unit_types.issubset(set(plugin_api.list_unit_models())):
transfer_units = RepoUnitAssociationManager._units_from_criteria(source_repo, criteria)
else:
# else, search via old style
associate_us = load_associated_units(source_repo_id, criteria)
# If units were supposed to be filtered but none matched, we're done
if len(associate_us) == 0:
# Return an empty list to indicate nothing was copied
return {'units_successful': []}
# Convert all of the units into the plugin standard representation if
# a filter was specified
transfer_units = None
if associate_us is not None:
transfer_units = create_transfer_units(associate_us)
# Convert the two repos into the plugin API model
transfer_dest_repo = dest_repo.to_transfer_repo()
transfer_source_repo = source_repo.to_transfer_repo()
# Invoke the importer
importer_instance, plugin_config = plugin_api.get_importer_by_id(
dest_repo_importer.importer_type_id)
call_config = PluginCallConfiguration(plugin_config, dest_repo_importer.config,
import_config_override)
conduit = ImportUnitConduit(
source_repo_id, dest_repo_id, source_repo_importer.importer_type_id,
dest_repo_importer.importer_type_id)
try:
copied_units = importer_instance.import_units(
transfer_source_repo, transfer_dest_repo, conduit, call_config,
units=transfer_units)
if isinstance(copied_units, tuple):
suc_units_ids = [u.to_id_dict() for u in copied_units[0] if u is not None]
unsuc_units_ids = [u.to_id_dict() for u in copied_units[1]]
repo_controller.rebuild_content_unit_counts(dest_repo)
return {'units_successful': suc_units_ids,
'units_failed_signature_filter': unsuc_units_ids}
unit_ids = [u.to_id_dict() for u in copied_units if u is not None]
repo_controller.rebuild_content_unit_counts(dest_repo)
return {'units_successful': unit_ids}
except Exception as e:
msg = _('Exception from importer [%(i)s] while importing units into repository [%(r)s]')
msg_dict = {'i': dest_repo_importer.importer_type_id, 'r': dest_repo_id}
logger.exception(msg % msg_dict)
raise (e, None, sys.exc_info()[2])
def associate_from_repo(source_repo_id, dest_repo_id, criteria=None,
import_config_override=None):
"""
Creates associations in a repository based on the contents of a source
repository. Units from the source repository can be filtered by
specifying a criteria object.
The destination repository must have an importer that can support
the types of units being associated. This is done by analyzing the
unit list and the importer metadata and takes place before the
destination repository is called.
Pulp does not actually perform the associations as part of this call.
The unit list is determined and passed to the destination repository's
importer. It is the job of the importer to make the associate calls
back into Pulp where applicable.
If criteria is None, the effect of this call is to copy the source
repository's associations into the destination repository.
:param source_repo_id: identifies the source repository
:type source_repo_id: str
:param dest_repo_id: identifies the destination repository
:type dest_repo_id: str
:param criteria: optional; if specified, will filter the units retrieved from
the source repository
:type criteria: pulp.server.db.model.criteria.UnitAssociationCriteria
:param import_config_override: optional config containing values to use for this import only
:type import_config_override: dict
:return: dict with key 'units_successful' whose
value is a list of unit keys that were copied.
units that were associated by this operation
:rtype: dict
:raise MissingResource: if either of the specified repositories don't exist
"""
importer_manager = manager_factory.repo_importer_manager()
source_repo = model.Repository.objects.get_repo_or_missing_resource(source_repo_id)
dest_repo = model.Repository.objects.get_repo_or_missing_resource(dest_repo_id)
# This will raise MissingResource if there isn't one, which is the
# behavior we want this method to exhibit, so just let it bubble up.
dest_repo_importer = importer_manager.get_importer(dest_repo_id)
source_repo_importer = importer_manager.get_importer(source_repo_id)
# The docs are incorrect on the list_importer_types call; it actually
# returns a dict with the types under key "types" for some reason.
supported_type_ids = set(plugin_api.list_importer_types(
dest_repo_importer['importer_type_id'])['types'])
# Get the unit types from the repo source repo
source_repo_unit_types = set(source_repo.content_unit_counts.keys())
# Now we can make sure the destination repository's importer is capable
# of importing either the selected units or all of the units
if not source_repo_unit_types.issubset(supported_type_ids):
raise exceptions.PulpCodedException(
error_code=error_codes.PLP0000,
message='The the target importer does not support the types from the source')
transfer_units = None
# If criteria is specified, retrieve the list of units now
if criteria is not None:
# if all source types have been converted to mongo - search via new style
if source_repo_unit_types.issubset(set(plugin_api.list_unit_models())):
association_q = mongoengine.Q(__raw__=criteria.association_spec)
unit_q = mongoengine.Q(__raw__=criteria.unit_spec)
transfer_units = repo_controller.find_repo_content_units(
repository=source_repo,
repo_content_unit_q=association_q,
units_q=unit_q,
yield_content_unit=True)
else:
# else, search via old style
associate_us = load_associated_units(source_repo_id, criteria)
# If units were supposed to be filtered but none matched, we're done
if len(associate_us) == 0:
# Return an empty list to indicate nothing was copied
return {'units_successful': []}
# Convert all of the units into the plugin standard representation if
# a filter was specified
transfer_units = None
if associate_us is not None:
associated_unit_type_ids = calculate_associated_type_ids(source_repo_id,
associate_us)
transfer_units = create_transfer_units(associate_us, associated_unit_type_ids)
# Convert the two repos into the plugin API model
transfer_dest_repo = dest_repo.to_transfer_repo()
transfer_source_repo = source_repo.to_transfer_repo()
# Invoke the importer
importer_instance, plugin_config = plugin_api.get_importer_by_id(
dest_repo_importer['importer_type_id'])
call_config = PluginCallConfiguration(plugin_config, dest_repo_importer['config'],
import_config_override)
conduit = ImportUnitConduit(
source_repo_id, dest_repo_id, source_repo_importer['id'], dest_repo_importer['id'])
try:
copied_units = importer_instance.import_units(
transfer_source_repo, transfer_dest_repo, conduit, call_config,
units=transfer_units)
unit_ids = [u.to_id_dict() for u in copied_units]
return {'units_successful': unit_ids}
except Exception:
msg = _('Exception from importer [%(i)s] while importing units into repository [%(r)s]')
msg_dict = {'i': dest_repo_importer['importer_type_id'], 'r': dest_repo_id}
logger.exception(msg % msg_dict)
raise exceptions.PulpExecutionException(), None, sys.exc_info()[2]
def associate_from_repo(cls, source_repo_id, dest_repo_id, criteria,
import_config_override=None):
"""
Creates associations in a repository based on the contents of a source
repository. Units from the source repository can be filtered by
specifying a criteria object.
The destination repository must have an importer that can support
the types of units being associated. This is done by analyzing the
unit list and the importer metadata and takes place before the
destination repository is called.
Pulp does not actually perform the associations as part of this call.
The unit list is determined and passed to the destination repository's
importer. It is the job of the importer to make the associate calls
back into Pulp where applicable.
If criteria is None, the effect of this call is to copy the source
repository's associations into the destination repository.
:param source_repo_id: identifies the source repository
:type source_repo_id: str
:param dest_repo_id: identifies the destination repository
:type dest_repo_id: str
:param criteria: optional; if specified, will filter the units retrieved from
the source repository
:type criteria: pulp.server.db.model.criteria.UnitAssociationCriteria
:param import_config_override: optional config containing values to use for this import only
:type import_config_override: dict
:return: dict with key 'units_successful' whose
value is a list of unit keys that were copied.
units that were associated by this operation
:rtype: dict
:raise MissingResource: if either of the specified repositories don't exist
"""
criteria = UnitAssociationCriteria.from_dict(criteria)
source_repo = model.Repository.objects.get_repo_or_missing_resource(source_repo_id)
dest_repo = model.Repository.objects.get_repo_or_missing_resource(dest_repo_id)
dest_repo_importer = model.Importer.objects.get_or_404(repo_id=dest_repo_id)
source_repo_importer = model.Importer.objects.get_or_404(repo_id=source_repo_id)
# The docs are incorrect on the list_importer_types call; it actually
# returns a dict with the types under key "types" for some reason.
supported_type_ids = set(plugin_api.list_importer_types(
dest_repo_importer.importer_type_id)['types'])
# Get the unit types from the repo source repo
source_repo_unit_types = set(source_repo.content_unit_counts.keys())
# Now we can make sure the destination repository's importer is capable
# of importing either the selected units or all of the units
if not source_repo_unit_types.issubset(supported_type_ids):
raise exceptions.PulpCodedException(error_code=error_codes.PLP0044)
transfer_units = None
# if all source types have been converted to mongo - search via new style
if source_repo_unit_types.issubset(set(plugin_api.list_unit_models())):
transfer_units = cls._units_from_criteria(source_repo, criteria)
else:
# else, search via old style
associate_us = load_associated_units(source_repo_id, criteria)
# If units were supposed to be filtered but none matched, we're done
if len(associate_us) == 0:
# Return an empty list to indicate nothing was copied
return {'units_successful': []}
# Convert all of the units into the plugin standard representation if
# a filter was specified
transfer_units = None
if associate_us is not None:
transfer_units = create_transfer_units(associate_us)
# Convert the two repos into the plugin API model
transfer_dest_repo = dest_repo.to_transfer_repo()
transfer_source_repo = source_repo.to_transfer_repo()
# Invoke the importer
importer_instance, plugin_config = plugin_api.get_importer_by_id(
dest_repo_importer.importer_type_id)
call_config = PluginCallConfiguration(plugin_config, dest_repo_importer.config,
import_config_override)
conduit = ImportUnitConduit(
source_repo_id, dest_repo_id, source_repo_importer.importer_type_id,
dest_repo_importer.importer_type_id)
try:
copied_units = importer_instance.import_units(
transfer_source_repo, transfer_dest_repo, conduit, call_config,
units=transfer_units)
if isinstance(copied_units, tuple):
suc_units_ids = [u.to_id_dict() for u in copied_units[0] if u is not None]
unsuc_units_ids = [u.to_id_dict() for u in copied_units[1]]
repo_controller.rebuild_content_unit_counts(dest_repo)
return {'units_successful': suc_units_ids,
'units_failed_signature_filter': unsuc_units_ids}
unit_ids = [u.to_id_dict() for u in copied_units if u is not None]
repo_controller.rebuild_content_unit_counts(dest_repo)
return {'units_successful': unit_ids}
except Exception:
msg = _('Exception from importer [%(i)s] while importing units into repository [%(r)s]')
msg_dict = {'i': dest_repo_importer.importer_type_id, 'r': dest_repo_id}
logger.exception(msg % msg_dict)
raise exceptions.PulpExecutionException(), None, sys.exc_info()[2]
def test_list_unit_models(self, mock_manager, mock_is_initialized):
mock_is_initialized.return_value = True
mock_manager.unit_models = {'foo': 'apples', 'bar': 'pears'}
return_val = api.list_unit_models()
self.assertEquals(return_val, ['foo', 'bar'])
