def update_unit_count(repo_id, unit_type_id, delta):
"""
Updates the total count of units associated with the repo. Each repo has an attribute
'content_unit_counts' which is a dict where keys are content type IDs and values are the
number of content units of that type in the repository.
example: {'rpm': 12, 'srpm': 3}
:param repo_id: identifies the repo
:type repo_id: str
:param unit_type_id: identifies the unit type to update
:type unit_type_id: str
:param delta: amount by which to increment the total count
:type delta: int
:raises pulp_exceptions.PulpCodedException: if there is an error in the update
"""
atomic_inc_key = 'inc__content_unit_counts__{unit_type_id}'.format(
unit_type_id=unit_type_id)
if delta:
try:
model.Repository.objects(repo_id=repo_id).update_one(
**{atomic_inc_key: delta})
except OperationError:
message = 'There was a problem updating repository %s' % repo_id
raise pulp_exceptions.PulpExecutionException(
message), None, sys.exc_info()[2]
def create_bind_payload(repo_id, distributor_id, binding_config):
"""
Requests the distributor plugin to generate the consumer bind payload.
:param repo_id: identifies the repo being bound
:type repo_id: basestring
:param distributor_id: identifies the distributor
:type distributor_id: basestring
:param binding_config: config applicable only to the binding whose payload is being created
:type binding_config: object or None
:return: payload to pass to the consumer
:rtype: dict
:raise PulpExecutionException: if the distributor raises an error
"""
dist = model.Distributor.objects.get_or_404(repo_id=repo_id, distributor_id=distributor_id)
repo_obj = model.Repository.objects.get_repo_or_missing_resource(repo_id)
distributor_instance, plugin_config = plugin_api.get_distributor_by_id(dist.distributor_type_id)
# Let the distributor plugin verify the configuration
call_config = PluginCallConfiguration(plugin_config, dist.config)
transfer_repo = repo_obj.to_transfer_repo()
try:
return distributor_instance.create_consumer_payload(transfer_repo, call_config,
binding_config)
except Exception:
msg = _('Exception raised from distributor [%(d)s] generating consumer payload')
msg = msg % {'d': distributor_id}
_logger.exception(msg)
raise exceptions.PulpExecutionException(), None, sys.exc_info()[2]
def set_importer(repo_id, importer_type_id, repo_plugin_config):
"""
Configures an importer to be used for the given repository.
:param repo: repository object that the importer should be associated with
:type repo: pulp.server.db.model.Repository
:param importer_type_id: type of importer, must correspond to a plugin loaded at server startup
:type importer_type_id: str
:param repo_plugin_config: configuration values for the importer; may be None
:type repo_plugin_config: dict or None
:return: key-value pairs describing the importer that was set
:rtype: dict
:raises PulpExecutionException: if something goes wrong in the plugin
:raises exceptions.InvalidValue: if the values passed to create the importer are invalid
"""
repo = model.Repository.objects.get_repo_or_missing_resource(repo_id)
if not plugin_api.is_valid_importer(importer_type_id):
raise exceptions.PulpCodedValidationException(
error_code=error_codes.PLP1008, importer_type_id=importer_type_id)
importer_instance, plugin_config = plugin_api.get_importer_by_id(
importer_type_id)
clean_config = clean_config_dict(repo_plugin_config)
# Let the importer plugin verify the configuration
call_config = PluginCallConfiguration(plugin_config, clean_config)
transfer_repo = repo.to_transfer_repo()
validate_importer_config(repo, importer_type_id, clean_config)
try:
remove_importer(repo_id)
except exceptions.MissingResource:
pass # it didn't exist, so no harm done
# Let the importer plugin initialize the importer
try:
importer_instance.importer_added(transfer_repo, call_config)
except Exception:
_logger.exception('Error initializing importer [%s] for repo [%s]' %
(importer_type_id, repo.repo_id))
raise exceptions.PulpExecutionException(), None, sys.exc_info()[2]
importer = model.Importer(repo_id, importer_type_id, clean_config)
try:
importer.save()
except ValidationError, e:
raise exceptions.InvalidValue(e.to_dict().keys())
def associate_from_repo(source_repo_id,
dest_repo_id,
criteria=None,
import_config_override=None):
"""
Creates associations in a repository based on the contents of a source
repository. Units from the source repository can be filtered by
specifying a criteria object.
The destination repository must have an importer that can support
the types of units being associated. This is done by analyzing the
unit list and the importer metadata and takes place before the
destination repository is called.
Pulp does not actually perform the associations as part of this call.
The unit list is determined and passed to the destination repository's
importer. It is the job of the importer to make the associate calls
back into Pulp where applicable.
If criteria is None, the effect of this call is to copy the source
repository's associations into the destination repository.
:param source_repo_id: identifies the source repository
:type source_repo_id: str
:param dest_repo_id: identifies the destination repository
:type dest_repo_id: str
:param criteria: optional; if specified, will filter the units retrieved from
the source repository
:type criteria: UnitAssociationCriteria
:param import_config_override: optional config containing values to use for this import only
:type import_config_override: dict
:return: dict with key 'units_successful' whose
value is a list of unit keys that were copied.
units that were associated by this operation
:rtype: dict
:raise MissingResource: if either of the specified repositories don't exist
"""
importer_manager = manager_factory.repo_importer_manager()
source_repo = model.Repository.objects.get_repo_or_missing_resource(
source_repo_id)
dest_repo = model.Repository.objects.get_repo_or_missing_resource(
dest_repo_id)
# This will raise MissingResource if there isn't one, which is the
# behavior we want this method to exhibit, so just let it bubble up.
dest_repo_importer = importer_manager.get_importer(dest_repo_id)
source_repo_importer = importer_manager.get_importer(source_repo_id)
# The docs are incorrect on the list_importer_types call; it actually
# returns a dict with the types under key "types" for some reason.
supported_type_ids = plugin_api.list_importer_types(
dest_repo_importer['importer_type_id'])['types']
# If criteria is specified, retrieve the list of units now
associate_us = None
if criteria is not None:
associate_us = load_associated_units(source_repo_id, criteria)
# If units were supposed to be filtered but none matched, we're done
if len(associate_us) == 0:
# Return an empty list to indicate nothing was copied
return {'units_successful': []}
# Now we can make sure the destination repository's importer is capable
# of importing either the selected units or all of the units
associated_unit_type_ids = calculate_associated_type_ids(
source_repo_id, associate_us)
unsupported_types = [
t for t in associated_unit_type_ids if t not in supported_type_ids
]
if len(unsupported_types) > 0:
raise exceptions.InvalidValue(['types'])
# Convert all of the units into the plugin standard representation if
# a filter was specified
transfer_units = None
if associate_us is not None:
transfer_units = create_transfer_units(associate_us,
associated_unit_type_ids)
# Convert the two repos into the plugin API model
transfer_dest_repo = dest_repo.to_transfer_repo()
transfer_source_repo = source_repo.to_transfer_repo()
# Invoke the importer
importer_instance, plugin_config = plugin_api.get_importer_by_id(
dest_repo_importer['importer_type_id'])
call_config = PluginCallConfiguration(plugin_config,
dest_repo_importer['config'],
import_config_override)
conduit = ImportUnitConduit(source_repo_id, dest_repo_id,
source_repo_importer['id'],
dest_repo_importer['id'])
try:
copied_units = importer_instance.import_units(transfer_source_repo,
transfer_dest_repo,
conduit,
call_config,
units=transfer_units)
unit_ids = [u.to_id_dict() for u in copied_units]
return {'units_successful': unit_ids}
except Exception:
msg = _(
'Exception from importer [%(i)s] while importing units into repository [%(r)s]'
)
msg = msg % {
'i': dest_repo_importer['importer_type_id'],
'r': dest_repo_id
}
logger.exception(msg)
raise exceptions.PulpExecutionException(), None, sys.exc_info()[2]
RepoContentUnit.get_collection().remove({'repo_id': repo_id},
safe=True)
except Exception, e:
msg = _(
'Error updating one or more database collections while removing repo [%(r)s]'
)
msg = msg % {'r': repo_id}
_logger.exception(msg)
error_tuples.append(e)
# remove the repo from any groups it was a member of
group_manager = manager_factory.repo_group_manager()
group_manager.remove_repo_from_groups(repo_id)
if len(error_tuples) > 0:
pe = pulp_exceptions.PulpExecutionException()
pe.child_exceptions = error_tuples
raise pe
# append unbind itineraries foreach bound consumer
options = {}
consumer_bind_manager = manager_factory.consumer_bind_manager()
additional_tasks = []
errors = []
for bind in consumer_bind_manager.find_by_repo(repo_id):
try:
report = consumer_controller.unbind(bind['consumer_id'],
bind['repo_id'],
bind['distributor_id'],
options)
def associate_from_repo(self, source_repo_id, dest_repo_id, criteria=None, import_config_override=None):
"""
Creates associations in a repository based on the contents of a source
repository. Units from the source repository can be filtered by
specifying a criteria object.
The destination repository must have an importer that can support
the types of units being associated. This is done by analyzing the
unit list and the importer metadata and takes place before the
destination repository is called.
Pulp does not actually perform the associations as part of this call.
The unit list is determined and passed to the destination repository's
importer. It is the job of the importer to make the associate calls
back into Pulp where applicable.
If criteria is None, the effect of this call is to copy the source
repository's associations into the destination repository.
@param source_repo_id: identifies the source repository
@type source_repo_id: str
@param dest_repo_id: identifies the destination repository
@type dest_repo_id: str
@param criteria: optional; if specified, will filter the units retrieved
from the source repository
@type criteria: L{UnitAssociationCriteria}
@param import_config_override: optional config containing values to use
for this import only
@type import_config_override: dict
@return: list of unit IDs (see pulp.plugins.model.Unit.to_id_dict) for units that
were associated by this operation
@rtype: list
@raise MissingResource: if either of the specified repositories don't exist
"""
# Validation
repo_query_manager = manager_factory.repo_query_manager()
importer_manager = manager_factory.repo_importer_manager()
source_repo = repo_query_manager.get_repository(source_repo_id)
dest_repo = repo_query_manager.get_repository(dest_repo_id)
# This will raise MissingResource if there isn't one, which is the
# behavior we want this method to exhibit, so just let it bubble up.
dest_repo_importer = importer_manager.get_importer(dest_repo_id)
source_repo_importer = importer_manager.get_importer(source_repo_id)
# The docs are incorrect on the list_importer_types call; it actually
# returns a dict with the types under key "types" for some reason.
supported_type_ids = plugin_api.list_importer_types(dest_repo_importer['importer_type_id'])['types']
# If criteria is specified, retrieve the list of units now
associate_us = None
if criteria is not None:
associate_us = load_associated_units(source_repo_id, criteria)
# If units were supposed to be filtered but none matched, we're done
if len(associate_us) is 0:
# Return an empty list to indicate nothing was copied
return []
# Now we can make sure the destination repository's importer is capable
# of importing either the selected units or all of the units
associated_unit_type_ids = calculate_associated_type_ids(source_repo_id, associate_us)
unsupported_types = [t for t in associated_unit_type_ids if t not in supported_type_ids]
if len(unsupported_types) > 0:
raise exceptions.InvalidValue(['types'])
# Convert all of the units into the plugin standard representation if
# a filter was specified
transfer_units = None
if associate_us is not None:
transfer_units = create_transfer_units(associate_us, associated_unit_type_ids)
# Convert the two repos into the plugin API model
transfer_dest_repo = common_utils.to_transfer_repo(dest_repo)
transfer_dest_repo.working_dir = common_utils.importer_working_dir(dest_repo_importer['importer_type_id'],
dest_repo['id'], mkdir=True)
transfer_source_repo = common_utils.to_transfer_repo(source_repo)
transfer_source_repo.working_dir = common_utils.importer_working_dir(source_repo_importer['importer_type_id'],
source_repo['id'], mkdir=True)
# Invoke the importer
importer_instance, plugin_config = plugin_api.get_importer_by_id(dest_repo_importer['importer_type_id'])
call_config = PluginCallConfiguration(plugin_config, dest_repo_importer['config'], import_config_override)
login = manager_factory.principal_manager().get_principal()['login']
conduit = ImportUnitConduit(source_repo_id, dest_repo_id, source_repo_importer['id'],
dest_repo_importer['id'], RepoContentUnit.OWNER_TYPE_USER, login)
try:
copied_units = importer_instance.import_units(transfer_source_repo, transfer_dest_repo, conduit,
call_config, units=transfer_units)
unit_ids = [u.to_id_dict() for u in copied_units]
return unit_ids
except Exception:
_LOG.exception('Exception from importer [%s] while importing units into repository [%s]' %
(dest_repo_importer['importer_type_id'], dest_repo_id))
raise exceptions.PulpExecutionException(), None, sys.exc_info()[2]
def associate_from_repo(cls,
source_repo_id,
dest_repo_id,
criteria,
import_config_override=None):
"""
Creates associations in a repository based on the contents of a source
repository. Units from the source repository can be filtered by
specifying a criteria object.
The destination repository must have an importer that can support
the types of units being associated. This is done by analyzing the
unit list and the importer metadata and takes place before the
destination repository is called.
Pulp does not actually perform the associations as part of this call.
The unit list is determined and passed to the destination repository's
importer. It is the job of the importer to make the associate calls
back into Pulp where applicable.
If criteria is None, the effect of this call is to copy the source
repository's associations into the destination repository.
:param source_repo_id: identifies the source repository
:type source_repo_id: str
:param dest_repo_id: identifies the destination repository
:type dest_repo_id: str
:param criteria: optional; if specified, will filter the units retrieved from
the source repository
:type criteria: pulp.server.db.model.criteria.UnitAssociationCriteria
:param import_config_override: optional config containing values to use for this import only
:type import_config_override: dict
:return: dict with key 'units_successful' whose
value is a list of unit keys that were copied.
units that were associated by this operation
:rtype: dict
:raise MissingResource: if either of the specified repositories don't exist
"""
criteria = UnitAssociationCriteria.from_dict(criteria)
source_repo = model.Repository.objects.get_repo_or_missing_resource(
source_repo_id)
dest_repo = model.Repository.objects.get_repo_or_missing_resource(
dest_repo_id)
dest_repo_importer = model.Importer.objects.get_or_404(
repo_id=dest_repo_id)
source_repo_importer = model.Importer.objects.get_or_404(
repo_id=source_repo_id)
# The docs are incorrect on the list_importer_types call; it actually
# returns a dict with the types under key "types" for some reason.
supported_type_ids = set(
plugin_api.list_importer_types(
dest_repo_importer.importer_type_id)['types'])
# Get the unit types from the repo source repo
source_repo_unit_types = set(source_repo.content_unit_counts.keys())
# Now we can make sure the destination repository's importer is capable
# of importing either the selected units or all of the units
if not source_repo_unit_types.issubset(supported_type_ids):
raise exceptions.PulpCodedException(
error_code=error_codes.PLP0000,
message=
'The the target importer does not support the types from the source'
)
transfer_units = None
# if all source types have been converted to mongo - search via new style
if source_repo_unit_types.issubset(set(plugin_api.list_unit_models())):
transfer_units = cls._units_from_criteria(source_repo, criteria)
else:
# else, search via old style
associate_us = load_associated_units(source_repo_id, criteria)
# If units were supposed to be filtered but none matched, we're done
if len(associate_us) == 0:
# Return an empty list to indicate nothing was copied
return {'units_successful': []}
# Convert all of the units into the plugin standard representation if
# a filter was specified
transfer_units = None
if associate_us is not None:
transfer_units = create_transfer_units(associate_us)
# Convert the two repos into the plugin API model
transfer_dest_repo = dest_repo.to_transfer_repo()
transfer_source_repo = source_repo.to_transfer_repo()
# Invoke the importer
importer_instance, plugin_config = plugin_api.get_importer_by_id(
dest_repo_importer.importer_type_id)
call_config = PluginCallConfiguration(plugin_config,
dest_repo_importer.config,
import_config_override)
conduit = ImportUnitConduit(source_repo_id, dest_repo_id,
source_repo_importer.importer_type_id,
dest_repo_importer.importer_type_id)
try:
copied_units = importer_instance.import_units(transfer_source_repo,
transfer_dest_repo,
conduit,
call_config,
units=transfer_units)
unit_ids = [u.to_id_dict() for u in copied_units]
repo_controller.rebuild_content_unit_counts(dest_repo)
return {'units_successful': unit_ids}
except Exception:
msg = _(
'Exception from importer [%(i)s] while importing units into repository [%(r)s]'
)
msg_dict = {
'i': dest_repo_importer.importer_type_id,
'r': dest_repo_id
}
logger.exception(msg % msg_dict)
raise exceptions.PulpExecutionException(), None, sys.exc_info()[2]
