def associate_unit_by_id(self,
repo_id,
unit_type_id,
unit_id,
update_repo_metadata=True):
"""
Creates an association between the given repository and content unit.
If there is already an association between the given repo and content
unit where all other metadata matches the input to this method,
this call has no effect.
Both repo and unit must exist in the database prior to this call,
however this call will not verify that for performance reasons. Care
should be taken by the caller to preserve the data integrity.
@param repo_id: identifies the repo
@type repo_id: str
@param unit_type_id: identifies the type of unit being added
@type unit_type_id: str
@param unit_id: uniquely identifies the unit within the given type
@type unit_id: str
@param update_repo_metadata: if True, updates the unit association count
after the new association is made. The last
unit added field will also be updated. Set this
to False when doing bulk associations, and
make one call to update the count at the end.
defaults to True
@type update_repo_metadata: bool
@raise InvalidType: if the given owner type is not of the valid enumeration
"""
# If the association already exists, no need to do anything else
spec = {
'repo_id': repo_id,
'unit_id': unit_id,
'unit_type_id': unit_type_id
}
existing_association = RepoContentUnit.get_collection().find_one(spec)
if existing_association is not None:
return
similar_exists = False
if update_repo_metadata:
similar_exists = RepoUnitAssociationManager.association_exists(
repo_id, unit_id, unit_type_id)
# Create the database entry
association = RepoContentUnit(repo_id, unit_id, unit_type_id)
RepoContentUnit.get_collection().save(association, safe=True)
# update the count and times of associated units on the repo object
if update_repo_metadata and not similar_exists:
repo_controller.update_unit_count(repo_id, unit_type_id, 1)
repo_controller.update_last_unit_added(repo_id)
def associate_unit_by_id(self, repo_id, unit_type_id, unit_id, update_repo_metadata=True):
"""
Creates an association between the given repository and content unit.
If there is already an association between the given repo and content
unit where all other metadata matches the input to this method,
this call has no effect.
Both repo and unit must exist in the database prior to this call,
however this call will not verify that for performance reasons. Care
should be taken by the caller to preserve the data integrity.
@param repo_id: identifies the repo
@type repo_id: str
@param unit_type_id: identifies the type of unit being added
@type unit_type_id: str
@param unit_id: uniquely identifies the unit within the given type
@type unit_id: str
@param update_repo_metadata: if True, updates the unit association count
after the new association is made. The last
unit added field will also be updated. Set this
to False when doing bulk associations, and
make one call to update the count at the end.
defaults to True
@type update_repo_metadata: bool
@raise InvalidType: if the given owner type is not of the valid enumeration
"""
# If the association already exists, no need to do anything else
spec = {'repo_id': repo_id,
'unit_id': unit_id,
'unit_type_id': unit_type_id}
existing_association = RepoContentUnit.get_collection().find_one(spec)
if existing_association is not None:
return
similar_exists = False
if update_repo_metadata:
similar_exists = RepoUnitAssociationManager.association_exists(repo_id, unit_id,
unit_type_id)
# Create the database entry
association = RepoContentUnit(repo_id, unit_id, unit_type_id)
RepoContentUnit.get_collection().save(association, safe=True)
# update the count and times of associated units on the repo object
if update_repo_metadata and not similar_exists:
repo_controller.update_unit_count(repo_id, unit_type_id, 1)
repo_controller.update_last_unit_added(repo_id)
def associate_all_by_ids(self, repo_id, unit_type_id, unit_id_list):
"""
Creates multiple associations between the given repo and content units.
See associate_unit_by_id for semantics.
@param repo_id: identifies the repo
@type repo_id: str
@param unit_type_id: identifies the type of unit being added
@type unit_type_id: str
@param unit_id_list: list or generator of unique identifiers for units within the given type
@type unit_id_list: list or generator of str
:return: number of new units added to the repo
:rtype: int
@raise InvalidType: if the given owner type is not of the valid enumeration
"""
# There may be a way to batch this in mongo which would be ideal for a
# bulk operation like this. But for deadline purposes, this call will
# simply loop and call the single method.
unique_count = 0
for unit_id in unit_id_list:
if not RepoUnitAssociationManager.association_exists(
repo_id, unit_id, unit_type_id):
unique_count += 1
self.associate_unit_by_id(repo_id, unit_type_id, unit_id, False)
# update the count of associated units on the repo object
if unique_count:
repo_controller.update_unit_count(repo_id, unit_type_id,
unique_count)
repo_controller.update_last_unit_added(repo_id)
return unique_count
def associate_all_by_ids(self, repo_id, unit_type_id, unit_id_list):
"""
Creates multiple associations between the given repo and content units.
See associate_unit_by_id for semantics.
@param repo_id: identifies the repo
@type repo_id: str
@param unit_type_id: identifies the type of unit being added
@type unit_type_id: str
@param unit_id_list: list or generator of unique identifiers for units within the given type
@type unit_id_list: list or generator of str
:return: number of new units added to the repo
:rtype: int
@raise InvalidType: if the given owner type is not of the valid enumeration
"""
# There may be a way to batch this in mongo which would be ideal for a
# bulk operation like this. But for deadline purposes, this call will
# simply loop and call the single method.
unique_count = 0
for unit_id in unit_id_list:
if not RepoUnitAssociationManager.association_exists(repo_id, unit_id, unit_type_id):
unique_count += 1
self.associate_unit_by_id(repo_id, unit_type_id, unit_id, False)
# update the count of associated units on the repo object
if unique_count:
repo_controller.update_unit_count(repo_id, unit_type_id, unique_count)
repo_controller.update_last_unit_added(repo_id)
return unique_count
def associate_from_repo(source_repo_id, dest_repo_id, criteria,
import_config_override=None):
"""
Creates associations in a repository based on the contents of a source
repository. Units from the source repository can be filtered by
specifying a criteria object.
The destination repository must have an importer that can support
the types of units being associated. This is done by analyzing the
unit list and the importer metadata and takes place before the
destination repository is called.
Pulp does not actually perform the associations as part of this call.
The unit list is determined and passed to the destination repository's
importer. It is the job of the importer to make the associate calls
back into Pulp where applicable.
If criteria is None, the effect of this call is to copy the source
repository's associations into the destination repository.
:param source_repo_id: identifies the source repository
:type source_repo_id: str
:param dest_repo_id: identifies the destination repository
:type dest_repo_id: str
:param criteria: optional; if specified, will filter the units retrieved from
the source repository
:type criteria: pulp.server.db.model.criteria.UnitAssociationCriteria
:param import_config_override: optional config containing values to use for this import only
:type import_config_override: dict
:return: dict with key 'units_successful' whose
value is a list of unit keys that were copied.
units that were associated by this operation
:rtype: dict
:raise MissingResource: if either of the specified repositories don't exist
"""
criteria = UnitAssociationCriteria.from_dict(criteria)
source_repo = model.Repository.objects.get_repo_or_missing_resource(source_repo_id)
dest_repo = model.Repository.objects.get_repo_or_missing_resource(dest_repo_id)
dest_repo_importer = model.Importer.objects.get_or_404(repo_id=dest_repo_id)
source_repo_importer = model.Importer.objects.get_or_404(repo_id=source_repo_id)
# The docs are incorrect on the list_importer_types call; it actually
# returns a dict with the types under key "types" for some reason.
supported_type_ids = set(plugin_api.list_importer_types(
dest_repo_importer.importer_type_id)['types'])
# Get the unit types from the repo source repo
source_repo_unit_types = set(source_repo.content_unit_counts.keys())
# Now we can make sure the destination repository's importer is capable
# of importing either the selected units or all of the units
if not source_repo_unit_types.issubset(supported_type_ids):
raise exceptions.PulpCodedException(error_code=error_codes.PLP0044)
transfer_units = None
# if all source types have been converted to mongo - search via new style
if source_repo_unit_types.issubset(set(plugin_api.list_unit_models())):
transfer_units = RepoUnitAssociationManager._units_from_criteria(source_repo, criteria)
else:
# else, search via old style
associate_us = load_associated_units(source_repo_id, criteria)
# If units were supposed to be filtered but none matched, we're done
if len(associate_us) == 0:
# Return an empty list to indicate nothing was copied
return {'units_successful': []}
# Convert all of the units into the plugin standard representation if
# a filter was specified
transfer_units = None
if associate_us is not None:
transfer_units = create_transfer_units(associate_us)
# Convert the two repos into the plugin API model
transfer_dest_repo = dest_repo.to_transfer_repo()
transfer_source_repo = source_repo.to_transfer_repo()
# Invoke the importer
importer_instance, plugin_config = plugin_api.get_importer_by_id(
dest_repo_importer.importer_type_id)
call_config = PluginCallConfiguration(plugin_config, dest_repo_importer.config,
import_config_override)
conduit = ImportUnitConduit(
source_repo_id, dest_repo_id, source_repo_importer.importer_type_id,
dest_repo_importer.importer_type_id)
try:
copied_units = importer_instance.import_units(
transfer_source_repo, transfer_dest_repo, conduit, call_config,
units=transfer_units)
if isinstance(copied_units, tuple):
suc_units_ids = [u.to_id_dict() for u in copied_units[0] if u is not None]
unsuc_units_ids = [u.to_id_dict() for u in copied_units[1]]
repo_controller.rebuild_content_unit_counts(dest_repo)
if suc_units_ids:
repo_controller.update_last_unit_added(dest_repo.repo_id)
return {'units_successful': suc_units_ids,
'units_failed_signature_filter': unsuc_units_ids}
unit_ids = [u.to_id_dict() for u in copied_units if u is not None]
repo_controller.rebuild_content_unit_counts(dest_repo)
if unit_ids:
repo_controller.update_last_unit_added(dest_repo.repo_id)
return {'units_successful': unit_ids}
except Exception as e:
msg = _('Exception from importer [%(i)s] while importing units into repository [%(r)s]')
msg_dict = {'i': dest_repo_importer.importer_type_id, 'r': dest_repo_id}
logger.exception(msg % msg_dict)
raise (e, None, sys.exc_info()[2])
def import_uploaded_unit(repo_id,
unit_type_id,
unit_key,
unit_metadata,
upload_id,
override_config=None):
"""
Called to trigger the importer's handling of an uploaded unit. This
should not be called until the bits have finished uploading. The
importer is then responsible for moving the file to the correct location,
adding it to the Pulp server's inventory, and associating it with the
repository.
This call will first call is_valid_upload to check the integrity of the
destination repository. See that method's documentation for exception
possibilities.
:param repo_id: identifies the repository into which the unit is uploaded
:type repo_id: str
:param unit_type_id: type of unit being uploaded
:type unit_type_id: str
:param unit_key: unique identifier for the unit (user-specified)
:type unit_key: dict
:param unit_metadata: any user-specified information about the unit
:type unit_metadata: dict
:param upload_id: upload being imported
:type upload_id: str
:return: A dictionary describing the success or failure of the upload. It must
contain the following keys:
'success_flag': bool. Indicates whether the upload was successful
'summary': json-serializable object, providing summary
'details': json-serializable object, providing details
:rtype: dict
:raises MissingResource: if upload request was for the non-existent repository
:raises PulpCodedException: if import was unsuccessful and it was handled by the importer
:raises PulpException: if import was unsuccessful and it was not handled by the importer
:raises PulpExecutionException: if an unexpected error occured during the upload
"""
# If it doesn't raise an exception, it's good to go
ContentUploadManager.is_valid_upload(repo_id, unit_type_id)
repo_obj = model.Repository.objects.get_repo_or_missing_resource(
repo_id)
repo_importer = model.Importer.objects.get_or_404(repo_id=repo_id)
try:
importer_instance, plugin_config = plugin_api.get_importer_by_id(
repo_importer['importer_type_id'])
except plugin_exceptions.PluginNotFound:
raise MissingResource(repo_id), None, sys.exc_info()[2]
# Assemble the data needed for the import
conduit = UploadConduit(repo_id, repo_importer['id'])
call_config = PluginCallConfiguration(plugin_config,
repo_importer['config'],
override_config)
transfer_repo = repo_obj.to_transfer_repo()
file_path = ContentUploadManager._upload_file_path(upload_id)
# Invoke the importer
try:
result = importer_instance.upload_unit(transfer_repo, unit_type_id,
unit_key, unit_metadata,
file_path, conduit,
call_config)
if not result['success_flag']:
raise PulpCodedException(
error_code=error_codes.PLP0047,
repo_id=transfer_repo.id,
importer_id=repo_importer['importer_type_id'],
unit_type=unit_type_id,
summary=result['summary'],
details=result['details'])
repo_controller.rebuild_content_unit_counts(repo_obj)
repo_controller.update_last_unit_added(repo_obj.repo_id)
return result
except PulpCodedException:
# propagate coded exceptions.
raise
except PulpException:
msg = _(
'Error from the importer while importing uploaded unit to repository [%(r)s]'
)
msg = msg % {'r': repo_id}
logger.exception(msg)
raise
except Exception, e:
msg = _(
'Error from the importer while importing uploaded unit to repository [%(r)s]'
)
msg = msg % {'r': repo_id}
logger.exception(msg)
raise PulpExecutionException(e), None, sys.exc_info()[2]
def import_uploaded_unit(repo_id, unit_type_id, unit_key, unit_metadata, upload_id,
override_config=None):
"""
Called to trigger the importer's handling of an uploaded unit. This
should not be called until the bits have finished uploading. The
importer is then responsible for moving the file to the correct location,
adding it to the Pulp server's inventory, and associating it with the
repository.
This call will first call is_valid_upload to check the integrity of the
destination repository. See that method's documentation for exception
possibilities.
:param repo_id: identifies the repository into which the unit is uploaded
:type repo_id: str
:param unit_type_id: type of unit being uploaded
:type unit_type_id: str
:param unit_key: unique identifier for the unit (user-specified)
:type unit_key: dict
:param unit_metadata: any user-specified information about the unit
:type unit_metadata: dict
:param upload_id: upload being imported
:type upload_id: str
:return: A dictionary describing the success or failure of the upload. It must
contain the following keys:
'success_flag': bool. Indicates whether the upload was successful
'summary': json-serializable object, providing summary
'details': json-serializable object, providing details
:rtype: dict
:raises MissingResource: if upload request was for the non-existent repository
:raises PulpCodedException: if import was unsuccessful and it was handled by the importer
:raises PulpException: if import was unsuccessful and it was not handled by the importer
:raises PulpExecutionException: if an unexpected error occured during the upload
"""
# If it doesn't raise an exception, it's good to go
ContentUploadManager.is_valid_upload(repo_id, unit_type_id)
repo_obj = model.Repository.objects.get_repo_or_missing_resource(repo_id)
repo_importer = model.Importer.objects.get_or_404(repo_id=repo_id)
try:
importer_instance, plugin_config = plugin_api.get_importer_by_id(
repo_importer['importer_type_id'])
except plugin_exceptions.PluginNotFound:
raise MissingResource(repo_id), None, sys.exc_info()[2]
# Assemble the data needed for the import
conduit = UploadConduit(repo_id, repo_importer['id'])
call_config = PluginCallConfiguration(plugin_config, repo_importer['config'],
override_config)
transfer_repo = repo_obj.to_transfer_repo()
file_path = ContentUploadManager._upload_file_path(upload_id)
# Invoke the importer
try:
result = importer_instance.upload_unit(transfer_repo, unit_type_id, unit_key,
unit_metadata, file_path, conduit, call_config)
if not result['success_flag']:
raise PulpCodedException(
error_code=error_codes.PLP0047, repo_id=transfer_repo.id,
importer_id=repo_importer['importer_type_id'],
unit_type=unit_type_id, summary=result['summary'], details=result['details']
)
repo_controller.rebuild_content_unit_counts(repo_obj)
repo_controller.update_last_unit_added(repo_obj.repo_id)
return result
except PulpCodedException:
# propagate coded exceptions.
raise
except PulpException:
msg = _('Error from the importer while importing uploaded unit to repository [%(r)s]')
msg = msg % {'r': repo_id}
logger.exception(msg)
raise
except Exception, e:
msg = _('Error from the importer while importing uploaded unit to repository [%(r)s]')
msg = msg % {'r': repo_id}
logger.exception(msg)
raise PulpExecutionException(e), None, sys.exc_info()[2]
def associate_from_repo(source_repo_id,
dest_repo_id,
criteria,
import_config_override=None):
"""
Creates associations in a repository based on the contents of a source
repository. Units from the source repository can be filtered by
specifying a criteria object.
The destination repository must have an importer that can support
the types of units being associated. This is done by analyzing the
unit list and the importer metadata and takes place before the
destination repository is called.
Pulp does not actually perform the associations as part of this call.
The unit list is determined and passed to the destination repository's
importer. It is the job of the importer to make the associate calls
back into Pulp where applicable.
If criteria is None, the effect of this call is to copy the source
repository's associations into the destination repository.
:param source_repo_id: identifies the source repository
:type source_repo_id: str
:param dest_repo_id: identifies the destination repository
:type dest_repo_id: str
:param criteria: optional; if specified, will filter the units retrieved from
the source repository
:type criteria: pulp.server.db.model.criteria.UnitAssociationCriteria
:param import_config_override: optional config containing values to use for this import only
:type import_config_override: dict
:return: dict with key 'units_successful' whose
value is a list of unit keys that were copied.
units that were associated by this operation
:rtype: dict
:raise MissingResource: if either of the specified repositories don't exist
"""
criteria = UnitAssociationCriteria.from_dict(criteria)
source_repo = model.Repository.objects.get_repo_or_missing_resource(
source_repo_id)
dest_repo = model.Repository.objects.get_repo_or_missing_resource(
dest_repo_id)
dest_repo_importer = model.Importer.objects.get_or_404(
repo_id=dest_repo_id)
source_repo_importer = model.Importer.objects.get_or_404(
repo_id=source_repo_id)
# The docs are incorrect on the list_importer_types call; it actually
# returns a dict with the types under key "types" for some reason.
supported_type_ids = set(
plugin_api.list_importer_types(
dest_repo_importer.importer_type_id)['types'])
# Get the unit types from the repo source repo
source_repo_unit_types = set(source_repo.content_unit_counts.keys())
# Now we can make sure the destination repository's importer is capable
# of importing either the selected units or all of the units
if not source_repo_unit_types.issubset(supported_type_ids):
raise exceptions.PulpCodedException(error_code=error_codes.PLP0044)
transfer_units = None
# if all source types have been converted to mongo - search via new style
if source_repo_unit_types.issubset(set(plugin_api.list_unit_models())):
transfer_units = RepoUnitAssociationManager._units_from_criteria(
source_repo, criteria)
else:
# else, search via old style
associate_us = load_associated_units(source_repo_id, criteria)
# If units were supposed to be filtered but none matched, we're done
if len(associate_us) == 0:
# Return an empty list to indicate nothing was copied
return {'units_successful': []}
# Convert all of the units into the plugin standard representation if
# a filter was specified
transfer_units = None
if associate_us is not None:
transfer_units = create_transfer_units(associate_us)
# Convert the two repos into the plugin API model
transfer_dest_repo = dest_repo.to_transfer_repo()
transfer_source_repo = source_repo.to_transfer_repo()
# Invoke the importer
importer_instance, plugin_config = plugin_api.get_importer_by_id(
dest_repo_importer.importer_type_id)
call_config = PluginCallConfiguration(plugin_config,
dest_repo_importer.config,
import_config_override)
conduit = ImportUnitConduit(source_repo_id, dest_repo_id,
source_repo_importer.importer_type_id,
dest_repo_importer.importer_type_id)
try:
copied_units = importer_instance.import_units(transfer_source_repo,
transfer_dest_repo,
conduit,
call_config,
units=transfer_units)
if isinstance(copied_units, tuple):
suc_units_ids = [
u.to_id_dict() for u in copied_units[0] if u is not None
]
unsuc_units_ids = [u.to_id_dict() for u in copied_units[1]]
repo_controller.rebuild_content_unit_counts(dest_repo)
if suc_units_ids:
repo_controller.update_last_unit_added(dest_repo.repo_id)
return {
'units_successful': suc_units_ids,
'units_failed_signature_filter': unsuc_units_ids
}
unit_ids = [u.to_id_dict() for u in copied_units if u is not None]
repo_controller.rebuild_content_unit_counts(dest_repo)
if unit_ids:
repo_controller.update_last_unit_added(dest_repo.repo_id)
return {'units_successful': unit_ids}
except Exception as e:
msg = _(
'Exception from importer [%(i)s] while importing units into repository [%(r)s]'
)
msg_dict = {
'i': dest_repo_importer.importer_type_id,
'r': dest_repo_id
}
logger.exception(msg % msg_dict)
raise (e, None, sys.exc_info()[2])
