def search_all_units(self, type_id, criteria):
"""
Searches for units of a given type in the server, regardless of their
associations to any repositories.
@param type_id: indicates the type of units being retrieved
@type type_id: str
@param criteria: used to query which units are returned
@type criteria: pulp.server.db.model.criteria.Criteria
@return: list of unit instances
@rtype: list of L{Unit}
"""
try:
query_manager = manager_factory.content_query_manager()
units = query_manager.find_by_criteria(type_id, criteria)
type_def = types_db.type_definition(type_id)
transfer_units = []
for pulp_unit in units:
u = common_utils.to_plugin_unit(pulp_unit, type_def)
transfer_units.append(u)
return transfer_units
except Exception, e:
_LOG.exception('Exception from server requesting all units of type [%s]' % type_id)
raise self.exception_class(e), None, sys.exc_info()[2]
def find_unit_by_unit_key(self, type_id, unit_key):
"""
Finds a unit based on its unit key. If more than one unit comes back,
an exception will be raised.
@param type_id: indicates the type of units being retrieved
@type type_id: str
@param unit_key: the unit key for the unit
@type unit_key: dict
@return: a single unit
@rtype: L{Unit}
"""
content_query_manager = manager_factory.content_query_manager()
try:
# this call returns a unit or raises MissingResource
existing_unit = content_query_manager.get_content_unit_by_keys_dict(
type_id, unit_key)
unit_key_fields = units_controller.get_unit_key_fields_for_type(
type_id)
plugin_unit = common_utils.to_plugin_unit(existing_unit, type_id,
unit_key_fields)
return plugin_unit
except pulp_exceptions.MissingResource:
return None
def GET(self, type_id):
"""
List all the available content units.
"""
cqm = factory.content_query_manager()
units = cqm.find_by_criteria(type_id, Criteria())
return self.ok([self.process_unit(unit) for unit in units])
def delete_orphans_by_id(self, orphans):
"""
Delete a list of orphaned content units by their content type and unit ids.
@param orphans: list of documents with 'content_type' and 'content_id' keys
@type orphans: list
"""
# XXX this does no validation of the orphans
# munge the orphans into something more programmatic-ly convenient
orphans_by_id = {}
for o in orphans:
if 'content_type_id' not in o or 'unit_id' not in o:
raise pulp_exceptions.InvalidValue(['content_type_id', 'unit_id'])
id_list = orphans_by_id.setdefault(o['content_type_id'], [])
id_list.append(o['unit_id'])
# iterate through the types and ids
content_query_manager = manager_factory.content_query_manager()
for content_type, content_id_list in orphans_by_id.items():
# build a list of the on-disk contents
orphaned_paths = []
for unit_id in content_id_list:
content_unit = content_query_manager.get_content_unit_by_id(content_type, unit_id, model_fields=['_storage_path'])
if content_unit['_storage_path'] is not None:
orphaned_paths.append(content_unit['_storage_path'])
# remove the orphans from the db
collection = content_types_db.type_units_collection(content_type)
spec = {'_id': {'$in': content_id_list}}
collection.remove(spec, safe=True)
# delete the on-disk contents
for path in orphaned_paths:
self.delete_orphaned_file(path)
def GET(self, type_id):
"""
List all the available content units.
"""
cqm = factory.content_query_manager()
units = cqm.find_by_criteria(type_id, Criteria())
return self.ok([self.process_unit(unit) for unit in units])
def test_syntactic_sugar_methods(self):
"""
Tests the syntactic sugar methods for retrieving specific managers.
"""
# Setup
factory.initialize()
# Test
self.assertTrue(isinstance(factory.authentication_manager(), AuthenticationManager))
self.assertTrue(isinstance(factory.cert_generation_manager(), CertGenerationManager))
self.assertTrue(isinstance(factory.certificate_manager(), CertificateManager))
self.assertTrue(isinstance(factory.password_manager(), PasswordManager))
self.assertTrue(isinstance(factory.permission_manager(), PermissionManager))
self.assertTrue(isinstance(factory.permission_query_manager(), PermissionQueryManager))
self.assertTrue(isinstance(factory.role_manager(), RoleManager))
self.assertTrue(isinstance(factory.role_query_manager(), RoleQueryManager))
self.assertTrue(isinstance(factory.user_manager(), UserManager))
self.assertTrue(isinstance(factory.user_query_manager(), UserQueryManager))
self.assertTrue(isinstance(factory.repo_manager(), RepoManager))
self.assertTrue(isinstance(factory.repo_unit_association_manager(), RepoUnitAssociationManager))
self.assertTrue(isinstance(factory.repo_publish_manager(), RepoPublishManager))
self.assertTrue(isinstance(factory.repo_query_manager(), RepoQueryManager))
self.assertTrue(isinstance(factory.repo_sync_manager(), RepoSyncManager))
self.assertTrue(isinstance(factory.content_manager(), ContentManager))
self.assertTrue(isinstance(factory.content_query_manager(), ContentQueryManager))
self.assertTrue(isinstance(factory.content_upload_manager(), ContentUploadManager))
self.assertTrue(isinstance(factory.consumer_manager(), ConsumerManager))
self.assertTrue(isinstance(factory.topic_publish_manager(), TopicPublishManager))
def search_all_units(self, type_id, criteria):
"""
Searches for units of a given type in the server, regardless of their
associations to any repositories.
@param type_id: indicates the type of units being retrieved
@type type_id: str
@param criteria: used to query which units are returned
@type criteria: pulp.server.db.model.criteria.Criteria
@return: list of unit instances
@rtype: list of L{Unit}
"""
try:
query_manager = manager_factory.content_query_manager()
units = query_manager.find_by_criteria(type_id, criteria)
type_def = types_db.type_definition(type_id)
transfer_units = []
for pulp_unit in units:
u = common_utils.to_plugin_unit(pulp_unit, type_def)
transfer_units.append(u)
return transfer_units
except Exception, e:
logger.exception(
'Exception from server requesting all units of type [%s]' %
type_id)
raise self.exception_class(e), None, sys.exc_info()[2]
def get(self, request, type_id):
"""
Return a response with a serialized list of the content units of the specified type.
:param request: WSGI request object
:type request: django.core.handlers.wsgi.WSGIRequest
:param type_id: the list of content units will be limited to this type
:type type_id: str
:return: response with a serialized list of dicts, one for each unit of the type.
:rtype: django.http.HttpResponse
"""
cqm = factory.content_query_manager()
all_units = cqm.find_by_criteria(type_id, Criteria())
all_processed_units = []
for unit in all_units:
unit = serialization.content.content_unit_obj(unit)
unit.update({
'_href':
'/'.join(
[request.get_full_path().rstrip('/'), unit['_id'], ''])
})
unit.update({
'children':
serialization.content.content_unit_child_link_objs(unit)
})
all_processed_units.append(unit)
return generate_json_response_with_pulp_encoder(all_processed_units)
def _update_unit(self, unit, pulp_unit):
"""
Update a unit. If it is not found, add it.
:param unit: the unit to be updated
:type unit: pulp.plugins.model.Unit
:param pulp_unit: the unit to be updated, as a dict
:type pulp_unit: dict
:return: id of the updated unit
:rtype: basestring
"""
content_query_manager = manager_factory.content_query_manager()
content_manager = manager_factory.content_manager()
try:
existing_unit = content_query_manager.get_content_unit_by_keys_dict(
unit.type_id, unit.unit_key)
unit_id = existing_unit['_id']
content_manager.update_content_unit(unit.type_id, unit_id,
pulp_unit)
self._updated_count += 1
return unit_id
except MissingResource:
logger.debug(
_('cannot update unit; does not exist. adding instead.'))
return self._add_unit(unit, pulp_unit)
def search_unit_ids(self, type_id, criteria):
"""
Searches for units of a given type in the server, regardless of their
associations to any repositories and returns a list of unit ids.
@param type_id: indicates the type of units being retrieved
@type type_id: str
@param criteria: used to query which units are returned
@type criteria: pulp.server.db.model.criteria.Criteria
@return: list of unit ids
@rtype: list of str
"""
try:
query_manager = managers.content_query_manager()
criteria["fields"] = ['_id']
units = query_manager.find_by_criteria(type_id, criteria)
unit_ids = [u['_id'] for u in units]
return unit_ids
except Exception, e:
_LOG.exception(
_('Exception from server searching units of type [%s]' %
type_id))
raise self.exception_class(e), None, sys.exc_info()[2]
def get(self, request, type_id):
"""
Return a response containing a dict with info about a content type.
:param request: WSGI request object
:type request: django.core.handlers.wsgi.WSGIRequest
:param type_id: type of content unit
:type type_id: str
:return: response containing a dict that contains info about a content type or
404 response if the specified content type is not found.
:rtype : django.http.HttpResponse or HttpResponseNotFound
"""
cqm = factory.content_query_manager()
content_type = cqm.get_content_type(type_id)
if content_type is None:
msg = _('No content type resource: %(r)s') % {'r': type_id}
return generate_json_response(msg,
response_class=HttpResponseNotFound)
resource = serialization.content.content_type_obj(content_type)
# These urls are not valid endpoints but are left here for for semantic versioning.
# BZ - 1187287
links = {
'actions': {
'_href':
'/'.join([request.get_full_path().rstrip('/'), 'actions/'])
},
'content_units': {
'_href':
'/'.join([request.get_full_path().rstrip('/'), 'units/'])
}
}
resource.update(links)
return generate_json_response_with_pulp_encoder(resource)
def test_syntactic_sugar_methods(self):
"""
Tests the syntactic sugar methods for retrieving specific managers.
"""
# Setup
factory.initialize()
# Test
self.assertTrue(isinstance(factory.authentication_manager(), AuthenticationManager))
self.assertTrue(isinstance(factory.cert_generation_manager(), CertGenerationManager))
self.assertTrue(isinstance(factory.certificate_manager(), CertificateManager))
self.assertTrue(isinstance(factory.password_manager(), PasswordManager))
self.assertTrue(isinstance(factory.permission_manager(), PermissionManager))
self.assertTrue(isinstance(factory.permission_query_manager(), PermissionQueryManager))
self.assertTrue(isinstance(factory.role_manager(), RoleManager))
self.assertTrue(isinstance(factory.role_query_manager(), RoleQueryManager))
self.assertTrue(isinstance(factory.user_manager(), UserManager))
self.assertTrue(isinstance(factory.user_query_manager(), UserQueryManager))
self.assertTrue(isinstance(factory.repo_manager(), RepoManager))
self.assertTrue(isinstance(factory.repo_unit_association_manager(),
RepoUnitAssociationManager))
self.assertTrue(isinstance(factory.repo_publish_manager(), RepoPublishManager))
self.assertTrue(isinstance(factory.repo_query_manager(), RepoQueryManager))
self.assertTrue(isinstance(factory.repo_sync_manager(), RepoSyncManager))
self.assertTrue(isinstance(factory.content_manager(), ContentManager))
self.assertTrue(isinstance(factory.content_query_manager(), ContentQueryManager))
self.assertTrue(isinstance(factory.content_upload_manager(), ContentUploadManager))
self.assertTrue(isinstance(factory.consumer_manager(), ConsumerManager))
self.assertTrue(isinstance(factory.topic_publish_manager(), TopicPublishManager))
def get(self, request, type_id, unit_id):
"""
Return a response containing information about the requested content unit.
:param request: WSGI request object
:type request: django.core.handlers.wsgi.WSGIRequest
:param type_id: type of content contained in the repo
:type type_id: str
:param unit_id: unique id of a unit
:type unit_id: str
:return: response containing a dict with data about requested unit
:rtype : django.http.HttpResponse
"""
cqm = factory.content_query_manager()
try:
unit = cqm.get_content_unit_by_id(type_id, unit_id)
except MissingResource:
msg = _('No content unit resource: %(r)s') % {'r': unit_id}
return generate_json_response(msg,
response_class=HttpResponseNotFound)
resource = serial_content.content_unit_obj(unit)
resource.update({
'children':
serial_content.content_unit_child_link_objs(resource)
})
return generate_json_response_with_pulp_encoder(resource)
def get(self, request, type_id):
"""
Return a response containing a dict with info about a content type.
:param request: WSGI request object
:type request: django.core.handlers.wsgi.WSGIRequest
:param type_id: type of content unit
:type type_id: str
:return: response containing a dict that contains info about a content type or
404 response if the specified content type is not found.
:rtype : django.http.HttpResponse or HttpResponseNotFound
"""
cqm = factory.content_query_manager()
content_type = cqm.get_content_type(type_id)
if content_type is None:
msg = _('No content type resource: %(r)s') % {'r': type_id}
return generate_json_response(msg, response_class=HttpResponseNotFound)
resource = serialization.content.content_type_obj(content_type)
# These urls are not valid endpoints but are left here for for semantic versioning.
# BZ - 1187287
links = {
'actions': {'_href': '/'.join([request.get_full_path().rstrip('/'), 'actions/'])},
'content_units': {'_href': '/'.join([request.get_full_path().rstrip('/'), 'units/'])}
}
resource.update(links)
return generate_json_response_with_pulp_encoder(resource)
def __unit_ids_to_plugin_unit_keys(self, unit_ids_by_type, repo_ids):
"""
Parse a dictionary of unit ids keyed by content type id and return a dictionary of
corresponding plugin unit keys keyed by content type id.
:param unit_ids_by_type: dictionary of <content type id> : <list of unit ids>
:type unit_ids_by_type: dict
:return: if units are specified, return the corresponding plugin unit_keys. If unit_ids_by_type dict
is empty, return plugin unit keys corresponging to all units in given repo ids.
If unit ids list for a particular unit type is empty, return all plugin unit_keys
in given repo ids with that unit type.
:rtype: dict
"""
repo_unit_association_query_manager = managers.repo_unit_association_query_manager()
content_query_manager = managers.content_query_manager()
result_unit_keys = {}
if unit_ids_by_type is not None:
for unit_type_id, unit_ids in unit_ids_by_type.items():
# Get unit type specific collection
collection = content_query_manager.get_content_unit_collection(type_id=unit_type_id)
type_def = content_types_db.type_definition(unit_type_id)
if not unit_ids:
# If unit_list is empty for a unit_type, consider all units of specific type
criteria = UnitAssociationCriteria(unit_fields = ['unit_id'])
for repo_id in repo_ids:
repo_units = repo_unit_association_query_manager.get_units_by_type(repo_id, unit_type_id, criteria)
# Get metadata for each unit from type specific collection
pulp_units = [collection.find_one({'_id': u['unit_id']}) for u in repo_units]
# Convert pulp units to plugin unit keys
plugin_unit_keys = [common_utils.to_plugin_unit(u, type_def).unit_key for u in pulp_units]
result_unit_keys.setdefault(unit_type_id, []).extend(plugin_unit_keys)
else:
# Get metadata for each unit from type specific collection
pulp_units = [collection.find_one({'_id': unit_id}) for unit_id in unit_ids]
# Convert pulp units to plugin unit keys
plugin_unit_keys = [common_utils.to_plugin_unit(u, type_def).unit_key for u in pulp_units]
result_unit_keys.setdefault(unit_type_id, []).extend(plugin_unit_keys)
else:
# If units are not specified, consider all units in given repos.
for repo_id in repo_ids:
all_unit_type_ids = content_types_db.all_type_ids()
for unit_type_id in all_unit_type_ids:
criteria = UnitAssociationCriteria(type_ids=[unit_type_id], unit_fields = ['unit_id', 'unit_type_id'])
repo_units = repo_unit_association_query_manager.get_units(repo_id, criteria)
# Get unit metadata for each unit from type specific collection
collection = content_query_manager.get_content_unit_collection(type_id=unit_type_id)
pulp_units = [collection.find_one({'_id': u['unit_id']}) for u in repo_units]
# Convert pulp units to plugin unit keys
type_def = content_types_db.type_definition(unit_type_id)
plugin_unit_keys = [common_utils.to_plugin_unit(u, type_def).unit_key for u in pulp_units]
result_unit_keys.setdefault(unit_type_id, []).extend(plugin_unit_keys)
return result_unit_keys
def __parse_units(self, user_units, repo_ids):
"""
Parse units specified by user and return a dictionary of all plugin unit_keys
to be considered for applicability keyed by unit_type_id.
:param user_units: dictionary of unit metadata filters keyed by unit-type-id specified by user
:type user_units: dict
:return: if specific units are specified, return the corresponding plugin unit_keys. If units dict is empty,
return all plugin unit_keys corresponging to units in given repo ids keyed by unit_type_id.
If units list for a particular unit type in units is empty, return all plugin unit_keys
in given repo ids with that unit type keyed by unit_type_id.
:rtype: dict
"""
repo_unit_association_query_manager = managers.repo_unit_association_query_manager()
content_query_manager = managers.content_query_manager()
result_unit_keys = {}
if user_units is not None:
for unit_type_id, unit_list in user_units.items():
# Get unit type specific collection
collection = content_query_manager.get_content_unit_collection(type_id=unit_type_id)
type_def = content_types_db.type_definition(unit_type_id)
if not unit_list:
# If unit_list is empty for a unit_type, consider all units of specific type
criteria = UnitAssociationCriteria(unit_fields = ['unit_id'])
for repo_id in repo_ids:
repo_units = repo_unit_association_query_manager.get_units_by_type(repo_id, unit_type_id, criteria)
# Get unit metadata for each unit from type specific collection
pulp_units = [collection.find_one({'_id': u['unit_id']}) for u in repo_units]
plugin_units = [common_utils.to_plugin_unit(u, type_def) for u in pulp_units]
plugin_unit_keys = [u.unit_key for u in plugin_units]
result_unit_keys.setdefault(unit_type_id, []).extend(plugin_unit_keys)
else:
for unit in unit_list:
criteria = UnitAssociationCriteria(unit_filters=unit, unit_fields = ['unit_id'])
for repo_id in repo_ids:
repo_units = repo_unit_association_query_manager.get_units_by_type(repo_id, unit_type_id, criteria)
# Get unit metadata for each unit from type specific collection
pulp_units = [collection.find_one({'_id': u['unit_id']}) for u in repo_units]
plugin_units = [common_utils.to_plugin_unit(u, type_def) for u in pulp_units]
plugin_unit_keys = [u.unit_key for u in plugin_units]
result_unit_keys.setdefault(unit_type_id, []).extend(plugin_unit_keys)
else:
# If units are not specified, consider all units in repo_ids list.
for repo_id in repo_ids:
criteria = UnitAssociationCriteria(unit_fields = ['unit_id','unit_type_id'])
repo_units = repo_unit_association_query_manager.get_units(repo_id, criteria)
# Get unit metadata for each unit from type specific collection
for repo_unit in repo_units:
collection = content_query_manager.get_content_unit_collection(type_id=repo_unit['unit_type_id'])
type_def = content_types_db.type_definition(repo_unit['unit_type_id'])
pulp_unit = collection.find_one({'_id': repo_unit['unit_id']})
plugin_unit = common_utils.to_plugin_unit(pulp_unit, type_def)
result_unit_keys.setdefault(repo_unit['unit_type_id'], []).append(plugin_unit.unit_key)
return result_unit_keys
def init_unit(self, type_id, unit_key, metadata, relative_path):
"""
Initializes the Pulp representation of a content unit. The conduit will
use the provided information to generate any unit metadata that it needs
to. A populated transfer object representation of the unit will be
returned from this call. The returned unit should be used in subsequent
calls to this conduit.
This call makes no changes to the Pulp server. At the end of this call,
the unit's id field will *not* be populated.
The unit_key and metadata will be merged as they are saved in Pulp to
form the full representation of the unit. If values are specified in
both dictionaries, the unit_key value takes precedence.
If the importer wants to save the bits for the unit, the relative_path
value should be used to indicate a unique -- with respect to the type
of unit -- relative path where it will be saved. Pulp will convert this
into an absolute path on disk where the unit should actually be saved.
The absolute path is stored in the returned unit object.
@param type_id: must correspond to a type definition in Pulp
@type type_id: str
@param unit_key: dictionary of whatever fields are necessary to uniquely
identify this unit from others of the same type
@type unit_key: dict
@param metadata: dictionary of key-value pairs to describe the unit
@type metadata: dict
@param relative_path: see above; may be None
@type relative_path: str, None
@return: object representation of the unit, populated by Pulp with both
provided and derived values
@rtype: pulp.plugins.model.Unit
"""
try:
# Generate the storage location
if relative_path is not None:
content_query_manager = manager_factory.content_query_manager()
path = content_query_manager.request_content_unit_file_path(
type_id, relative_path)
else:
path = None
u = Unit(type_id, unit_key, metadata, path)
return u
except Exception, e:
msg = _(
'Exception from server requesting unit filename for relative path [%s]'
)
msg = msg % relative_path
_logger.exception(msg)
raise ImporterConduitException(e), None, sys.exc_info()[2]
def delete_all_orphans(self):
"""
Delete all orphaned content units.
"""
# iterate through the types and delete all orphans of each type
content_query_manager = manager_factory.content_query_manager()
content_types = content_query_manager.list_content_types()
for content_type in content_types:
self.delete_orphans_by_type(content_type)
def unassociate_all_by_ids(self, repo_id, unit_type_id, unit_id_list, owner_type, owner_id):
"""
Removes the association between a repo and a number of units. Only the
association made by the given owner will be removed. It is possible the
repo will still have a manually created association will for the unit.
@param repo_id: identifies the repo
@type repo_id: str
@param unit_type_id: identifies the type of units being removed
@type unit_type_id: str
@param unit_id_list: list of unique identifiers for units within the given type
@type unit_id_list: list of str
@param owner_type: category of the caller who created the association;
must be one of the OWNER_* variables in this module
@type owner_type: str
@param owner_id: identifies the caller who created the association, either
the importer ID or user login
@type owner_id: str
"""
spec = {'repo_id' : repo_id,
'unit_type_id' : unit_type_id,
'unit_id' : {'$in' : unit_id_list},
'owner_type' : owner_type,
'owner_id' : owner_id,
}
unit_coll = RepoContentUnit.get_collection()
unit_coll.remove(spec, safe=True)
unique_count = sum(1 for unit_id in unit_id_list
if not self.association_exists(repo_id, unit_id, unit_type_id))
# update the count of associated units on the repo object
if unique_count:
manager_factory.repo_manager().update_unit_count(
repo_id, -unique_count)
try:
repo_query_manager = manager_factory.repo_query_manager()
repo = repo_query_manager.get_repository(repo_id)
content_query_manager = manager_factory.content_query_manager()
content_units = content_query_manager.get_multiple_units_by_ids(unit_type_id, unit_id_list)
importer_manager = manager_factory.repo_importer_manager()
importer = importer_manager.get_importer(repo_id)
importer.remove_units(repo, content_units)
except:
_LOG.exception('Exception informing importer for [%s] of unassociation' % repo_id)
def __init__(self, repo_id, importer_id):
RepoScratchPadMixin.__init__(self, repo_id, ImporterConduitException)
ImporterScratchPadMixin.__init__(self, repo_id, importer_id)
AddUnitMixin.__init__(self, repo_id, importer_id)
SingleRepoUnitsMixin.__init__(self, repo_id, ImporterConduitException)
StatusMixin.__init__(self, importer_id, ImporterConduitException)
SearchUnitsMixin.__init__(self, ImporterConduitException)
self._association_manager = manager_factory.repo_unit_association_manager()
self._content_query_manager = manager_factory.content_query_manager()
self._removed_count = 0
def GET(self):
"""
List the available content types.
"""
collection = []
cqm = factory.content_query_manager()
type_ids = cqm.list_content_types()
for id in type_ids:
link = serialization.link.child_link_obj(id)
link.update({'content_type': id})
collection.append(link)
return self.ok(collection)
def GET(self, type_id, unit_id):
"""
Return information about a content unit.
"""
cqm = factory.content_query_manager()
try:
unit = cqm.get_content_unit_by_id(type_id, unit_id)
except MissingResource:
return self.not_found(_("No content unit resource: %(r)s") % {"r": unit_id})
resource = serialization.content.content_unit_obj(unit)
resource.update({"children": serialization.content.content_unit_child_link_objs(resource)})
return self.ok(resource)
def init_unit(self, type_id, unit_key, metadata, relative_path):
"""
Initializes the Pulp representation of a content unit. The conduit will
use the provided information to generate any unit metadata that it needs
to. A populated transfer object representation of the unit will be
returned from this call. The returned unit should be used in subsequent
calls to this conduit.
This call makes no changes to the Pulp server. At the end of this call,
the unit's id field will *not* be populated.
The unit_key and metadata will be merged as they are saved in Pulp to
form the full representation of the unit. If values are specified in
both dictionaries, the unit_key value takes precedence.
If the importer wants to save the bits for the unit, the relative_path
value should be used to indicate a unique -- with respect to the type
of unit -- relative path where it will be saved. Pulp will convert this
into an absolute path on disk where the unit should actually be saved.
The absolute path is stored in the returned unit object.
@param type_id: must correspond to a type definition in Pulp
@type type_id: str
@param unit_key: dictionary of whatever fields are necessary to uniquely
identify this unit from others of the same type
@type unit_key: dict
@param metadata: dictionary of key-value pairs to describe the unit
@type metadata: dict
@param relative_path: see above; may be None
@type relative_path: str, None
@return: object representation of the unit, populated by Pulp with both
provided and derived values
@rtype: pulp.plugins.model.Unit
"""
try:
# Generate the storage location
if relative_path is not None:
content_query_manager = manager_factory.content_query_manager()
path = content_query_manager.request_content_unit_file_path(type_id, relative_path)
else:
path = None
u = Unit(type_id, unit_key, metadata, path)
return u
except Exception, e:
msg = _('Exception from server requesting unit filename for relative path [%s]')
msg = msg % relative_path
_logger.exception(msg)
raise ImporterConduitException(e), None, sys.exc_info()[2]
def GET(self):
"""
List the available content types.
"""
collection = []
cqm = factory.content_query_manager()
type_ids = cqm.list_content_types()
for type_id in type_ids:
link = serialization.link.child_link_obj(type_id)
link.update({'content_type': type_id})
collection.append(link)
return self.ok(collection)
def __init__(self, repo_id, importer_id):
RepoScratchPadMixin.__init__(self, repo_id, ImporterConduitException)
ImporterScratchPadMixin.__init__(self, repo_id, importer_id)
AddUnitMixin.__init__(self, repo_id, importer_id)
SingleRepoUnitsMixin.__init__(self, repo_id, ImporterConduitException)
StatusMixin.__init__(self, importer_id, ImporterConduitException)
SearchUnitsMixin.__init__(self, ImporterConduitException)
self._association_manager = manager_factory.repo_unit_association_manager(
)
self._content_query_manager = manager_factory.content_query_manager()
self._removed_count = 0
def GET(self, type_id, unit_id):
"""
Return information about a content unit.
"""
cqm = factory.content_query_manager()
try:
unit = cqm.get_content_unit_by_id(type_id, unit_id)
except MissingResource:
return self.not_found(_('No content unit resource: %(r)s') %
{'r': unit_id})
resource = serialization.content.content_unit_obj(unit)
resource.update({'children': serialization.content.content_unit_child_link_objs(resource)})
return self.ok(resource)
def GET(self, type_id):
"""
Return information about a content type.
"""
cqm = factory.content_query_manager()
content_type = cqm.get_content_type(type_id)
if content_type is None:
return self.not_found(_('No content type resource: %(r)s') %
{'r': type_id})
resource = serialization.content.content_type_obj(content_type)
links = {'actions': serialization.link.child_link_obj('actions'),
'content_units': serialization.link.child_link_obj('units')}
resource.update(links)
return self.ok(resource)
def GET(self, type_id):
"""
Return information about a content type.
"""
cqm = factory.content_query_manager()
content_type = cqm.get_content_type(type_id)
if content_type is None:
return self.not_found(_('No content type resource: %(r)s') %
{'r': type_id})
resource = serialization.content.content_type_obj(content_type)
links = {'actions': serialization.link.child_link_obj('actions'),
'content_units': serialization.link.child_link_obj('units')}
resource.update(links)
return self.ok(resource)
def test_syntactic_sugar_methods(self):
"""
Tests the syntactic sugar methods for retrieving specific managers.
"""
# Test
self.assertTrue(isinstance(factory.repo_manager(), RepoManager))
self.assertTrue(isinstance(factory.repo_unit_association_manager(), RepoUnitAssociationManager))
self.assertTrue(isinstance(factory.repo_publish_manager(), RepoPublishManager))
self.assertTrue(isinstance(factory.repo_query_manager(), RepoQueryManager))
self.assertTrue(isinstance(factory.repo_sync_manager(), RepoSyncManager))
self.assertTrue(isinstance(factory.content_manager(), ContentManager))
self.assertTrue(isinstance(factory.content_query_manager(), ContentQueryManager))
self.assertTrue(isinstance(factory.content_upload_manager(), ContentUploadManager))
self.assertTrue(isinstance(factory.consumer_manager(), ConsumerManager))
def list_all_orphans(self):
"""
List all content units that are not associated with a repository.
@return: list of content units
@rtype: list
"""
# iterate through all types and get the orphaned units for each
orphaned_units = []
content_query_manager = manager_factory.content_query_manager()
content_types = content_query_manager.list_content_types()
for content_type in content_types:
orphaned_units_of_type = self.list_orphans_by_type(content_type)
orphaned_units.extend(orphaned_units_of_type)
return orphaned_units
def save_unit(self, unit):
"""
Performs two distinct steps on the Pulp server:
- Creates or updates Pulp's knowledge of the content unit.
- Associates the unit to the repository being synchronized.
This call is idempotent. If the unit already exists or the association
already exists, this call will have no effect.
A reference to the provided unit is returned from this call. This call
will populate the unit's id field with the UUID for the unit.
@param unit: unit object returned from the init_unit call
@type unit: L{Unit}
@return: object reference to the provided unit, its state updated from the call
@rtype: L{Unit}
"""
try:
content_query_manager = manager_factory.content_query_manager()
content_manager = manager_factory.content_manager()
association_manager = manager_factory.repo_unit_association_manager(
)
# Save or update the unit
pulp_unit = common_utils.to_pulp_unit(unit)
try:
existing_unit = content_query_manager.get_content_unit_by_keys_dict(
unit.type_id, unit.unit_key)
unit.id = existing_unit['_id']
content_manager.update_content_unit(unit.type_id, unit.id,
pulp_unit)
self._updated_count += 1
except MissingResource:
unit.id = content_manager.add_content_unit(
unit.type_id, None, pulp_unit)
self._added_count += 1
# Associate it with the repo
association_manager.associate_unit_by_id(
self.repo_id, unit.type_id, unit.id,
self.association_owner_type, self.association_owner_id)
return unit
except Exception, e:
_LOG.exception(
_('Content unit association failed [%s]' % str(unit)))
raise ImporterConduitException(e), None, sys.exc_info()[2]
def get(self, request, type_id):
"""
Return a response with a serialized list of the content units of the specified type.
:param request: WSGI request object
:type request: django.core.handlers.wsgi.WSGIRequest
:param type_id: the list of content units will be limited to this type
:type type_id: str
:return: response with a serialized list of dicts, one for each unit of the type.
:rtype: django.http.HttpResponse
"""
cqm = factory.content_query_manager()
all_units = cqm.find_by_criteria(type_id, Criteria())
all_processed_units = [_process_content_unit(unit, type_id) for unit in all_units]
return generate_json_response_with_pulp_encoder(all_processed_units)
def get(self, request, type_id):
"""
Return a response with a serialized list of the content units of the specified type.
:param request: WSGI request object
:type request: django.core.handlers.wsgi.WSGIRequest
:param type_id: the list of content units will be limited to this type
:type type_id: str
:return: response with a serialized list of dicts, one for each unit of the type.
:rtype: django.http.HttpResponse
"""
cqm = factory.content_query_manager()
all_units = cqm.find_by_criteria(type_id, Criteria())
all_processed_units = [
_process_content_unit(unit, type_id) for unit in all_units
]
return generate_json_response_with_pulp_encoder(all_processed_units)
def get(self, request):
"""
Returns a response continaing a list of dicts, one for each available content type.
:param request: WSGI request object
:type request: django.core.handlers.wsgi.WSGIRequest
:return: response containing a serialized list dicts, one for each content type
:rtype : django.http.HttpResponse
"""
collection = []
cqm = factory.content_query_manager()
type_ids = cqm.list_content_types()
for type_id in type_ids:
link = {'_href': reverse('content_type_resource', kwargs={'type_id': type_id})}
link.update({'content_type': type_id})
collection.append(link)
return generate_json_response(collection)
def GET(self, type_id, unit_id):
"""
Return information about a content unit.
:param type_id: The Unit's type id.
:type type_id: basestring
:param unit_id: The id of the unit that you wish to set the pulp_user_metadata field on
:type unit_id: basestring
"""
cqm = factory.content_query_manager()
try:
unit = cqm.get_content_unit_by_id(type_id, unit_id)
except MissingResource:
return self.not_found(_('No content unit resource: %(r)s') % {'r': unit_id})
resource = serialization.content.content_unit_obj(
unit[constants.PULP_USER_METADATA_FIELDNAME])
return self.ok(resource)
def PUT(self, type_id, unit_id):
"""
Set the pulp_user_metadata field on the existing unit.
:param type_id: The Unit's type id.
:type type_id: basestring
:param unit_id: The id of the unit that you wish to set the pulp_user_metadata field on
:type unit_id: basestring
"""
cqm = factory.content_query_manager()
try:
cqm.get_content_unit_by_id(type_id, unit_id)
except MissingResource:
return self.not_found(_('No content unit resource: %(r)s') % {'r': unit_id})
cm = factory.content_manager()
delta = {constants.PULP_USER_METADATA_FIELDNAME: self.params()}
cm.update_content_unit(type_id, unit_id, delta)
return self.ok(None)
def save_unit(self, unit):
"""
Performs two distinct steps on the Pulp server:
- Creates or updates Pulp's knowledge of the content unit.
- Associates the unit to the repository being synchronized.
If a unit with the provided unit key already exists, it is updated with
the attributes on the passed-in unit.
A reference to the provided unit is returned from this call. This call
will populate the unit's id field with the UUID for the unit.
@param unit: unit object returned from the init_unit call
@type unit: L{Unit}
@return: object reference to the provided unit, its state updated from the call
@rtype: L{Unit}
"""
try:
content_query_manager = manager_factory.content_query_manager()
content_manager = manager_factory.content_manager()
association_manager = manager_factory.repo_unit_association_manager()
# Save or update the unit
pulp_unit = common_utils.to_pulp_unit(unit)
try:
existing_unit = content_query_manager.get_content_unit_by_keys_dict(unit.type_id, unit.unit_key)
unit.id = existing_unit['_id']
content_manager.update_content_unit(unit.type_id, unit.id, pulp_unit)
self._updated_count += 1
except MissingResource:
unit.id = content_manager.add_content_unit(unit.type_id, None, pulp_unit)
self._added_count += 1
# Associate it with the repo
association_manager.associate_unit_by_id(self.repo_id, unit.type_id, unit.id, self.association_owner_type, self.association_owner_id)
return unit
except Exception, e:
_LOG.exception(_('Content unit association failed [%s]' % str(unit)))
raise ImporterConduitException(e), None, sys.exc_info()[2]
def __init__(self, importer_type, unit_type, unit_key_fields, working_dir):
"""
:param importer_type: unique identifier for the type of importer
:type importer_type: basestring
:param unit_type: unique identifier for the unit type in use
:type unit_type: basestring
:param unit_key_fields: a list of field names in the unit type's unit key.
:type unit_key_fields: list
:param working_dir: full path to a working directory
:type working_dir: basestring
"""
super(GetLocalUnitsStep, self).__init__(step_type=reporting_constants.SYNC_STEP_GET_LOCAL,
plugin_type=importer_type,
working_dir=working_dir)
self.description = _('Copying units already in pulp')
self.unit_type = unit_type
self.unit_key_fields = unit_key_fields
self.content_query_manager = manager_factory.content_query_manager()
# list of unit keys
self.units_to_download = []
def __init__(self, importer_type, unit_type, unit_key_fields, working_dir):
"""
:param importer_type: unique identifier for the type of importer
:type importer_type: basestring
:param unit_type: unique identifier for the unit type in use
:type unit_type: basestring
:param unit_key_fields: a list of field names in the unit type's unit key.
:type unit_key_fields: list
:param working_dir: full path to a working directory
:type working_dir: basestring
"""
super(GetLocalUnitsStep, self).__init__(step_type=reporting_constants.SYNC_STEP_GET_LOCAL,
plugin_type=importer_type,
working_dir=working_dir)
self.description = _('Copying units already in pulp')
self.unit_type = unit_type
self.unit_key_fields = unit_key_fields
self.content_query_manager = manager_factory.content_query_manager()
# list of unit keys
self.units_to_download = []
def get(self, request):
"""
Returns a response continaing a list of dicts, one for each available content type.
:param request: WSGI request object
:type request: django.core.handlers.wsgi.WSGIRequest
:return: response containing a serialized list dicts, one for each content type
:rtype : django.http.HttpResponse
"""
collection = []
cqm = factory.content_query_manager()
type_ids = cqm.list_content_types()
for type_id in type_ids:
link = {
'_href':
reverse('content_type_resource', kwargs={'type_id': type_id})
}
link.update({'content_type': type_id})
collection.append(link)
return generate_json_response(collection)
def get(self, request, type_id, unit_id):
"""
Return user metadata for a content unit.
:param type_id: The Unit's type id.
:type type_id: basestring
:param unit_id: The id of the unit that you wish to set the pulp_user_metadata field on
:type unit_id: basestring
:return: response containing pulp user metadata field
:rtype: django.http.HttpResponse or HttpResponseNotFound
"""
cqm = factory.content_query_manager()
try:
unit = cqm.get_content_unit_by_id(type_id, unit_id)
except MissingResource:
msg = _("No content unit resource: %(r)s") % {"r": unit_id}
return generate_json_response(msg, HttpResponseNotFound)
resource = serial_content.content_unit_obj(unit[constants.PULP_USER_METADATA_FIELDNAME])
return generate_json_response(resource)
def _proxy_query_method(self, criteria):
"""
Normally the constructor passes a manager's query method to the
super-class constructor. Since our manager's query method takes an extra
parameter to tell it what content type to look in, we have this proxy
query method that will make the correct call at the time.
Also, at the time of instantiation, we don't know what the content
type_id will be, so each request handler method will set self._type_id
to the correct value, and this method will use it at the time of being
called.
This sounds like it's asking for a race condition, I know, but web.py
instantiates a new controller for each and every request, so that isn't
a concern.
@param criteria: Criteria representing a search
@type criteria: models.db.criteria.Criteria
@return: same as PulpCollection.query
"""
return factory.content_query_manager().find_by_criteria(self._type_id, criteria)
def find_unit_by_unit_key(self, type_id, unit_key):
"""
Finds a unit based on its unit key. If more than one unit comes back,
an exception will be raised.
@param type_id: indicates the type of units being retrieved
@type type_id: str
@param unit_key: the unit key for the unit
@type unit_key: dict
@return: a single unit
@rtype: L{Unit}
"""
content_query_manager = manager_factory.content_query_manager()
try:
# this call returns a unit or raises MissingResource
existing_unit = content_query_manager.get_content_unit_by_keys_dict(type_id, unit_key)
unit_key_fields = units_controller.get_unit_key_fields_for_type(type_id)
plugin_unit = common_utils.to_plugin_unit(existing_unit, type_id, unit_key_fields)
return plugin_unit
except pulp_exceptions.MissingResource:
return None
def get(self, request, type_id, unit_id):
"""
Return user metadata for a content unit.
:param type_id: The Unit's type id.
:type type_id: basestring
:param unit_id: The id of the unit that you wish to set the pulp_user_metadata field on
:type unit_id: basestring
:return: response containing pulp user metadata field
:rtype: django.http.HttpResponse or HttpResponseNotFound
"""
cqm = factory.content_query_manager()
try:
unit = cqm.get_content_unit_by_id(type_id, unit_id)
except MissingResource:
msg = _('No content unit resource: %(r)s') % {'r': unit_id}
return generate_json_response(msg, HttpResponseNotFound)
resource = serial_content.content_unit_obj(
unit[constants.PULP_USER_METADATA_FIELDNAME])
return generate_json_response(resource)
def get(self, request, type_id):
"""
Return a response with a serialized list of the content units of the specified type.
:param request: WSGI request object
:type request: django.core.handlers.wsgi.WSGIRequest
:param type_id: the list of content units will be limited to this type
:type type_id: str
:return: response with a serialized list of dicts, one for each unit of the type.
:rtype: django.http.HttpResponse
"""
cqm = factory.content_query_manager()
all_units = cqm.find_by_criteria(type_id, Criteria())
all_processed_units = []
for unit in all_units:
unit = serialization.content.content_unit_obj(unit)
unit.update({'_href': '/'.join([request.get_full_path().rstrip('/'),
unit['_id'], ''])})
unit.update({'children': serialization.content.content_unit_child_link_objs(unit)})
all_processed_units.append(unit)
return generate_json_response_with_pulp_encoder(all_processed_units)
def _update_unit(self, unit, pulp_unit):
"""
Update a unit. If it is not found, add it.
:param unit: the unit to be updated
:type unit: pulp.plugins.model.Unit
:param pulp_unit: the unit to be updated, as a dict
:type pulp_unit: dict
:return: id of the updated unit
:rtype: basestring
"""
content_query_manager = manager_factory.content_query_manager()
content_manager = manager_factory.content_manager()
try:
existing_unit = content_query_manager.get_content_unit_by_keys_dict(unit.type_id, unit.unit_key)
unit_id = existing_unit['_id']
content_manager.update_content_unit(unit.type_id, unit_id, pulp_unit)
self._updated_count += 1
return unit_id
except MissingResource:
logger.debug(_('cannot update unit; does not exist. adding instead.'))
return self._add_unit(unit, pulp_unit)
def _proxy_query_method(self, criteria):
"""
Normally the constructor passes a manager's query method to the
super-class constructor. Since our manager's query method takes an extra
parameter to tell it what content type to look in, we have this proxy
query method that will make the correct call at the time.
Also, at the time of instantiation, we don't know what the content
type_id will be, so each request handler method will set self._type_id
to the correct value, and this method will use it at the time of being
called.
This sounds like it's asking for a race condition, I know, but web.py
instantiates a new controller for each and every request, so that isn't
a concern.
@param criteria: Criteria representing a search
@type criteria: models.db.criteria.Criteria
@return: same as PulpCollection.query
"""
return factory.content_query_manager().find_by_criteria(
self._type_id, criteria)
def get(self, request, type_id, unit_id):
"""
Return a response containing information about the requested content unit.
:param request: WSGI request object
:type request: django.core.handlers.wsgi.WSGIRequest
:param type_id: type of content contained in the repo
:type type_id: str
:param unit_id: unique id of a unit
:type unit_id: str
:return: response containing a dict with data about requested unit
:rtype : django.http.HttpResponse
"""
cqm = factory.content_query_manager()
try:
unit = cqm.get_content_unit_by_id(type_id, unit_id)
except MissingResource:
msg = _("No content unit resource: %(r)s") % {"r": unit_id}
return generate_json_response(msg, response_class=HttpResponseNotFound)
resource = serial_content.content_unit_obj(unit)
resource.update({"children": serial_content.content_unit_child_link_objs(resource)})
return generate_json_response_with_pulp_encoder(resource)
def put(self, request, type_id, unit_id):
"""
Set the pulp_user_metadata field on a content unit.
:param type_id: The Unit's type id.
:type type_id: basestring
:param unit_id: The id of the unit that you wish to set the pulp_user_metadata field on
:type unit_id: basestring
:return: response containing pulp user metadata_field
:rtype: django.http.HttpResponse or HttpResponseNotFound
"""
params = request.body_as_json
cqm = factory.content_query_manager()
try:
cqm.get_content_unit_by_id(type_id, unit_id)
except MissingResource:
msg = _("No content unit resource: %(r)s") % {"r": unit_id}
return generate_json_response(msg, HttpResponseNotFound)
cm = factory.content_manager()
delta = {constants.PULP_USER_METADATA_FIELDNAME: params}
cm.update_content_unit(type_id, unit_id, delta)
return generate_json_response(None)
def put(self, request, type_id, unit_id):
"""
Set the pulp_user_metadata field on a content unit.
:param type_id: The Unit's type id.
:type type_id: basestring
:param unit_id: The id of the unit that you wish to set the pulp_user_metadata field on
:type unit_id: basestring
:return: response containing pulp user metadata_field
:rtype: django.http.HttpResponse or HttpResponseNotFound
"""
params = request.body_as_json
cqm = factory.content_query_manager()
try:
cqm.get_content_unit_by_id(type_id, unit_id)
except MissingResource:
msg = _('No content unit resource: %(r)s') % {'r': unit_id}
return generate_json_response(msg, HttpResponseNotFound)
cm = factory.content_manager()
delta = {constants.PULP_USER_METADATA_FIELDNAME: params}
cm.update_content_unit(type_id, unit_id, delta)
return generate_json_response(None)
def find_applicable_units(self, consumer_criteria=None, repo_criteria=None, unit_criteria=None, override_config=None):
"""
Determine and report which of the content units specified by the unit_criteria
are applicable to consumers specified by the consumer_criteria
with repos specified by repo_criteria. If consumer_criteria is None,
all consumers registered to the Pulp server are checked for applicability.
If repo_criteria is None, all repos bound to the consumer are taken
into consideration. If unit_criteria contains an empty list for a specific type,
all units with specific type in the repos bound to the consumer
are taken into consideration.
:param consumer_criteria: The consumer selection criteria.
:type consumer_criteria: dict
:param repo_criteria: The repo selection criteria.
:type repo_criteria: dict
:param unit_criteria: A dictionary of type_id : unit selection criteria
:type units: dict
{<type_id1> : <unit_criteria_for_type_id1>,
<type_id2> : <unit_criteria_for_type_id2>}
:param override_config: Additional configuration options to be accepted from user
:type override_config: dict
:return: applicability reports dictionary keyed by content type id
:rtype: dict
"""
result = {}
conduit = ProfilerConduit()
consumer_query_manager = managers.consumer_query_manager()
bind_manager = managers.consumer_bind_manager()
# Process Repo Criteria
if repo_criteria:
# Get repo ids satisfied by specified repo criteria
repo_query_manager = managers.repo_query_manager()
repo_criteria_ids = [r['id'] for r in repo_query_manager.find_by_criteria(repo_criteria)]
# if repo_criteria is specified and there are no repos satisfying the criteria, return empty result
if not repo_criteria_ids:
return result
else:
repo_criteria_ids = None
# Process Consumer Criteria
if consumer_criteria:
# Get consumer ids satisfied by specified consumer criteria
consumer_ids = [c['id'] for c in consumer_query_manager.find_by_criteria(consumer_criteria)]
else:
if repo_criteria_ids:
# If repo_criteria is specified, get all the consumers bound to the repos
# satisfied by repo_criteria
bind_criteria = Criteria(filters={"repo_id": {"$in": repo_criteria_ids}})
consumer_ids = [b['consumer_id'] for b in bind_manager.find_by_criteria(bind_criteria)]
# Remove duplicate consumer ids
consumer_ids = list(set(consumer_ids))
else:
# Get all consumer ids registered to the Pulp server
consumer_ids = [c['id'] for c in consumer_query_manager.find_all()]
# if there are no relevant consumers, return empty result
if not consumer_ids:
return result
else:
# Based on the consumers, get all the repos bound to the consumers in consideration
# and find intersection of repo_criteria_ids and consumer_repo_ids
bind_criteria = Criteria(filters={"consumer_id": {"$in": consumer_ids}})
consumer_repo_ids = [b['repo_id'] for b in bind_manager.find_by_criteria(bind_criteria)]
if not repo_criteria_ids:
repo_criteria_ids = list(set(consumer_repo_ids))
else:
repo_criteria_ids = list(set(consumer_repo_ids) & set(repo_criteria_ids))
if not repo_criteria_ids:
return result
# Process Unit Criteria
if unit_criteria:
# If unit_criteria is specified, get unit ids satisfied by the criteria for each content type
# and save them in a dictionary keyed by the content type
unit_ids_by_type = {}
content_query_manager = managers.content_query_manager()
for type_id, criteria in unit_criteria.items():
if criteria:
criteria_ids = [u['_id'] for u in content_query_manager.find_by_criteria(type_id, criteria)]
# If there are no units satisfied by the criteria, skip adding it to the dictionary
if criteria_ids:
unit_ids_by_type[type_id] = criteria_ids
else:
# If criteria for a content type id is None or empty dictionary, add it to the dictionary
# with empty list as a value. This will be interpreted as all units of that specific type
unit_ids_by_type[type_id] = []
else:
# If unit_criteria is not specified set unit_ids_by_type to None to differentiate between
# considering all units vs considering 0 units since no units were found satisfying given criteria
unit_ids_by_type = None
# Create a dictionary with consumer profile and repo_ids bound to the consumer keyed by consumer id
consumer_profile_and_repo_ids = {}
all_relevant_repo_ids = set()
for consumer_id in consumer_ids:
# Find repos bound to the consumer in consideration and find an intersection of bound repos to the
# repos specified by repo_criteria
consumer_bound_repo_ids = [b['repo_id'] for b in bind_manager.find_by_consumer(consumer_id)]
consumer_bound_repo_ids = list(set(consumer_bound_repo_ids))
# If repo_criteria is not specified, use repos bound to the consumer, else take intersection
# of repos specified in the criteria and repos bound to the consumer.
if repo_criteria_ids is None:
repo_ids = consumer_bound_repo_ids
else:
repo_ids = list(set(consumer_bound_repo_ids) & set(repo_criteria_ids))
if repo_ids:
# Save all eligible repo ids to get relevant plugin unit keys when unit_criteria is not specified
all_relevant_repo_ids = (all_relevant_repo_ids | set(repo_ids))
consumer_profile_and_repo_ids[consumer_id] = {'repo_ids': repo_ids}
consumer_profile_and_repo_ids[consumer_id]['profiled_consumer'] = self.__profiled_consumer(consumer_id)
# Get relevant units if unit_criteria is not specified
units = self.__populate_units(unit_ids_by_type, list(all_relevant_repo_ids))
if units:
# Call respective profiler api according to the unit type to check for applicability
for typeid, unit_ids in units.items():
# if unit_ids is None or empty, continue to the next type id
if not unit_ids:
continue
# Find a profiler for each type id and find units applicable using that profiler.
profiler, cfg = self.__profiler(typeid)
call_config = PluginCallConfiguration(plugin_config=cfg, repo_plugin_config=None,
override_config=override_config)
try:
report_list = profiler.find_applicable_units(consumer_profile_and_repo_ids, typeid, unit_ids, call_config, conduit)
except PulpExecutionException:
report_list = None
if report_list is None:
_LOG.warn("Profiler for unit type [%s] is not returning applicability reports" % typeid)
else:
result[typeid] = report_list
return result
def find_applicable_units(self,
consumer_criteria=None,
repo_criteria=None,
unit_criteria=None,
override_config=None):
"""
Determine and report which of the content units specified by the unit_criteria
are applicable to consumers specified by the consumer_criteria
with repos specified by repo_criteria. If consumer_criteria is None,
all consumers registered to the Pulp server are checked for applicability.
If repo_criteria is None, all repos bound to the consumer are taken
into consideration. If unit_criteria contains an empty list for a specific type,
all units with specific type in the repos bound to the consumer
are taken into consideration.
:param consumer_criteria: The consumer selection criteria.
:type consumer_criteria: dict
:param repo_criteria: The repo selection criteria.
:type repo_criteria: dict
:param unit_criteria: A dictionary of type_id : unit selection criteria
:type units: dict
{<type_id1> : <unit_criteria_for_type_id1>,
<type_id2> : <unit_criteria_for_type_id2>}
:param override_config: Additional configuration options to be accepted from user
:type override_config: dict
:return: applicability reports dictionary keyed by content type id
:rtype: dict
"""
result = {}
conduit = ProfilerConduit()
consumer_query_manager = managers.consumer_query_manager()
bind_manager = managers.consumer_bind_manager()
# Process Repo Criteria
if repo_criteria:
# Get repo ids satisfied by specified repo criteria
repo_query_manager = managers.repo_query_manager()
repo_criteria_ids = [
r['id']
for r in repo_query_manager.find_by_criteria(repo_criteria)
]
# if repo_criteria is specified and there are no repos satisfying the criteria, return empty result
if not repo_criteria_ids:
return result
else:
repo_criteria_ids = None
# Process Consumer Criteria
if consumer_criteria:
# Get consumer ids satisfied by specified consumer criteria
consumer_ids = [
c['id'] for c in consumer_query_manager.find_by_criteria(
consumer_criteria)
]
else:
if repo_criteria_ids:
# If repo_criteria is specified, get all the consumers bound to the repos
# satisfied by repo_criteria
bind_criteria = Criteria(
filters={"repo_id": {
"$in": repo_criteria_ids
}})
consumer_ids = [
b['consumer_id']
for b in bind_manager.find_by_criteria(bind_criteria)
]
# Remove duplicate consumer ids
consumer_ids = list(set(consumer_ids))
else:
# Get all consumer ids registered to the Pulp server
consumer_ids = [
c['id'] for c in consumer_query_manager.find_all()
]
# if there are no relevant consumers, return empty result
if not consumer_ids:
return result
else:
# Based on the consumers, get all the repos bound to the consumers in consideration
# and find intersection of repo_criteria_ids and consumer_repo_ids
bind_criteria = Criteria(
filters={"consumer_id": {
"$in": consumer_ids
}})
consumer_repo_ids = [
b['repo_id']
for b in bind_manager.find_by_criteria(bind_criteria)
]
if not repo_criteria_ids:
repo_criteria_ids = list(set(consumer_repo_ids))
else:
repo_criteria_ids = list(
set(consumer_repo_ids) & set(repo_criteria_ids))
if not repo_criteria_ids:
return result
# Process Unit Criteria
if unit_criteria:
# If unit_criteria is specified, get unit ids satisfied by the criteria for each content type
# and save them in a dictionary keyed by the content type
unit_ids_by_type = {}
content_query_manager = managers.content_query_manager()
for type_id, criteria in unit_criteria.items():
if criteria:
criteria_ids = [
u['_id']
for u in content_query_manager.find_by_criteria(
type_id, criteria)
]
# If there are no units satisfied by the criteria, skip adding it to the dictionary
if criteria_ids:
unit_ids_by_type[type_id] = criteria_ids
else:
# If criteria for a content type id is None or empty dictionary, add it to the dictionary
# with empty list as a value. This will be interpreted as all units of that specific type
unit_ids_by_type[type_id] = []
else:
# If unit_criteria is not specified set unit_ids_by_type to None to differentiate between
# considering all units vs considering 0 units since no units were found satisfying given criteria
unit_ids_by_type = None
# Create a dictionary with consumer profile and repo_ids bound to the consumer keyed by consumer id
consumer_profile_and_repo_ids = {}
all_relevant_repo_ids = set()
for consumer_id in consumer_ids:
# Find repos bound to the consumer in consideration and find an intersection of bound repos to the
# repos specified by repo_criteria
consumer_bound_repo_ids = [
b['repo_id']
for b in bind_manager.find_by_consumer(consumer_id)
]
consumer_bound_repo_ids = list(set(consumer_bound_repo_ids))
# If repo_criteria is not specified, use repos bound to the consumer, else take intersection
# of repos specified in the criteria and repos bound to the consumer.
if repo_criteria_ids is None:
repo_ids = consumer_bound_repo_ids
else:
repo_ids = list(
set(consumer_bound_repo_ids) & set(repo_criteria_ids))
if repo_ids:
# Save all eligible repo ids to get relevant plugin unit keys when unit_criteria is not specified
all_relevant_repo_ids = (all_relevant_repo_ids | set(repo_ids))
consumer_profile_and_repo_ids[consumer_id] = {
'repo_ids': repo_ids
}
consumer_profile_and_repo_ids[consumer_id][
'profiled_consumer'] = self.__profiled_consumer(
consumer_id)
# Get relevant units if unit_criteria is not specified
units = self.__populate_units(unit_ids_by_type,
list(all_relevant_repo_ids))
if units:
# Call respective profiler api according to the unit type to check for applicability
for typeid, unit_ids in units.items():
# if unit_ids is None or empty, continue to the next type id
if not unit_ids:
continue
# Find a profiler for each type id and find units applicable using that profiler.
profiler, cfg = self.__profiler(typeid)
call_config = PluginCallConfiguration(
plugin_config=cfg,
repo_plugin_config=None,
override_config=override_config)
try:
report_list = profiler.find_applicable_units(
consumer_profile_and_repo_ids, typeid, unit_ids,
call_config, conduit)
except PulpExecutionException:
report_list = None
if report_list is None:
_LOG.warn(
"Profiler for unit type [%s] is not returning applicability reports"
% typeid)
else:
result[typeid] = report_list
return result
