def test_returns_serializer(self, mock_get_model):
mock_get_model.return_value = DemoModelWithSerializer
serializer = units_controller.get_model_serializer_for_type(
'demo_model')
self.assertTrue(isinstance(serializer, DemoModelSerializer))
self.assertTrue(hasattr(serializer, 'model'))
def get_repo_units(self, repo_id, content_type_id, additional_unit_fields=None):
"""
Searches for units in the given repository with given content type
and returns a plugin unit containing unit id, unit key and any additional
fields requested.
:param repo_id: repo id
:type repo_id: str
:param content_type_id: content type id of the units
:type content_type_id: str
:param additional_unit_fields: additional fields from the unit metadata to be added
in the result
:type additional_unit_fields: list of str
:return: list of unit instances
:rtype: list of pulp.plugins.model.Unit
"""
additional_unit_fields = additional_unit_fields or []
try:
unit_key_fields = units_controller.get_unit_key_fields_for_type(content_type_id)
serializer = units_controller.get_model_serializer_for_type(content_type_id)
# Query repo association manager to get all units of given type
# associated with given repo. Limit data by requesting only the fields
# that are needed.
query_manager = managers.repo_unit_association_query_manager()
unit_fields = list(unit_key_fields) + list(additional_unit_fields)
criteria = UnitAssociationCriteria(association_fields=['unit_id'],
unit_fields=unit_fields)
units = query_manager.get_units_by_type(repo_id, content_type_id, criteria)
# Convert units to plugin units with unit_key and required metadata values for each unit
all_units = []
for unit in units:
if serializer:
serializer.serialize(unit['metadata'])
unit_key = {}
metadata = {}
for k in unit_key_fields:
unit_key[k] = unit['metadata'].pop(k)
# Add unit_id and any additional unit fields requested by plugins
metadata['unit_id'] = unit.pop('unit_id')
for field in additional_unit_fields:
metadata[field] = unit['metadata'].pop(field, None)
u = Unit(content_type_id, unit_key, metadata, None)
all_units.append(u)
return all_units
except Exception, e:
_logger.exception(_('Exception from server getting units from repo [%s]' % repo_id))
raise self.exception_class(e), None, sys.exc_info()[2]
def _associated_units_by_type_cursor(unit_type_id, criteria, associated_unit_ids):
"""
Retrieve a pymongo cursor for units associated with a repository of a
give unit type that meet to the provided criteria.
:type unit_type_id: str
:type criteria: UnitAssociationCriteria
:type associated_unit_ids: list
:rtype: pymongo.cursor.Cursor
"""
collection = types_db.type_units_collection(unit_type_id)
serializer = units.get_model_serializer_for_type(unit_type_id)
unit_filter = criteria.unit_filters.copy()
if unit_filter and serializer:
unit_filter = serializer.translate_filters(serializer.model, unit_filter)
spec = {
'$and': [
{'_id': {'$in': associated_unit_ids}},
unit_filter
]
}
fields = criteria.unit_fields
if fields is not None:
fields = list(fields)
# The _content_type_id is required for looking up the association
if '_content_type_id' not in fields:
fields.append('_content_type_id')
# translate incoming fields (e.g. id -> foo_id)
if serializer:
for index, field in enumerate(fields):
fields[index] = serializer.translate_field(serializer.model, field)
cursor = collection.find(spec, projection=fields)
sort = criteria.unit_sort
if sort is None:
sort = [('_id', SORT_ASCENDING)]
elif serializer:
sort = list(sort)
for index, (field, direction) in enumerate(sort):
sort[index] = (serializer.translate_field(serializer.model, field), direction)
cursor.sort(sort)
return cursor
def _associated_units_by_type_cursor(unit_type_id, criteria,
associated_unit_ids):
"""
Retrieve a pymongo cursor for units associated with a repository of a
give unit type that meet to the provided criteria.
:type unit_type_id: str
:type criteria: UnitAssociationCriteria
:type associated_unit_ids: list
:rtype: pymongo.cursor.Cursor
"""
collection = types_db.type_units_collection(unit_type_id)
serializer = units.get_model_serializer_for_type(unit_type_id)
unit_filter = criteria.unit_filters.copy()
if unit_filter and serializer:
unit_filter = serializer.translate_filters(serializer.model,
unit_filter)
spec = {'$and': [{'_id': {'$in': associated_unit_ids}}, unit_filter]}
fields = criteria.unit_fields
if fields is not None:
fields = list(fields)
# The _content_type_id is required for looking up the association
if '_content_type_id' not in fields:
fields.append('_content_type_id')
# translate incoming fields (e.g. id -> foo_id)
if serializer:
for index, field in enumerate(fields):
fields[index] = serializer.translate_field(
serializer.model, field)
cursor = collection.find(spec, projection=fields)
sort = criteria.unit_sort
if sort is None:
sort = [('_id', SORT_ASCENDING)]
elif serializer:
sort = list(sort)
for index, (field, direction) in enumerate(sort):
sort[index] = (serializer.translate_field(
serializer.model, field), direction)
cursor.sort(sort)
return cursor
def get_results(cls, query, search_method, options, *args, **kwargs):
"""
Overrides the base class so additional information can optionally be added.
"""
type_id = kwargs['type_id']
serializer = units_controller.get_model_serializer_for_type(type_id)
if serializer and query.get('filters') is not None:
# if we have a model serializer, translate the filter for this content unit type
query['filters'] = serializer.translate_filters(serializer.model, query['filters'])
units = list(search_method(type_id, query))
units = [_process_content_unit(unit, type_id) for unit in units]
if options.get('include_repos') is True:
cls._add_repo_memberships(units, type_id)
return units
def get_results(cls, query, search_method, options, *args, **kwargs):
"""
Overrides the base class so additional information can optionally be added.
"""
type_id = kwargs['type_id']
serializer = units_controller.get_model_serializer_for_type(type_id)
if serializer and query.get('filters') is not None:
# if we have a model serializer, translate the filter for this content unit type
query['filters'] = serializer.translate_filters(
serializer.model, query['filters'])
units = list(search_method(type_id, query))
units = [_process_content_unit(unit, type_id) for unit in units]
if options.get('include_repos') is True:
cls._add_repo_memberships(units, type_id)
return units
def serialize_unit_with_serializer(content_unit):
"""
Serialize unit in-place to its dictionary form using its serializer.
This will also handle remapping that unit's fields from MongoEngine field names to their
expected field names in the API
This is a small workaround to help in cases where REST views are returning the older objects
coming out of pymongo, but still need to have their fields remapped according to the rules of
the pymongo serializer. As a workaround, this is a "best effort" function, so serialization
failures will be written to the debug log and not raise exeptions.
Usage of pymongo objects is deprecated. Since this function is only concerned with serializing
pymongo objects, its usage is also deprecated. Furthermore, this function is only intended to
be used in the final serialization of objects before presentation in the REST API, and as a
result modifies objects in-place.
:param content_unit: Content unit to be converted
:type content_unit: dict
"""
try:
content_type_id = content_unit['_content_type_id']
except KeyError:
# content unit didn't have a content type id, usually means we're testing...
_logger.debug(
'No _content_type_id found in content unit when remapping fields: '
'{0!r}'.format(content_unit))
return
serializer = units.get_model_serializer_for_type(content_type_id)
if serializer is None:
# No serializer, nothing to do
return
# use the serialize method to do any needed unit-specific serialization on fields
serializer.serialize(content_unit)
# remap fields before responding -- do this after serializing exotic fields since
# the serialize method will be working with the non-remapped field names.
field_map = {}
for field in content_unit:
remapped_field = serializer.translate_field_reverse(field)
if remapped_field != field:
field_map[field] = remapped_field
for original_field, remapped_field in field_map.items():
content_unit[remapped_field] = content_unit.pop(original_field)
def serialize_unit_with_serializer(content_unit):
"""
Serialize unit in-place to its dictionary form using its serializer.
This will also handle remapping that unit's fields from MongoEngine field names to their
expected field names in the API
This is a small workaround to help in cases where REST views are returning the older objects
coming out of pymongo, but still need to have their fields remapped according to the rules of
the pymongo serializer. As a workaround, this is a "best effort" function, so serialization
failures will be written to the debug log and not raise exeptions.
Usage of pymongo objects is deprecated. Since this function is only concerned with serializing
pymongo objects, its usage is also deprecated. Furthermore, this function is only intended to
be used in the final serialization of objects before presentation in the REST API, and as a
result modifies objects in-place.
:param content_unit: Content unit to be converted
:type content_unit: dict
"""
try:
content_type_id = content_unit['_content_type_id']
except KeyError:
# content unit didn't have a content type id, usually means we're testing...
_logger.debug('No _content_type_id found in content unit when remapping fields: '
'{0!r}'.format(content_unit))
return
serializer = units.get_model_serializer_for_type(content_type_id)
if serializer is None:
# No serializer, nothing to do
return
# use the serialize method to do any needed unit-specific serialization on fields
serializer.serialize(content_unit)
# remap fields before responding -- do this after serializing exotic fields since
# the serialize method will be working with the non-remapped field names.
field_map = {}
for field in content_unit:
remapped_field = serializer.translate_field_reverse(field)
if remapped_field != field:
field_map[field] = remapped_field
for original_field, remapped_field in field_map.items():
content_unit[remapped_field] = content_unit.pop(original_field)
def _units_from_criteria(source_repo, criteria):
"""
Given a criteria, return an iterator of units
:param source_repo: repository to look for units in
:type source_repo: pulp.server.db.model.Repository
:param criteria: criteria object to use for the search parameters
:type criteria: pulp.server.db.model.criteria.UnitAssociationCriteria
:return: generator of pulp.server.db.model.ContentUnit instances
:rtype: generator
"""
association_q = mongoengine.Q(__raw__=criteria.association_spec)
if criteria.type_ids:
association_q &= mongoengine.Q(unit_type_id__in=criteria.type_ids)
unit_type_ids = criteria.type_ids
else:
# don't need to limit the association_q by content type here
# since filtering for every content_type seen in the repo is
# achieved by limiting the query to the repo
unit_type_ids = repo_controller.get_repo_unit_type_ids(
source_repo.repo_id)
# base unit_q, works as-is for non-mongoengine unit types
unit_q = mongoengine.Q(__raw__=criteria.unit_spec)
# for mongoengine unit types, use the unit model's serializer to translate the unit spec
# for each possible content unit type as determined by the search criteria
for unit_type_id in unit_type_ids:
serializer = units_controller.get_model_serializer_for_type(
unit_type_id)
if serializer:
unit_spec_t = serializer.translate_filters(
serializer.model, criteria.unit_spec)
# after translating the fields, this spec is only good for this unit type
unit_spec_t['_content_type_id'] = unit_type_id
unit_q |= mongoengine.Q(__raw__=unit_spec_t)
return repo_controller.find_repo_content_units(
repository=source_repo,
repo_content_unit_q=association_q,
units_q=unit_q,
unit_fields=criteria['unit_fields'],
yield_content_unit=True)
def _units_from_criteria(source_repo, criteria):
"""
Given a criteria, return an iterator of units
:param source_repo: repository to look for units in
:type source_repo: pulp.server.db.model.Repository
:param criteria: criteria object to use for the search parameters
:type criteria: pulp.server.db.model.criteria.UnitAssociationCriteria
:return: generator of pulp.server.db.model.ContentUnit instances
:rtype: generator
"""
association_q = mongoengine.Q(__raw__=criteria.association_spec)
if criteria.type_ids:
association_q &= mongoengine.Q(unit_type_id__in=criteria.type_ids)
unit_type_ids = criteria.type_ids
else:
# don't need to limit the association_q by content type here
# since filtering for every content_type seen in the repo is
# achieved by limiting the query to the repo
unit_type_ids = repo_controller.get_repo_unit_type_ids(source_repo.repo_id)
# base unit_q, works as-is for non-mongoengine unit types
unit_q = mongoengine.Q(__raw__=criteria.unit_spec)
# for mongoengine unit types, use the unit model's serializer to translate the unit spec
# for each possible content unit type as determined by the search criteria
for unit_type_id in unit_type_ids:
serializer = units_controller.get_model_serializer_for_type(unit_type_id)
if serializer:
unit_spec_t = serializer.translate_filters(serializer.model, criteria.unit_spec)
# after translating the fields, this spec is only good for this unit type
unit_spec_t['_content_type_id'] = unit_type_id
unit_q |= mongoengine.Q(__raw__=unit_spec_t)
return repo_controller.find_repo_content_units(
repository=source_repo,
repo_content_unit_q=association_q,
units_q=unit_q,
unit_fields=criteria['unit_fields'],
yield_content_unit=True)
def remap_fields_with_serializer(content_unit):
"""Remap fields in place in a pymongo object using a mongoengine serializer
:param content_unit: Content unit to modify
:type content_unit: dict
This is a small workaround to help in cases where REST views are returning the older objects
coming out of pymongo, but still need to have their fields remapped according to the rules of
the pymongo serializer. As a workaround, this is a "best effort" function, so serialization
failures will be written to the debug log and not raise exeptions.
Usage of pymongo objects is deprecated. Since this function is only concerned with serializing
pymongo objects, its usage is also deprecated. Furthermore, this function is only intended to
be used in the final serialization of objects before presentation in the REST API.
"""
try:
content_type_id = content_unit['_content_type_id']
except KeyError:
# content unit didn't have a content type id, usually means we're testing...
_logger.debug(
'No _content_type_id found in content unit when remapping fields: '
'{0!r}'.format(content_unit))
return
serializer = units.get_model_serializer_for_type(content_type_id)
if serializer is None:
# No serializer, nothing to do
return
# build the list of fields that need to be remapped
field_map = {}
for field in content_unit:
remapped_field = serializer.translate_field_reverse(field)
if remapped_field != field:
field_map[field] = remapped_field
# do the remapping
for original_field, remapped_field in field_map.items():
content_unit[remapped_field] = content_unit.pop(original_field)
def remap_fields_with_serializer(content_unit):
"""Remap fields in place in a pymongo object using a mongoengine serializer
:param content_unit: Content unit to modify
:type content_unit: dict
This is a small workaround to help in cases where REST views are returning the older objects
coming out of pymongo, but still need to have their fields remapped according to the rules of
the pymongo serializer. As a workaround, this is a "best effort" function, so serialization
failures will be written to the debug log and not raise exeptions.
Usage of pymongo objects is deprecated. Since this function is only concerned with serializing
pymongo objects, its usage is also deprecated. Furthermore, this function is only intended to
be used in the final serialization of objects before presentation in the REST API.
"""
try:
content_type_id = content_unit['_content_type_id']
except KeyError:
# content unit didn't have a content type id, usually means we're testing...
_logger.debug('No _content_type_id found in content unit when remapping fields: '
'{0!r}'.format(content_unit))
return
serializer = units.get_model_serializer_for_type(content_type_id)
if serializer is None:
# No serializer, nothing to do
return
# build the list of fields that need to be remapped
field_map = {}
for field in content_unit:
remapped_field = serializer.translate_field_reverse(field)
if remapped_field != field:
field_map[field] = remapped_field
# do the remapping
for original_field, remapped_field in field_map.items():
content_unit[remapped_field] = content_unit.pop(original_field)
def test_returns_none_if_no_model(self, mock_get_model):
mock_get_model.return_value = None
serializer = units_controller.get_model_serializer_for_type('demo_model')
self.assertTrue(serializer is None)
def test_returns_serializer(self, mock_get_model):
mock_get_model.return_value = DemoModelWithSerializer
serializer = units_controller.get_model_serializer_for_type('demo_model')
self.assertTrue(isinstance(serializer, DemoModelSerializer))
self.assertTrue(hasattr(serializer, 'model'))
def test_returns_none_if_no_model(self, mock_get_model):
mock_get_model.return_value = None
serializer = units_controller.get_model_serializer_for_type(
'demo_model')
self.assertTrue(serializer is None)
