def test_returns_serializer(self, mock_get_model): mock_get_model.return_value = DemoModelWithSerializer serializer = units_controller.get_model_serializer_for_type( 'demo_model') self.assertTrue(isinstance(serializer, DemoModelSerializer)) self.assertTrue(hasattr(serializer, 'model'))
def get_repo_units(self, repo_id, content_type_id, additional_unit_fields=None): """ Searches for units in the given repository with given content type and returns a plugin unit containing unit id, unit key and any additional fields requested. :param repo_id: repo id :type repo_id: str :param content_type_id: content type id of the units :type content_type_id: str :param additional_unit_fields: additional fields from the unit metadata to be added in the result :type additional_unit_fields: list of str :return: list of unit instances :rtype: list of pulp.plugins.model.Unit """ additional_unit_fields = additional_unit_fields or [] try: unit_key_fields = units_controller.get_unit_key_fields_for_type(content_type_id) serializer = units_controller.get_model_serializer_for_type(content_type_id) # Query repo association manager to get all units of given type # associated with given repo. Limit data by requesting only the fields # that are needed. query_manager = managers.repo_unit_association_query_manager() unit_fields = list(unit_key_fields) + list(additional_unit_fields) criteria = UnitAssociationCriteria(association_fields=['unit_id'], unit_fields=unit_fields) units = query_manager.get_units_by_type(repo_id, content_type_id, criteria) # Convert units to plugin units with unit_key and required metadata values for each unit all_units = [] for unit in units: if serializer: serializer.serialize(unit['metadata']) unit_key = {} metadata = {} for k in unit_key_fields: unit_key[k] = unit['metadata'].pop(k) # Add unit_id and any additional unit fields requested by plugins metadata['unit_id'] = unit.pop('unit_id') for field in additional_unit_fields: metadata[field] = unit['metadata'].pop(field, None) u = Unit(content_type_id, unit_key, metadata, None) all_units.append(u) return all_units except Exception, e: _logger.exception(_('Exception from server getting units from repo [%s]' % repo_id)) raise self.exception_class(e), None, sys.exc_info()[2]
def _associated_units_by_type_cursor(unit_type_id, criteria, associated_unit_ids): """ Retrieve a pymongo cursor for units associated with a repository of a give unit type that meet to the provided criteria. :type unit_type_id: str :type criteria: UnitAssociationCriteria :type associated_unit_ids: list :rtype: pymongo.cursor.Cursor """ collection = types_db.type_units_collection(unit_type_id) serializer = units.get_model_serializer_for_type(unit_type_id) unit_filter = criteria.unit_filters.copy() if unit_filter and serializer: unit_filter = serializer.translate_filters(serializer.model, unit_filter) spec = { '$and': [ {'_id': {'$in': associated_unit_ids}}, unit_filter ] } fields = criteria.unit_fields if fields is not None: fields = list(fields) # The _content_type_id is required for looking up the association if '_content_type_id' not in fields: fields.append('_content_type_id') # translate incoming fields (e.g. id -> foo_id) if serializer: for index, field in enumerate(fields): fields[index] = serializer.translate_field(serializer.model, field) cursor = collection.find(spec, projection=fields) sort = criteria.unit_sort if sort is None: sort = [('_id', SORT_ASCENDING)] elif serializer: sort = list(sort) for index, (field, direction) in enumerate(sort): sort[index] = (serializer.translate_field(serializer.model, field), direction) cursor.sort(sort) return cursor
def _associated_units_by_type_cursor(unit_type_id, criteria, associated_unit_ids): """ Retrieve a pymongo cursor for units associated with a repository of a give unit type that meet to the provided criteria. :type unit_type_id: str :type criteria: UnitAssociationCriteria :type associated_unit_ids: list :rtype: pymongo.cursor.Cursor """ collection = types_db.type_units_collection(unit_type_id) serializer = units.get_model_serializer_for_type(unit_type_id) unit_filter = criteria.unit_filters.copy() if unit_filter and serializer: unit_filter = serializer.translate_filters(serializer.model, unit_filter) spec = {'$and': [{'_id': {'$in': associated_unit_ids}}, unit_filter]} fields = criteria.unit_fields if fields is not None: fields = list(fields) # The _content_type_id is required for looking up the association if '_content_type_id' not in fields: fields.append('_content_type_id') # translate incoming fields (e.g. id -> foo_id) if serializer: for index, field in enumerate(fields): fields[index] = serializer.translate_field( serializer.model, field) cursor = collection.find(spec, projection=fields) sort = criteria.unit_sort if sort is None: sort = [('_id', SORT_ASCENDING)] elif serializer: sort = list(sort) for index, (field, direction) in enumerate(sort): sort[index] = (serializer.translate_field( serializer.model, field), direction) cursor.sort(sort) return cursor
def get_results(cls, query, search_method, options, *args, **kwargs): """ Overrides the base class so additional information can optionally be added. """ type_id = kwargs['type_id'] serializer = units_controller.get_model_serializer_for_type(type_id) if serializer and query.get('filters') is not None: # if we have a model serializer, translate the filter for this content unit type query['filters'] = serializer.translate_filters(serializer.model, query['filters']) units = list(search_method(type_id, query)) units = [_process_content_unit(unit, type_id) for unit in units] if options.get('include_repos') is True: cls._add_repo_memberships(units, type_id) return units
def get_results(cls, query, search_method, options, *args, **kwargs): """ Overrides the base class so additional information can optionally be added. """ type_id = kwargs['type_id'] serializer = units_controller.get_model_serializer_for_type(type_id) if serializer and query.get('filters') is not None: # if we have a model serializer, translate the filter for this content unit type query['filters'] = serializer.translate_filters( serializer.model, query['filters']) units = list(search_method(type_id, query)) units = [_process_content_unit(unit, type_id) for unit in units] if options.get('include_repos') is True: cls._add_repo_memberships(units, type_id) return units
def serialize_unit_with_serializer(content_unit): """ Serialize unit in-place to its dictionary form using its serializer. This will also handle remapping that unit's fields from MongoEngine field names to their expected field names in the API This is a small workaround to help in cases where REST views are returning the older objects coming out of pymongo, but still need to have their fields remapped according to the rules of the pymongo serializer. As a workaround, this is a "best effort" function, so serialization failures will be written to the debug log and not raise exeptions. Usage of pymongo objects is deprecated. Since this function is only concerned with serializing pymongo objects, its usage is also deprecated. Furthermore, this function is only intended to be used in the final serialization of objects before presentation in the REST API, and as a result modifies objects in-place. :param content_unit: Content unit to be converted :type content_unit: dict """ try: content_type_id = content_unit['_content_type_id'] except KeyError: # content unit didn't have a content type id, usually means we're testing... _logger.debug( 'No _content_type_id found in content unit when remapping fields: ' '{0!r}'.format(content_unit)) return serializer = units.get_model_serializer_for_type(content_type_id) if serializer is None: # No serializer, nothing to do return # use the serialize method to do any needed unit-specific serialization on fields serializer.serialize(content_unit) # remap fields before responding -- do this after serializing exotic fields since # the serialize method will be working with the non-remapped field names. field_map = {} for field in content_unit: remapped_field = serializer.translate_field_reverse(field) if remapped_field != field: field_map[field] = remapped_field for original_field, remapped_field in field_map.items(): content_unit[remapped_field] = content_unit.pop(original_field)
def serialize_unit_with_serializer(content_unit): """ Serialize unit in-place to its dictionary form using its serializer. This will also handle remapping that unit's fields from MongoEngine field names to their expected field names in the API This is a small workaround to help in cases where REST views are returning the older objects coming out of pymongo, but still need to have their fields remapped according to the rules of the pymongo serializer. As a workaround, this is a "best effort" function, so serialization failures will be written to the debug log and not raise exeptions. Usage of pymongo objects is deprecated. Since this function is only concerned with serializing pymongo objects, its usage is also deprecated. Furthermore, this function is only intended to be used in the final serialization of objects before presentation in the REST API, and as a result modifies objects in-place. :param content_unit: Content unit to be converted :type content_unit: dict """ try: content_type_id = content_unit['_content_type_id'] except KeyError: # content unit didn't have a content type id, usually means we're testing... _logger.debug('No _content_type_id found in content unit when remapping fields: ' '{0!r}'.format(content_unit)) return serializer = units.get_model_serializer_for_type(content_type_id) if serializer is None: # No serializer, nothing to do return # use the serialize method to do any needed unit-specific serialization on fields serializer.serialize(content_unit) # remap fields before responding -- do this after serializing exotic fields since # the serialize method will be working with the non-remapped field names. field_map = {} for field in content_unit: remapped_field = serializer.translate_field_reverse(field) if remapped_field != field: field_map[field] = remapped_field for original_field, remapped_field in field_map.items(): content_unit[remapped_field] = content_unit.pop(original_field)
def _units_from_criteria(source_repo, criteria): """ Given a criteria, return an iterator of units :param source_repo: repository to look for units in :type source_repo: pulp.server.db.model.Repository :param criteria: criteria object to use for the search parameters :type criteria: pulp.server.db.model.criteria.UnitAssociationCriteria :return: generator of pulp.server.db.model.ContentUnit instances :rtype: generator """ association_q = mongoengine.Q(__raw__=criteria.association_spec) if criteria.type_ids: association_q &= mongoengine.Q(unit_type_id__in=criteria.type_ids) unit_type_ids = criteria.type_ids else: # don't need to limit the association_q by content type here # since filtering for every content_type seen in the repo is # achieved by limiting the query to the repo unit_type_ids = repo_controller.get_repo_unit_type_ids( source_repo.repo_id) # base unit_q, works as-is for non-mongoengine unit types unit_q = mongoengine.Q(__raw__=criteria.unit_spec) # for mongoengine unit types, use the unit model's serializer to translate the unit spec # for each possible content unit type as determined by the search criteria for unit_type_id in unit_type_ids: serializer = units_controller.get_model_serializer_for_type( unit_type_id) if serializer: unit_spec_t = serializer.translate_filters( serializer.model, criteria.unit_spec) # after translating the fields, this spec is only good for this unit type unit_spec_t['_content_type_id'] = unit_type_id unit_q |= mongoengine.Q(__raw__=unit_spec_t) return repo_controller.find_repo_content_units( repository=source_repo, repo_content_unit_q=association_q, units_q=unit_q, unit_fields=criteria['unit_fields'], yield_content_unit=True)
def _units_from_criteria(source_repo, criteria): """ Given a criteria, return an iterator of units :param source_repo: repository to look for units in :type source_repo: pulp.server.db.model.Repository :param criteria: criteria object to use for the search parameters :type criteria: pulp.server.db.model.criteria.UnitAssociationCriteria :return: generator of pulp.server.db.model.ContentUnit instances :rtype: generator """ association_q = mongoengine.Q(__raw__=criteria.association_spec) if criteria.type_ids: association_q &= mongoengine.Q(unit_type_id__in=criteria.type_ids) unit_type_ids = criteria.type_ids else: # don't need to limit the association_q by content type here # since filtering for every content_type seen in the repo is # achieved by limiting the query to the repo unit_type_ids = repo_controller.get_repo_unit_type_ids(source_repo.repo_id) # base unit_q, works as-is for non-mongoengine unit types unit_q = mongoengine.Q(__raw__=criteria.unit_spec) # for mongoengine unit types, use the unit model's serializer to translate the unit spec # for each possible content unit type as determined by the search criteria for unit_type_id in unit_type_ids: serializer = units_controller.get_model_serializer_for_type(unit_type_id) if serializer: unit_spec_t = serializer.translate_filters(serializer.model, criteria.unit_spec) # after translating the fields, this spec is only good for this unit type unit_spec_t['_content_type_id'] = unit_type_id unit_q |= mongoengine.Q(__raw__=unit_spec_t) return repo_controller.find_repo_content_units( repository=source_repo, repo_content_unit_q=association_q, units_q=unit_q, unit_fields=criteria['unit_fields'], yield_content_unit=True)
def remap_fields_with_serializer(content_unit): """Remap fields in place in a pymongo object using a mongoengine serializer :param content_unit: Content unit to modify :type content_unit: dict This is a small workaround to help in cases where REST views are returning the older objects coming out of pymongo, but still need to have their fields remapped according to the rules of the pymongo serializer. As a workaround, this is a "best effort" function, so serialization failures will be written to the debug log and not raise exeptions. Usage of pymongo objects is deprecated. Since this function is only concerned with serializing pymongo objects, its usage is also deprecated. Furthermore, this function is only intended to be used in the final serialization of objects before presentation in the REST API. """ try: content_type_id = content_unit['_content_type_id'] except KeyError: # content unit didn't have a content type id, usually means we're testing... _logger.debug( 'No _content_type_id found in content unit when remapping fields: ' '{0!r}'.format(content_unit)) return serializer = units.get_model_serializer_for_type(content_type_id) if serializer is None: # No serializer, nothing to do return # build the list of fields that need to be remapped field_map = {} for field in content_unit: remapped_field = serializer.translate_field_reverse(field) if remapped_field != field: field_map[field] = remapped_field # do the remapping for original_field, remapped_field in field_map.items(): content_unit[remapped_field] = content_unit.pop(original_field)
def remap_fields_with_serializer(content_unit): """Remap fields in place in a pymongo object using a mongoengine serializer :param content_unit: Content unit to modify :type content_unit: dict This is a small workaround to help in cases where REST views are returning the older objects coming out of pymongo, but still need to have their fields remapped according to the rules of the pymongo serializer. As a workaround, this is a "best effort" function, so serialization failures will be written to the debug log and not raise exeptions. Usage of pymongo objects is deprecated. Since this function is only concerned with serializing pymongo objects, its usage is also deprecated. Furthermore, this function is only intended to be used in the final serialization of objects before presentation in the REST API. """ try: content_type_id = content_unit['_content_type_id'] except KeyError: # content unit didn't have a content type id, usually means we're testing... _logger.debug('No _content_type_id found in content unit when remapping fields: ' '{0!r}'.format(content_unit)) return serializer = units.get_model_serializer_for_type(content_type_id) if serializer is None: # No serializer, nothing to do return # build the list of fields that need to be remapped field_map = {} for field in content_unit: remapped_field = serializer.translate_field_reverse(field) if remapped_field != field: field_map[field] = remapped_field # do the remapping for original_field, remapped_field in field_map.items(): content_unit[remapped_field] = content_unit.pop(original_field)
def test_returns_none_if_no_model(self, mock_get_model): mock_get_model.return_value = None serializer = units_controller.get_model_serializer_for_type('demo_model') self.assertTrue(serializer is None)
def test_returns_serializer(self, mock_get_model): mock_get_model.return_value = DemoModelWithSerializer serializer = units_controller.get_model_serializer_for_type('demo_model') self.assertTrue(isinstance(serializer, DemoModelSerializer)) self.assertTrue(hasattr(serializer, 'model'))
def test_returns_none_if_no_model(self, mock_get_model): mock_get_model.return_value = None serializer = units_controller.get_model_serializer_for_type( 'demo_model') self.assertTrue(serializer is None)