def associated_unit(association, unit): """ Create a dictionary that is a composite of a unit association and the unit. :param association: A unit association DB record. :type association: dict :param unit: A DB unit record. :type unit: dict :return: A composite of the unit association and the unit. :rtype: dict """ unit_key = {} unit_id = unit.pop('_id') type_id = association['unit_type_id'] for key in get_unit_key_fields_for_type(type_id): unit_key[key] = unit.pop(key, None) storage_dir = pulp_conf.get('server', 'storage_dir') storage_path = unit.pop('_storage_path', None) last_updated = unit.pop('_last_updated', 0.0) if storage_path: relative_path = storage_path[len(storage_dir):].lstrip('/') else: relative_path = None return dict(unit_id=unit_id, type_id=type_id, unit_key=unit_key, storage_path=storage_path, relative_path=relative_path, last_updated=last_updated, metadata=unit)
def find_unit_by_unit_key(self, type_id, unit_key): """ Finds a unit based on its unit key. If more than one unit comes back, an exception will be raised. @param type_id: indicates the type of units being retrieved @type type_id: str @param unit_key: the unit key for the unit @type unit_key: dict @return: a single unit @rtype: L{Unit} """ content_query_manager = manager_factory.content_query_manager() try: # this call returns a unit or raises MissingResource existing_unit = content_query_manager.get_content_unit_by_keys_dict( type_id, unit_key) unit_key_fields = units_controller.get_unit_key_fields_for_type( type_id) plugin_unit = common_utils.to_plugin_unit(existing_unit, type_id, unit_key_fields) return plugin_unit except pulp_exceptions.MissingResource: return None
def search_all_units(self, type_id, criteria): """ Searches for units of a given type in the server, regardless of their associations to any repositories. @param type_id: indicates the type of units being retrieved @type type_id: str @param criteria: used to query which units are returned @type criteria: pulp.server.db.model.criteria.Criteria @return: list of unit instances @rtype: list of L{Unit} """ try: query_manager = manager_factory.content_query_manager() units = query_manager.find_by_criteria(type_id, criteria) unit_key_fields = units_controller.get_unit_key_fields_for_type(type_id) transfer_units = [] for pulp_unit in units: u = common_utils.to_plugin_unit(pulp_unit, type_id, unit_key_fields) transfer_units.append(u) return transfer_units except Exception, e: _logger.exception('Exception from server requesting all units of type [%s]' % type_id) raise self.exception_class(e), None, sys.exc_info()[2]
def get_content_unit_keys(self, content_type, unit_ids): """ Return the keys and values that will uniquely identify the content units that match the given unique ids. @param content_type: unique id of content collection @type content_type: str @param unit_ids: list of unique content unit ids @type unit_ids: list of str's @return: two tuples of the same length, one of ids the second of key dicts the same index in each tuple corresponds to a single content unit @rtype: tuple of (possibly empty) tuples """ try: key_fields = units_controller.get_unit_key_fields_for_type( content_type) except ValueError: raise InvalidValue(['content_type']) all_fields = ['_id'] _flatten_keys(all_fields, key_fields) collection = content_types_db.type_units_collection(content_type) cursor = collection.find({'_id': { '$in': unit_ids }}, projection=all_fields) dicts = tuple(dict(d) for d in cursor) ids = tuple(d.pop('_id') for d in dicts) return (ids, dicts)
def associated_unit(association, unit): """ Create a dictionary that is a composite of a unit association and the unit. :param association: A unit association DB record. :type association: dict :param unit: A DB unit record. :type unit: dict :return: A composite of the unit association and the unit. :rtype: dict """ unit_key = {} unit_id = unit.pop('_id') type_id = association['unit_type_id'] for key in get_unit_key_fields_for_type(type_id): unit_key[key] = unit.pop(key, None) storage_dir = pulp_conf.get('server', 'storage_dir') storage_path = unit.pop('_storage_path', None) last_updated = unit.pop('_last_updated', 0.0) if storage_path: relative_path = storage_path[len(storage_dir):].lstrip('/') else: relative_path = None return dict( unit_id=unit_id, type_id=type_id, unit_key=unit_key, storage_path=storage_path, relative_path=relative_path, last_updated=last_updated, metadata=unit)
def get_repo_units(self, repo_id, content_type_id, additional_unit_fields=None): """ Searches for units in the given repository with given content type and returns a plugin unit containing unit id, unit key and any additional fields requested. :param repo_id: repo id :type repo_id: str :param content_type_id: content type id of the units :type content_type_id: str :param additional_unit_fields: additional fields from the unit metadata to be added in the result :type additional_unit_fields: list of str :return: list of unit instances :rtype: list of pulp.plugins.model.Unit """ additional_unit_fields = additional_unit_fields or [] try: unit_key_fields = units_controller.get_unit_key_fields_for_type( content_type_id) # Query repo association manager to get all units of given type # associated with given repo. Limit data by requesting only the fields # that are needed. query_manager = managers.repo_unit_association_query_manager() unit_fields = list(unit_key_fields) + list(additional_unit_fields) criteria = UnitAssociationCriteria(association_fields=['unit_id'], unit_fields=unit_fields) units = query_manager.get_units_by_type(repo_id, content_type_id, criteria) # Convert units to plugin units with unit_key and required metadata values for each unit all_units = [] for unit in units: unit_key = {} metadata = {} for k in unit_key_fields: unit_key[k] = unit['metadata'].pop(k) # Add unit_id and any additional unit fields requested by plugins metadata['unit_id'] = unit.pop('unit_id') for field in additional_unit_fields: metadata[field] = unit['metadata'].pop(field, None) u = Unit(content_type_id, unit_key, metadata, None) all_units.append(u) return all_units except Exception, e: _logger.exception( _('Exception from server getting units from repo [%s]' % repo_id)) raise self.exception_class(e), None, sys.exc_info()[2]
def test_returns_from_typedb(self, mock_type_def, mock_get_model): """ test when the requested type is defined the old way """ mock_get_model.return_value = None mock_type_def.return_value = {'unit_key': ['id']} ret = units_controller.get_unit_key_fields_for_type('faketype') self.assertEqual(ret, ('id',))
def test_returns_from_model(self, mock_type_def, mock_get_model): """ test when the requested type is a mongoengine model """ mock_get_model.return_value = DemoModel mock_type_def.return_value = None ret = units_controller.get_unit_key_fields_for_type(DemoModel.type_id) self.assertEqual(ret, DemoModel.unit_key_fields)
def test_returns_from_typedb(self, mock_type_def, mock_get_model): """ test when the requested type is defined the old way """ mock_get_model.return_value = None mock_type_def.return_value = {'unit_key': ['id']} ret = units_controller.get_unit_key_fields_for_type('faketype') self.assertEqual(ret, ('id',))
def _transfer_object_generator(): unit_key_fields_cache = {} for u in units: type_id = u['unit_type_id'] if type_id not in unit_key_fields_cache: fields = units_controller.get_unit_key_fields_for_type(type_id) unit_key_fields_cache[type_id] = fields yield common_utils.to_plugin_associated_unit(u, type_id, unit_key_fields_cache[type_id])
def test_returns_from_model(self, mock_type_def, mock_get_model): """ test when the requested type is a mongoengine model """ mock_get_model.return_value = DemoModel mock_type_def.return_value = None ret = units_controller.get_unit_key_fields_for_type(DemoModel.type_id) self.assertEqual(ret, DemoModel.unit_key_fields)
def get_repo_units(self, repo_id, content_type_id, additional_unit_fields=None): """ Searches for units in the given repository with given content type and returns a plugin unit containing unit id, unit key and any additional fields requested. :param repo_id: repo id :type repo_id: str :param content_type_id: content type id of the units :type content_type_id: str :param additional_unit_fields: additional fields from the unit metadata to be added in the result :type additional_unit_fields: list of str :return: list of unit instances :rtype: list of pulp.plugins.model.Unit """ additional_unit_fields = additional_unit_fields or [] try: unit_key_fields = units_controller.get_unit_key_fields_for_type(content_type_id) serializer = units_controller.get_model_serializer_for_type(content_type_id) # Query repo association manager to get all units of given type # associated with given repo. Limit data by requesting only the fields # that are needed. query_manager = managers.repo_unit_association_query_manager() unit_fields = list(unit_key_fields) + list(additional_unit_fields) criteria = UnitAssociationCriteria(association_fields=['unit_id'], unit_fields=unit_fields) units = query_manager.get_units_by_type(repo_id, content_type_id, criteria) # Convert units to plugin units with unit_key and required metadata values for each unit all_units = [] for unit in units: if serializer: serializer.serialize(unit['metadata']) unit_key = {} metadata = {} for k in unit_key_fields: unit_key[k] = unit['metadata'].pop(k) # Add unit_id and any additional unit fields requested by plugins metadata['unit_id'] = unit.pop('unit_id') for field in additional_unit_fields: metadata[field] = unit['metadata'].pop(field, None) u = Unit(content_type_id, unit_key, metadata, None) all_units.append(u) return all_units except Exception, e: _logger.exception(_('Exception from server getting units from repo [%s]' % repo_id)) raise self.exception_class(e), None, sys.exc_info()[2]
def create_transfer_units(associate_units): unit_key_fields = {} transfer_units = [] for unit in associate_units: type_id = unit['unit_type_id'] if type_id not in unit_key_fields: unit_key_fields[type_id] = units_controller.get_unit_key_fields_for_type(type_id) u = conduit_common_utils.to_plugin_associated_unit(unit, type_id, unit_key_fields[type_id]) transfer_units.append(u) return transfer_units
def create_transfer_units(associate_units): unit_key_fields = {} transfer_units = [] for unit in associate_units: type_id = unit['unit_type_id'] if type_id not in unit_key_fields: unit_key_fields[type_id] = units_controller.get_unit_key_fields_for_type(type_id) u = conduit_common_utils.to_plugin_associated_unit(unit, type_id, unit_key_fields[type_id]) transfer_units.append(u) return transfer_units
def _build_multi_keys_spec(content_type, unit_keys_dicts): """ Build a mongo db spec document for a query on the given content_type collection out of multiple content unit key dictionaries. :param content_type: unique id of the content type collection :type content_type: str :param unit_keys_dicts: list of key dictionaries whose key, value pairs can be used as unique identifiers for a single content unit :type unit_keys_dicts: list of dict :return: mongo db spec document for locating documents in a collection :rtype: dict :raises ValueError: if any of the key dictionaries do not match the unique fields of the collection """ # keys dicts validation constants try: unit_key_fields = units_controller.get_unit_key_fields_for_type( content_type) except ValueError: raise InvalidValue(['content_type']) key_fields = [] _flatten_keys(key_fields, unit_key_fields) key_fields_set = set(key_fields) extra_keys_msg = _( 'keys dictionary found with superfluous keys %(a)s, valid keys are %(b)s' ) missing_keys_msg = _( 'keys dictionary missing keys %(a)s, required keys are %(b)s') keys_errors = [] # Validate all of the keys in the unit_keys_dict for keys_dict in unit_keys_dicts: # keys dict validation keys_dict_set = set(keys_dict) extra_keys = keys_dict_set.difference(key_fields_set) if extra_keys: keys_errors.append(extra_keys_msg % { 'a': ','.join(extra_keys), 'b': ','.join(key_fields) }) missing_keys = key_fields_set.difference(keys_dict_set) if missing_keys: keys_errors.append(missing_keys_msg % { 'a': ','.join(missing_keys), 'b': ','.join(key_fields) }) if keys_errors: value_error_msg = '\n'.join(keys_errors) raise ValueError(value_error_msg) # Build the spec spec = {'$or': unit_keys_dicts} return spec
def delete(self, request, content_type): """ Dispatch a delete_orphans_by_type task. :param request: WSGI request object :type request: django.core.handlers.wsgi.WSGIRequest :param content_type: restrict the list of orphans to be deleted to this content type :type content_type: str :raises: OperationPostponed when an async operation is performed :raises: MissingResource when the content type does not exist """ try: # this tests if the type exists units.get_unit_key_fields_for_type(content_type) except ValueError: raise MissingResource(content_type_id=content_type) task_tags = [tags.resource_tag(tags.RESOURCE_CONTENT_UNIT_TYPE, 'orphans')] async_task = content_orphan.delete_orphans_by_type.apply_async( (content_type,), tags=task_tags ) raise OperationPostponed(async_task)
def delete_orphans_by_type(content_type_id, content_unit_ids=None): """ Delete the orphaned content units for the given content type. If the content_unit_ids parameter is not None, is acts as a filter of the specific orphaned content units that may be deleted. NOTE: this method deletes the content unit's bits from disk, if applicable. :param content_type_id: id of the content type :type content_type_id: basestring :param content_unit_ids: list of content unit ids to delete; None means delete them all :type content_unit_ids: iterable or None :return: count of units deleted :rtype: int """ content_units_collection = content_types_db.type_units_collection( content_type_id) content_model = plugin_api.get_unit_model_by_id(content_type_id) try: unit_key_fields = units_controller.get_unit_key_fields_for_type( content_type_id) except ValueError: raise MissingResource(content_type_id=content_type_id) fields = ('_id', '_storage_path') + unit_key_fields count = 0 for content_unit in OrphanManager.generate_orphans_by_type( content_type_id, fields=fields): if content_unit_ids is not None and content_unit[ '_id'] not in content_unit_ids: continue model.LazyCatalogEntry.objects( unit_id=content_unit['_id'], unit_type_id=content_type_id).delete() content_units_collection.remove(content_unit['_id']) if hasattr(content_model, 'do_post_delete_actions'): content_model.do_post_delete_actions(content_unit) storage_path = content_unit.get('_storage_path', None) if storage_path is not None: OrphanManager.delete_orphaned_file(storage_path) count += 1 return count
def delete(self, request, content_type): """ Dispatch a delete_orphans_by_type task. :param request: WSGI request object :type request: django.core.handlers.wsgi.WSGIRequest :param content_type: restrict the list of orphans to be deleted to this content type :type content_type: str :raises: OperationPostponed when an async operation is performed :raises: MissingResource when the content type does not exist """ try: # this tests if the type exists units.get_unit_key_fields_for_type(content_type) except ValueError: raise MissingResource(content_type_id=content_type) task_tags = [ tags.resource_tag(tags.RESOURCE_CONTENT_UNIT_TYPE, 'orphans') ] async_task = content_orphan.delete_orphans_by_type.apply_async( (content_type, ), tags=task_tags) raise OperationPostponed(async_task)
def _build_multi_keys_spec(content_type, unit_keys_dicts): """ Build a mongo db spec document for a query on the given content_type collection out of multiple content unit key dictionaries. :param content_type: unique id of the content type collection :type content_type: str :param unit_keys_dicts: list of key dictionaries whose key, value pairs can be used as unique identifiers for a single content unit :type unit_keys_dicts: list of dict :return: mongo db spec document for locating documents in a collection :rtype: dict :raises ValueError: if any of the key dictionaries do not match the unique fields of the collection """ # keys dicts validation constants try: unit_key_fields = units_controller.get_unit_key_fields_for_type(content_type) except ValueError: raise InvalidValue(['content_type']) key_fields = [] _flatten_keys(key_fields, unit_key_fields) key_fields_set = set(key_fields) extra_keys_msg = _('keys dictionary found with superfluous keys %(a)s, valid keys are %(b)s') missing_keys_msg = _('keys dictionary missing keys %(a)s, required keys are %(b)s') keys_errors = [] # Validate all of the keys in the unit_keys_dict for keys_dict in unit_keys_dicts: # keys dict validation keys_dict_set = set(keys_dict) extra_keys = keys_dict_set.difference(key_fields_set) if extra_keys: keys_errors.append(extra_keys_msg % {'a': ','.join(extra_keys), 'b': ','.join(key_fields)}) missing_keys = key_fields_set.difference(keys_dict_set) if missing_keys: keys_errors.append(missing_keys_msg % {'a': ','.join(missing_keys), 'b': ','.join(key_fields)}) if keys_errors: value_error_msg = '\n'.join(keys_errors) raise ValueError(value_error_msg) # Build the spec spec = {'$or': unit_keys_dicts} return spec
def _associated_units_by_type_cursor(unit_type_id, criteria, associated_unit_ids): """ Retrieve a pymongo cursor for units associated with a repository of a give unit type that meet to the provided criteria. :type unit_type_id: str :type criteria: UnitAssociationCriteria :type associated_unit_ids: list :rtype: pymongo.cursor.Cursor """ collection = types_db.type_units_collection(unit_type_id) spec = criteria.unit_filters.copy() spec['_id'] = {'$in': associated_unit_ids} fields = criteria.unit_fields # The _content_type_id is required for looking up the association. if fields is not None and '_content_type_id' not in fields: fields = list(fields) fields.append('_content_type_id') cursor = collection.find(spec, fields=fields) sort = criteria.unit_sort if sort is None: try: unit_key = units_controller.get_unit_key_fields_for_type( unit_type_id) except ValueError: unit_key = None if unit_key is not None: sort = [(u, SORT_ASCENDING) for u in unit_key] if sort is not None: cursor.sort(sort) return cursor
def generate_orphans_by_type_with_unit_keys(content_type_id): """ Return an generator of all orphaned content units of the given content type. Each content unit will contain the fields specified in the content type definition's search indexes. :param content_type_id: id of the content type :type content_type_id: basestring :return: generator of orphaned content units for the given content type :rtype: generator """ try: unit_key_fields = units_controller.get_unit_key_fields_for_type(content_type_id) except ValueError: raise MissingResource(content_type_id=content_type_id) fields = ['_id', '_content_type_id'] fields.extend(unit_key_fields) for content_unit in OrphanManager.generate_orphans_by_type(content_type_id, fields): yield content_unit
def generate_orphans_by_type_with_unit_keys(content_type_id): """ Return an generator of all orphaned content units of the given content type. Each content unit will contain the fields specified in the content type definition's search indexes. :param content_type_id: id of the content type :type content_type_id: basestring :return: generator of orphaned content units for the given content type :rtype: generator """ try: unit_key_fields = units_controller.get_unit_key_fields_for_type(content_type_id) except ValueError: raise MissingResource(content_type_id=content_type_id) fields = ['_id', '_content_type_id'] fields.extend(unit_key_fields) for content_unit in OrphanManager.generate_orphans_by_type(content_type_id, fields): yield content_unit
def _associated_units_by_type_cursor(unit_type_id, criteria, associated_unit_ids): """ Retrieve a pymongo cursor for units associated with a repository of a give unit type that meet to the provided criteria. :type unit_type_id: str :type criteria: UnitAssociationCriteria :type associated_unit_ids: list :rtype: pymongo.cursor.Cursor """ collection = types_db.type_units_collection(unit_type_id) spec = criteria.unit_filters.copy() spec['_id'] = {'$in': associated_unit_ids} fields = criteria.unit_fields # The _content_type_id is required for looking up the association. if fields is not None and '_content_type_id' not in fields: fields = list(fields) fields.append('_content_type_id') cursor = collection.find(spec, fields=fields) sort = criteria.unit_sort if sort is None: try: unit_key = units_controller.get_unit_key_fields_for_type(unit_type_id) except ValueError: unit_key = None if unit_key is not None: sort = [(u, SORT_ASCENDING) for u in unit_key] if sort is not None: cursor.sort(sort) return cursor
def find_unit_by_unit_key(self, type_id, unit_key): """ Finds a unit based on its unit key. If more than one unit comes back, an exception will be raised. @param type_id: indicates the type of units being retrieved @type type_id: str @param unit_key: the unit key for the unit @type unit_key: dict @return: a single unit @rtype: L{Unit} """ content_query_manager = manager_factory.content_query_manager() try: # this call returns a unit or raises MissingResource existing_unit = content_query_manager.get_content_unit_by_keys_dict(type_id, unit_key) unit_key_fields = units_controller.get_unit_key_fields_for_type(type_id) plugin_unit = common_utils.to_plugin_unit(existing_unit, type_id, unit_key_fields) return plugin_unit except pulp_exceptions.MissingResource: return None
def get_content_unit_keys(self, content_type, unit_ids): """ Return the keys and values that will uniquely identify the content units that match the given unique ids. @param content_type: unique id of content collection @type content_type: str @param unit_ids: list of unique content unit ids @type unit_ids: list of str's @return: two tuples of the same length, one of ids the second of key dicts the same index in each tuple corresponds to a single content unit @rtype: tuple of (possibly empty) tuples """ try: key_fields = units_controller.get_unit_key_fields_for_type(content_type) except ValueError: raise InvalidValue(['content_type']) all_fields = ['_id'] _flatten_keys(all_fields, key_fields) collection = content_types_db.type_units_collection(content_type) cursor = collection.find({'_id': {'$in': unit_ids}}, projection=all_fields) dicts = tuple(dict(d) for d in cursor) ids = tuple(d.pop('_id') for d in dicts) return (ids, dicts)
def delete_orphan_content_units_by_type(type_id, content_unit_ids=None): """ Delete the orphaned content units for the given content type. This method only applies to new style content units that are loaded via entry points NOTE: this method deletes the content unit's bits from disk, if applicable. :param type_id: id of the content type :type type_id: basestring :param content_unit_ids: list of content unit ids to delete; None means delete them all :type content_unit_ids: iterable or None :return: count of units deleted :rtype: int """ # get the model matching the type content_model = plugin_api.get_unit_model_by_id(type_id) try: unit_key_fields = units_controller.get_unit_key_fields_for_type( type_id) except ValueError: raise MissingResource(content_type_id=type_id) fields = ('id', '_storage_path') + unit_key_fields if content_unit_ids: query_sets = [] for page in plugin_misc.paginate(content_unit_ids): qs = content_model.objects(id__in=page).only(*fields) query_sets.append(qs) content_units = itertools.chain(*query_sets) else: content_units = content_model.objects.only(*fields) count = 0 # Paginate the content units for units_group in plugin_misc.paginate(content_units): # Build the list of ids to search for an easier way to access units in the group by id unit_dict = dict() for unit in units_group: unit_dict[unit.id] = unit id_list = list(unit_dict.iterkeys()) # Clear the units that are currently associated from unit_dict non_orphan = model.RepositoryContentUnit.objects(unit_id__in=id_list)\ .distinct('unit_id') for non_orphan_id in non_orphan: unit_dict.pop(non_orphan_id) # Remove the unit, lazy catalog entries, and any content in storage. for unit_to_delete in unit_dict.itervalues(): model.LazyCatalogEntry.objects( unit_id=str(unit_to_delete.id), unit_type_id=str(type_id)).delete() unit_to_delete.delete() if hasattr(content_model, 'do_post_delete_actions'): content_model.do_post_delete_actions(unit_to_delete) if unit_to_delete._storage_path: OrphanManager.delete_orphaned_file( unit_to_delete._storage_path) count += 1 return count