Beispiel #1
0
    def search_all_units(self, type_id, criteria):
        """
        Searches for units of a given type in the server, regardless of their
        associations to any repositories.

        @param type_id: indicates the type of units being retrieved
        @type  type_id: str
        @param criteria: used to query which units are returned
        @type  criteria: pulp.server.db.model.criteria.Criteria

        @return: list of unit instances
        @rtype:  list of L{Unit}
        """

        try:
            query_manager = manager_factory.content_query_manager()
            units = query_manager.find_by_criteria(type_id, criteria)
            type_def = types_db.type_definition(type_id)

            transfer_units = []
            for pulp_unit in units:
                u = common_utils.to_plugin_unit(pulp_unit, type_def)
                transfer_units.append(u)

            return transfer_units

        except Exception, e:
            _LOG.exception('Exception from server requesting all units of type [%s]' % type_id)
            raise self.exception_class(e), None, sys.exc_info()[2]
Beispiel #2
0
    def find_unit_by_unit_key(self, type_id, unit_key):
        """
        Finds a unit based on its unit key. If more than one unit comes back,
        an exception will be raised.

        @param type_id: indicates the type of units being retrieved
        @type  type_id: str
        @param unit_key: the unit key for the unit
        @type  unit_key: dict

        @return: a single unit
        @rtype:  L{Unit}
        """
        content_query_manager = manager_factory.content_query_manager()
        try:
            # this call returns a unit or raises MissingResource
            existing_unit = content_query_manager.get_content_unit_by_keys_dict(
                type_id, unit_key)
            unit_key_fields = units_controller.get_unit_key_fields_for_type(
                type_id)
            plugin_unit = common_utils.to_plugin_unit(existing_unit, type_id,
                                                      unit_key_fields)
            return plugin_unit
        except pulp_exceptions.MissingResource:
            return None
Beispiel #3
0
 def GET(self, type_id):
     """
     List all the available content units.
     """
     cqm = factory.content_query_manager()
     units = cqm.find_by_criteria(type_id, Criteria())
     return self.ok([self.process_unit(unit) for unit in units])
Beispiel #4
0
    def delete_orphans_by_id(self, orphans):
        """
        Delete a list of orphaned content units by their content type and unit ids.
        @param orphans: list of documents with 'content_type' and 'content_id' keys
        @type  orphans: list
        """
        # XXX this does no validation of the orphans

        # munge the orphans into something more programmatic-ly convenient
        orphans_by_id = {}
        for o in orphans:
            if 'content_type_id' not in o or 'unit_id' not in o:
                raise pulp_exceptions.InvalidValue(['content_type_id', 'unit_id'])
            id_list = orphans_by_id.setdefault(o['content_type_id'], [])
            id_list.append(o['unit_id'])

        # iterate through the types and ids
        content_query_manager = manager_factory.content_query_manager()
        for content_type, content_id_list in orphans_by_id.items():

            # build a list of the on-disk contents
            orphaned_paths = []
            for unit_id in content_id_list:
                content_unit = content_query_manager.get_content_unit_by_id(content_type, unit_id, model_fields=['_storage_path'])
                if content_unit['_storage_path'] is not None:
                    orphaned_paths.append(content_unit['_storage_path'])

            # remove the orphans from the db
            collection = content_types_db.type_units_collection(content_type)
            spec = {'_id': {'$in': content_id_list}}
            collection.remove(spec, safe=True)

            # delete the on-disk contents
            for path in orphaned_paths:
                self.delete_orphaned_file(path)
Beispiel #5
0
 def GET(self, type_id):
     """
     List all the available content units.
     """
     cqm = factory.content_query_manager()
     units = cqm.find_by_criteria(type_id, Criteria())
     return self.ok([self.process_unit(unit) for unit in units])
Beispiel #6
0
    def test_syntactic_sugar_methods(self):
        """
        Tests the syntactic sugar methods for retrieving specific managers.
        """
        # Setup
        factory.initialize()

        # Test
        self.assertTrue(isinstance(factory.authentication_manager(), AuthenticationManager))
        self.assertTrue(isinstance(factory.cert_generation_manager(), CertGenerationManager))
        self.assertTrue(isinstance(factory.certificate_manager(), CertificateManager))
        self.assertTrue(isinstance(factory.password_manager(), PasswordManager))
        self.assertTrue(isinstance(factory.permission_manager(), PermissionManager))
        self.assertTrue(isinstance(factory.permission_query_manager(), PermissionQueryManager))
        self.assertTrue(isinstance(factory.role_manager(), RoleManager))
        self.assertTrue(isinstance(factory.role_query_manager(), RoleQueryManager))
        self.assertTrue(isinstance(factory.user_manager(), UserManager))             
        self.assertTrue(isinstance(factory.user_query_manager(), UserQueryManager))
        self.assertTrue(isinstance(factory.repo_manager(), RepoManager))
        self.assertTrue(isinstance(factory.repo_unit_association_manager(), RepoUnitAssociationManager))
        self.assertTrue(isinstance(factory.repo_publish_manager(), RepoPublishManager))
        self.assertTrue(isinstance(factory.repo_query_manager(), RepoQueryManager))
        self.assertTrue(isinstance(factory.repo_sync_manager(), RepoSyncManager))
        self.assertTrue(isinstance(factory.content_manager(), ContentManager))
        self.assertTrue(isinstance(factory.content_query_manager(), ContentQueryManager))
        self.assertTrue(isinstance(factory.content_upload_manager(), ContentUploadManager))
        self.assertTrue(isinstance(factory.consumer_manager(), ConsumerManager))
        self.assertTrue(isinstance(factory.topic_publish_manager(), TopicPublishManager))
Beispiel #7
0
    def search_all_units(self, type_id, criteria):
        """
        Searches for units of a given type in the server, regardless of their
        associations to any repositories.

        @param type_id: indicates the type of units being retrieved
        @type  type_id: str
        @param criteria: used to query which units are returned
        @type  criteria: pulp.server.db.model.criteria.Criteria

        @return: list of unit instances
        @rtype:  list of L{Unit}
        """

        try:
            query_manager = manager_factory.content_query_manager()
            units = query_manager.find_by_criteria(type_id, criteria)
            type_def = types_db.type_definition(type_id)

            transfer_units = []
            for pulp_unit in units:
                u = common_utils.to_plugin_unit(pulp_unit, type_def)
                transfer_units.append(u)

            return transfer_units

        except Exception, e:
            logger.exception(
                'Exception from server requesting all units of type [%s]' %
                type_id)
            raise self.exception_class(e), None, sys.exc_info()[2]
Beispiel #8
0
    def get(self, request, type_id):
        """
        Return a response with a serialized list of the content units of the specified type.

        :param request: WSGI request object
        :type  request: django.core.handlers.wsgi.WSGIRequest
        :param type_id: the list of content units will be limited to this type
        :type  type_id: str

        :return: response with a serialized list of dicts, one for each unit of the type.
        :rtype: django.http.HttpResponse
        """
        cqm = factory.content_query_manager()
        all_units = cqm.find_by_criteria(type_id, Criteria())
        all_processed_units = []
        for unit in all_units:
            unit = serialization.content.content_unit_obj(unit)
            unit.update({
                '_href':
                '/'.join(
                    [request.get_full_path().rstrip('/'), unit['_id'], ''])
            })
            unit.update({
                'children':
                serialization.content.content_unit_child_link_objs(unit)
            })
            all_processed_units.append(unit)

        return generate_json_response_with_pulp_encoder(all_processed_units)
Beispiel #9
0
    def _update_unit(self, unit, pulp_unit):
        """
        Update a unit. If it is not found, add it.

        :param unit:        the unit to be updated
        :type  unit:        pulp.plugins.model.Unit
        :param pulp_unit:   the unit to be updated, as a dict
        :type  pulp_unit:   dict

        :return:    id of the updated unit
        :rtype:     basestring
        """
        content_query_manager = manager_factory.content_query_manager()
        content_manager = manager_factory.content_manager()
        try:
            existing_unit = content_query_manager.get_content_unit_by_keys_dict(
                unit.type_id, unit.unit_key)
            unit_id = existing_unit['_id']
            content_manager.update_content_unit(unit.type_id, unit_id,
                                                pulp_unit)
            self._updated_count += 1
            return unit_id
        except MissingResource:
            logger.debug(
                _('cannot update unit; does not exist. adding instead.'))
            return self._add_unit(unit, pulp_unit)
Beispiel #10
0
    def search_unit_ids(self, type_id, criteria):
        """
        Searches for units of a given type in the server, regardless of their
        associations to any repositories and returns a list of unit ids.

        @param type_id: indicates the type of units being retrieved
        @type  type_id: str
        @param criteria: used to query which units are returned
        @type  criteria: pulp.server.db.model.criteria.Criteria

        @return: list of unit ids
        @rtype:  list of str
        """
        try:
            query_manager = managers.content_query_manager()
            criteria["fields"] = ['_id']
            units = query_manager.find_by_criteria(type_id, criteria)
            unit_ids = [u['_id'] for u in units]
            return unit_ids

        except Exception, e:
            _LOG.exception(
                _('Exception from server searching units of type [%s]' %
                  type_id))
            raise self.exception_class(e), None, sys.exc_info()[2]
Beispiel #11
0
    def get(self, request, type_id):
        """
        Return a response containing a dict with info about a content type.

        :param request: WSGI request object
        :type  request: django.core.handlers.wsgi.WSGIRequest
        :param type_id: type of content unit
        :type  type_id: str

        :return: response containing a dict that contains info about a content type or
                 404 response if the specified content type is not found.
        :rtype : django.http.HttpResponse or HttpResponseNotFound
        """
        cqm = factory.content_query_manager()
        content_type = cqm.get_content_type(type_id)
        if content_type is None:
            msg = _('No content type resource: %(r)s') % {'r': type_id}
            return generate_json_response(msg,
                                          response_class=HttpResponseNotFound)
        resource = serialization.content.content_type_obj(content_type)
        # These urls are not valid endpoints but are left here for for semantic versioning.
        # BZ - 1187287
        links = {
            'actions': {
                '_href':
                '/'.join([request.get_full_path().rstrip('/'), 'actions/'])
            },
            'content_units': {
                '_href':
                '/'.join([request.get_full_path().rstrip('/'), 'units/'])
            }
        }
        resource.update(links)
        return generate_json_response_with_pulp_encoder(resource)
Beispiel #12
0
    def test_syntactic_sugar_methods(self):
        """
        Tests the syntactic sugar methods for retrieving specific managers.
        """
        # Setup
        factory.initialize()

        # Test
        self.assertTrue(isinstance(factory.authentication_manager(), AuthenticationManager))
        self.assertTrue(isinstance(factory.cert_generation_manager(), CertGenerationManager))
        self.assertTrue(isinstance(factory.certificate_manager(), CertificateManager))
        self.assertTrue(isinstance(factory.password_manager(), PasswordManager))
        self.assertTrue(isinstance(factory.permission_manager(), PermissionManager))
        self.assertTrue(isinstance(factory.permission_query_manager(), PermissionQueryManager))
        self.assertTrue(isinstance(factory.role_manager(), RoleManager))
        self.assertTrue(isinstance(factory.role_query_manager(), RoleQueryManager))
        self.assertTrue(isinstance(factory.user_manager(), UserManager))
        self.assertTrue(isinstance(factory.user_query_manager(), UserQueryManager))
        self.assertTrue(isinstance(factory.repo_manager(), RepoManager))
        self.assertTrue(isinstance(factory.repo_unit_association_manager(),
                                   RepoUnitAssociationManager))
        self.assertTrue(isinstance(factory.repo_publish_manager(), RepoPublishManager))
        self.assertTrue(isinstance(factory.repo_query_manager(), RepoQueryManager))
        self.assertTrue(isinstance(factory.repo_sync_manager(), RepoSyncManager))
        self.assertTrue(isinstance(factory.content_manager(), ContentManager))
        self.assertTrue(isinstance(factory.content_query_manager(), ContentQueryManager))
        self.assertTrue(isinstance(factory.content_upload_manager(), ContentUploadManager))
        self.assertTrue(isinstance(factory.consumer_manager(), ConsumerManager))
        self.assertTrue(isinstance(factory.topic_publish_manager(), TopicPublishManager))
Beispiel #13
0
    def get(self, request, type_id, unit_id):
        """
        Return a response containing information about the requested content unit.

        :param request: WSGI request object
        :type  request: django.core.handlers.wsgi.WSGIRequest
        :param type_id: type of content contained in the repo
        :type  type_id: str
        :param unit_id: unique id of a unit
        :type  unit_id: str

        :return: response containing a dict with data about requested unit
        :rtype : django.http.HttpResponse
        """
        cqm = factory.content_query_manager()
        try:
            unit = cqm.get_content_unit_by_id(type_id, unit_id)
        except MissingResource:
            msg = _('No content unit resource: %(r)s') % {'r': unit_id}
            return generate_json_response(msg,
                                          response_class=HttpResponseNotFound)

        resource = serial_content.content_unit_obj(unit)
        resource.update({
            'children':
            serial_content.content_unit_child_link_objs(resource)
        })
        return generate_json_response_with_pulp_encoder(resource)
Beispiel #14
0
    def get(self, request, type_id):
        """
        Return a response containing a dict with info about a content type.

        :param request: WSGI request object
        :type  request: django.core.handlers.wsgi.WSGIRequest
        :param type_id: type of content unit
        :type  type_id: str

        :return: response containing a dict that contains info about a content type or
                 404 response if the specified content type is not found.
        :rtype : django.http.HttpResponse or HttpResponseNotFound
        """
        cqm = factory.content_query_manager()
        content_type = cqm.get_content_type(type_id)
        if content_type is None:
            msg = _('No content type resource: %(r)s') % {'r': type_id}
            return generate_json_response(msg, response_class=HttpResponseNotFound)
        resource = serialization.content.content_type_obj(content_type)
        # These urls are not valid endpoints but are left here for for semantic versioning.
        # BZ - 1187287
        links = {
            'actions': {'_href': '/'.join([request.get_full_path().rstrip('/'), 'actions/'])},
            'content_units': {'_href': '/'.join([request.get_full_path().rstrip('/'), 'units/'])}
        }
        resource.update(links)
        return generate_json_response_with_pulp_encoder(resource)
Beispiel #15
0
    def __unit_ids_to_plugin_unit_keys(self, unit_ids_by_type, repo_ids):
        """
        Parse a dictionary of unit ids keyed by content type id and return a dictionary of 
        corresponding plugin unit keys keyed by content type id.

        :param unit_ids_by_type: dictionary of <content type id> : <list of unit ids>
        :type unit_ids_by_type: dict

        :return: if units are specified, return the corresponding plugin unit_keys. If unit_ids_by_type dict
                 is empty, return plugin unit keys corresponging to all units in given repo ids.
                 If unit ids list for a particular unit type is empty, return all plugin unit_keys
                 in given repo ids with that unit type.
        :rtype: dict
        """
        repo_unit_association_query_manager = managers.repo_unit_association_query_manager()
        content_query_manager = managers.content_query_manager()

        result_unit_keys = {}

        if unit_ids_by_type is not None:
            for unit_type_id, unit_ids in unit_ids_by_type.items():
                # Get unit type specific collection
                collection = content_query_manager.get_content_unit_collection(type_id=unit_type_id)
                type_def = content_types_db.type_definition(unit_type_id)
                if not unit_ids:
                    # If unit_list is empty for a unit_type, consider all units of specific type
                    criteria = UnitAssociationCriteria(unit_fields = ['unit_id'])
                    for repo_id in repo_ids:
                        repo_units = repo_unit_association_query_manager.get_units_by_type(repo_id, unit_type_id, criteria)
                        # Get metadata for each unit from type specific collection
                        pulp_units = [collection.find_one({'_id': u['unit_id']}) for u in repo_units]
                        # Convert pulp units to plugin unit keys
                        plugin_unit_keys = [common_utils.to_plugin_unit(u, type_def).unit_key for u in pulp_units]
                        result_unit_keys.setdefault(unit_type_id, []).extend(plugin_unit_keys)
                else:
                    # Get metadata for each unit from type specific collection
                    pulp_units = [collection.find_one({'_id': unit_id}) for unit_id in unit_ids]
                    # Convert pulp units to plugin unit keys
                    plugin_unit_keys = [common_utils.to_plugin_unit(u, type_def).unit_key for u in pulp_units]
                    result_unit_keys.setdefault(unit_type_id, []).extend(plugin_unit_keys)
        else:
            # If units are not specified, consider all units in given repos.
            for repo_id in repo_ids:
                all_unit_type_ids = content_types_db.all_type_ids()
                for unit_type_id in all_unit_type_ids:
                    criteria = UnitAssociationCriteria(type_ids=[unit_type_id], unit_fields = ['unit_id', 'unit_type_id'])
                    repo_units = repo_unit_association_query_manager.get_units(repo_id, criteria)

                    # Get unit metadata for each unit from type specific collection
                    collection = content_query_manager.get_content_unit_collection(type_id=unit_type_id)
                    pulp_units = [collection.find_one({'_id': u['unit_id']}) for u in repo_units]
  
                    # Convert pulp units to plugin unit keys
                    type_def = content_types_db.type_definition(unit_type_id)
                    plugin_unit_keys = [common_utils.to_plugin_unit(u, type_def).unit_key for u in pulp_units]
                    result_unit_keys.setdefault(unit_type_id, []).extend(plugin_unit_keys)

        return result_unit_keys
Beispiel #16
0
    def __parse_units(self, user_units, repo_ids):
        """
        Parse units specified by user and return a dictionary of all plugin unit_keys 
        to be considered for applicability keyed by unit_type_id.

        :param user_units: dictionary of unit metadata filters keyed by unit-type-id specified by user
        :type user_units: dict

        :return: if specific units are specified, return the corresponding plugin unit_keys. If units dict is empty, 
                 return all plugin unit_keys corresponging to units in given repo ids keyed by unit_type_id. 
                 If units list for a particular unit type in units is empty, return all plugin unit_keys 
                 in given repo ids with that unit type keyed by unit_type_id.
        :rtype: dict
        """
        repo_unit_association_query_manager = managers.repo_unit_association_query_manager()
        content_query_manager = managers.content_query_manager()

        result_unit_keys = {}
        if user_units is not None:
            for unit_type_id, unit_list in user_units.items():
                # Get unit type specific collection
                collection = content_query_manager.get_content_unit_collection(type_id=unit_type_id)
                type_def = content_types_db.type_definition(unit_type_id)
                if not unit_list:
                    # If unit_list is empty for a unit_type, consider all units of specific type
                    criteria = UnitAssociationCriteria(unit_fields = ['unit_id'])
                    for repo_id in repo_ids:
                        repo_units = repo_unit_association_query_manager.get_units_by_type(repo_id, unit_type_id, criteria)
                        # Get unit metadata for each unit from type specific collection
                        pulp_units = [collection.find_one({'_id': u['unit_id']}) for u in repo_units]
                        plugin_units = [common_utils.to_plugin_unit(u, type_def) for u in pulp_units]
                        plugin_unit_keys = [u.unit_key for u in plugin_units]
                        result_unit_keys.setdefault(unit_type_id, []).extend(plugin_unit_keys)
                else:
                    for unit in unit_list:
                        criteria = UnitAssociationCriteria(unit_filters=unit, unit_fields = ['unit_id'])
                        for repo_id in repo_ids:
                            repo_units = repo_unit_association_query_manager.get_units_by_type(repo_id, unit_type_id, criteria)
                            # Get unit metadata for each unit from type specific collection
                            pulp_units = [collection.find_one({'_id': u['unit_id']}) for u in repo_units]
                            plugin_units = [common_utils.to_plugin_unit(u, type_def) for u in pulp_units]
                            plugin_unit_keys = [u.unit_key for u in plugin_units]
                            result_unit_keys.setdefault(unit_type_id, []).extend(plugin_unit_keys)
        else:
            # If units are not specified, consider all units in repo_ids list.
            for repo_id in repo_ids:
                criteria = UnitAssociationCriteria(unit_fields = ['unit_id','unit_type_id'])
                repo_units = repo_unit_association_query_manager.get_units(repo_id, criteria)
                # Get unit metadata for each unit from type specific collection
                for repo_unit in repo_units:
                    collection = content_query_manager.get_content_unit_collection(type_id=repo_unit['unit_type_id'])
                    type_def = content_types_db.type_definition(repo_unit['unit_type_id'])
                    pulp_unit = collection.find_one({'_id': repo_unit['unit_id']})
                    plugin_unit = common_utils.to_plugin_unit(pulp_unit, type_def)
                    result_unit_keys.setdefault(repo_unit['unit_type_id'], []).append(plugin_unit.unit_key)

        return result_unit_keys
Beispiel #17
0
    def init_unit(self, type_id, unit_key, metadata, relative_path):
        """
        Initializes the Pulp representation of a content unit. The conduit will
        use the provided information to generate any unit metadata that it needs
        to. A populated transfer object representation of the unit will be
        returned from this call. The returned unit should be used in subsequent
        calls to this conduit.

        This call makes no changes to the Pulp server. At the end of this call,
        the unit's id field will *not* be populated.

        The unit_key and metadata will be merged as they are saved in Pulp to
        form the full representation of the unit. If values are specified in
        both dictionaries, the unit_key value takes precedence.

        If the importer wants to save the bits for the unit, the relative_path
        value should be used to indicate a unique -- with respect to the type
        of unit -- relative path where it will be saved. Pulp will convert this
        into an absolute path on disk where the unit should actually be saved.
        The absolute path is stored in the returned unit object.

        @param type_id: must correspond to a type definition in Pulp
        @type  type_id: str

        @param unit_key: dictionary of whatever fields are necessary to uniquely
                         identify this unit from others of the same type
        @type  unit_key: dict

        @param metadata: dictionary of key-value pairs to describe the unit
        @type  metadata: dict

        @param relative_path: see above; may be None
        @type  relative_path: str, None

        @return: object representation of the unit, populated by Pulp with both
                 provided and derived values
        @rtype:  pulp.plugins.model.Unit
        """

        try:
            # Generate the storage location
            if relative_path is not None:
                content_query_manager = manager_factory.content_query_manager()
                path = content_query_manager.request_content_unit_file_path(
                    type_id, relative_path)
            else:
                path = None
            u = Unit(type_id, unit_key, metadata, path)
            return u
        except Exception, e:
            msg = _(
                'Exception from server requesting unit filename for relative path [%s]'
            )
            msg = msg % relative_path
            _logger.exception(msg)
            raise ImporterConduitException(e), None, sys.exc_info()[2]
Beispiel #18
0
    def delete_all_orphans(self):
        """
        Delete all orphaned content units.
        """

        # iterate through the types and delete all orphans of each type
        content_query_manager = manager_factory.content_query_manager()
        content_types = content_query_manager.list_content_types()
        for content_type in content_types:
            self.delete_orphans_by_type(content_type)
Beispiel #19
0
    def unassociate_all_by_ids(self, repo_id, unit_type_id, unit_id_list, owner_type, owner_id):
        """
        Removes the association between a repo and a number of units. Only the
        association made by the given owner will be removed. It is possible the
        repo will still have a manually created association will for the unit.

        @param repo_id: identifies the repo
        @type  repo_id: str

        @param unit_type_id: identifies the type of units being removed
        @type  unit_type_id: str

        @param unit_id_list: list of unique identifiers for units within the given type
        @type  unit_id_list: list of str

        @param owner_type: category of the caller who created the association;
                           must be one of the OWNER_* variables in this module
        @type  owner_type: str

        @param owner_id: identifies the caller who created the association, either
                         the importer ID or user login
        @type  owner_id: str
        """
        spec = {'repo_id' : repo_id,
                'unit_type_id' : unit_type_id,
                'unit_id' : {'$in' : unit_id_list},
                'owner_type' : owner_type,
                'owner_id' : owner_id,
                }

        unit_coll = RepoContentUnit.get_collection()
        unit_coll.remove(spec, safe=True)

        unique_count = sum(1 for unit_id in unit_id_list
            if not self.association_exists(repo_id, unit_id, unit_type_id))

        # update the count of associated units on the repo object
        if unique_count:
            manager_factory.repo_manager().update_unit_count(
                repo_id, -unique_count)

        try:
            repo_query_manager = manager_factory.repo_query_manager()
            repo = repo_query_manager.get_repository(repo_id)

            content_query_manager = manager_factory.content_query_manager()
            content_units = content_query_manager.get_multiple_units_by_ids(unit_type_id, unit_id_list)

            importer_manager = manager_factory.repo_importer_manager()
            importer = importer_manager.get_importer(repo_id)

            importer.remove_units(repo, content_units)
        except:
            _LOG.exception('Exception informing importer for [%s] of unassociation' % repo_id)
Beispiel #20
0
    def __init__(self, repo_id, importer_id):
        RepoScratchPadMixin.__init__(self, repo_id, ImporterConduitException)
        ImporterScratchPadMixin.__init__(self, repo_id, importer_id)
        AddUnitMixin.__init__(self, repo_id, importer_id)
        SingleRepoUnitsMixin.__init__(self, repo_id, ImporterConduitException)
        StatusMixin.__init__(self, importer_id, ImporterConduitException)
        SearchUnitsMixin.__init__(self, ImporterConduitException)

        self._association_manager = manager_factory.repo_unit_association_manager()
        self._content_query_manager = manager_factory.content_query_manager()

        self._removed_count = 0
Beispiel #21
0
 def GET(self):
     """
     List the available content types.
     """
     collection = []
     cqm = factory.content_query_manager()
     type_ids = cqm.list_content_types()
     for id in type_ids:
         link = serialization.link.child_link_obj(id)
         link.update({'content_type': id})
         collection.append(link)
     return self.ok(collection)
Beispiel #22
0
 def GET(self, type_id, unit_id):
     """
     Return information about a content unit.
     """
     cqm = factory.content_query_manager()
     try:
         unit = cqm.get_content_unit_by_id(type_id, unit_id)
     except MissingResource:
         return self.not_found(_("No content unit resource: %(r)s") % {"r": unit_id})
     resource = serialization.content.content_unit_obj(unit)
     resource.update({"children": serialization.content.content_unit_child_link_objs(resource)})
     return self.ok(resource)
Beispiel #23
0
    def init_unit(self, type_id, unit_key, metadata, relative_path):
        """
        Initializes the Pulp representation of a content unit. The conduit will
        use the provided information to generate any unit metadata that it needs
        to. A populated transfer object representation of the unit will be
        returned from this call. The returned unit should be used in subsequent
        calls to this conduit.

        This call makes no changes to the Pulp server. At the end of this call,
        the unit's id field will *not* be populated.

        The unit_key and metadata will be merged as they are saved in Pulp to
        form the full representation of the unit. If values are specified in
        both dictionaries, the unit_key value takes precedence.

        If the importer wants to save the bits for the unit, the relative_path
        value should be used to indicate a unique -- with respect to the type
        of unit -- relative path where it will be saved. Pulp will convert this
        into an absolute path on disk where the unit should actually be saved.
        The absolute path is stored in the returned unit object.

        @param type_id: must correspond to a type definition in Pulp
        @type  type_id: str

        @param unit_key: dictionary of whatever fields are necessary to uniquely
                         identify this unit from others of the same type
        @type  unit_key: dict

        @param metadata: dictionary of key-value pairs to describe the unit
        @type  metadata: dict

        @param relative_path: see above; may be None
        @type  relative_path: str, None

        @return: object representation of the unit, populated by Pulp with both
                 provided and derived values
        @rtype:  pulp.plugins.model.Unit
        """

        try:
            # Generate the storage location
            if relative_path is not None:
                content_query_manager = manager_factory.content_query_manager()
                path = content_query_manager.request_content_unit_file_path(type_id, relative_path)
            else:
                path = None
            u = Unit(type_id, unit_key, metadata, path)
            return u
        except Exception, e:
            msg = _('Exception from server requesting unit filename for relative path [%s]')
            msg = msg % relative_path
            _logger.exception(msg)
            raise ImporterConduitException(e), None, sys.exc_info()[2]
Beispiel #24
0
 def GET(self):
     """
     List the available content types.
     """
     collection = []
     cqm = factory.content_query_manager()
     type_ids = cqm.list_content_types()
     for type_id in type_ids:
         link = serialization.link.child_link_obj(type_id)
         link.update({'content_type': type_id})
         collection.append(link)
     return self.ok(collection)
Beispiel #25
0
    def __init__(self, repo_id, importer_id):
        RepoScratchPadMixin.__init__(self, repo_id, ImporterConduitException)
        ImporterScratchPadMixin.__init__(self, repo_id, importer_id)
        AddUnitMixin.__init__(self, repo_id, importer_id)
        SingleRepoUnitsMixin.__init__(self, repo_id, ImporterConduitException)
        StatusMixin.__init__(self, importer_id, ImporterConduitException)
        SearchUnitsMixin.__init__(self, ImporterConduitException)

        self._association_manager = manager_factory.repo_unit_association_manager(
        )
        self._content_query_manager = manager_factory.content_query_manager()

        self._removed_count = 0
Beispiel #26
0
 def GET(self, type_id, unit_id):
     """
     Return information about a content unit.
     """
     cqm = factory.content_query_manager()
     try:
         unit = cqm.get_content_unit_by_id(type_id, unit_id)
     except MissingResource:
         return self.not_found(_('No content unit resource: %(r)s') %
                               {'r': unit_id})
     resource = serialization.content.content_unit_obj(unit)
     resource.update({'children': serialization.content.content_unit_child_link_objs(resource)})
     return self.ok(resource)
Beispiel #27
0
 def GET(self, type_id):
     """
     Return information about a content type.
     """
     cqm = factory.content_query_manager()
     content_type = cqm.get_content_type(type_id)
     if content_type is None:
         return self.not_found(_('No content type resource: %(r)s') %
                               {'r': type_id})
     resource = serialization.content.content_type_obj(content_type)
     links = {'actions': serialization.link.child_link_obj('actions'),
              'content_units': serialization.link.child_link_obj('units')}
     resource.update(links)
     return self.ok(resource)
Beispiel #28
0
 def GET(self, type_id):
     """
     Return information about a content type.
     """
     cqm = factory.content_query_manager()
     content_type = cqm.get_content_type(type_id)
     if content_type is None:
         return self.not_found(_('No content type resource: %(r)s') %
                               {'r': type_id})
     resource = serialization.content.content_type_obj(content_type)
     links = {'actions': serialization.link.child_link_obj('actions'),
              'content_units': serialization.link.child_link_obj('units')}
     resource.update(links)
     return self.ok(resource)
Beispiel #29
0
    def test_syntactic_sugar_methods(self):
        """
        Tests the syntactic sugar methods for retrieving specific managers.
        """

        # Test
        self.assertTrue(isinstance(factory.repo_manager(), RepoManager))
        self.assertTrue(isinstance(factory.repo_unit_association_manager(), RepoUnitAssociationManager))
        self.assertTrue(isinstance(factory.repo_publish_manager(), RepoPublishManager))
        self.assertTrue(isinstance(factory.repo_query_manager(), RepoQueryManager))
        self.assertTrue(isinstance(factory.repo_sync_manager(), RepoSyncManager))
        self.assertTrue(isinstance(factory.content_manager(), ContentManager))
        self.assertTrue(isinstance(factory.content_query_manager(), ContentQueryManager))
        self.assertTrue(isinstance(factory.content_upload_manager(), ContentUploadManager))
        self.assertTrue(isinstance(factory.consumer_manager(), ConsumerManager))
Beispiel #30
0
    def list_all_orphans(self):
        """
        List all content units that are not associated with a repository.
        @return: list of content units
        @rtype:  list
        """

        # iterate through all types and get the orphaned units for each
        orphaned_units = []
        content_query_manager = manager_factory.content_query_manager()
        content_types = content_query_manager.list_content_types()
        for content_type in content_types:
            orphaned_units_of_type = self.list_orphans_by_type(content_type)
            orphaned_units.extend(orphaned_units_of_type)
        return orphaned_units
Beispiel #31
0
    def save_unit(self, unit):
        """
        Performs two distinct steps on the Pulp server:
        - Creates or updates Pulp's knowledge of the content unit.
        - Associates the unit to the repository being synchronized.

        This call is idempotent. If the unit already exists or the association
        already exists, this call will have no effect.

        A reference to the provided unit is returned from this call. This call
        will populate the unit's id field with the UUID for the unit.

        @param unit: unit object returned from the init_unit call
        @type  unit: L{Unit}

        @return: object reference to the provided unit, its state updated from the call
        @rtype:  L{Unit}
        """
        try:
            content_query_manager = manager_factory.content_query_manager()
            content_manager = manager_factory.content_manager()
            association_manager = manager_factory.repo_unit_association_manager(
            )

            # Save or update the unit
            pulp_unit = common_utils.to_pulp_unit(unit)
            try:
                existing_unit = content_query_manager.get_content_unit_by_keys_dict(
                    unit.type_id, unit.unit_key)
                unit.id = existing_unit['_id']
                content_manager.update_content_unit(unit.type_id, unit.id,
                                                    pulp_unit)
                self._updated_count += 1
            except MissingResource:
                unit.id = content_manager.add_content_unit(
                    unit.type_id, None, pulp_unit)
                self._added_count += 1

            # Associate it with the repo
            association_manager.associate_unit_by_id(
                self.repo_id, unit.type_id, unit.id,
                self.association_owner_type, self.association_owner_id)

            return unit
        except Exception, e:
            _LOG.exception(
                _('Content unit association failed [%s]' % str(unit)))
            raise ImporterConduitException(e), None, sys.exc_info()[2]
Beispiel #32
0
    def get(self, request, type_id):
        """
        Return a response with a serialized list of the content units of the specified type.

        :param request: WSGI request object
        :type  request: django.core.handlers.wsgi.WSGIRequest
        :param type_id: the list of content units will be limited to this type
        :type  type_id: str

        :return: response with a serialized list of dicts, one for each unit of the type.
        :rtype: django.http.HttpResponse
        """
        cqm = factory.content_query_manager()
        all_units = cqm.find_by_criteria(type_id, Criteria())
        all_processed_units = [_process_content_unit(unit, type_id) for unit in all_units]
        return generate_json_response_with_pulp_encoder(all_processed_units)
Beispiel #33
0
    def get(self, request, type_id):
        """
        Return a response with a serialized list of the content units of the specified type.

        :param request: WSGI request object
        :type  request: django.core.handlers.wsgi.WSGIRequest
        :param type_id: the list of content units will be limited to this type
        :type  type_id: str

        :return: response with a serialized list of dicts, one for each unit of the type.
        :rtype: django.http.HttpResponse
        """
        cqm = factory.content_query_manager()
        all_units = cqm.find_by_criteria(type_id, Criteria())
        all_processed_units = [
            _process_content_unit(unit, type_id) for unit in all_units
        ]
        return generate_json_response_with_pulp_encoder(all_processed_units)
Beispiel #34
0
    def get(self, request):
        """
        Returns a response continaing a list of dicts, one for each available content type.

        :param request: WSGI request object
        :type  request: django.core.handlers.wsgi.WSGIRequest

        :return: response containing a serialized list dicts, one for each content type
        :rtype : django.http.HttpResponse
        """
        collection = []
        cqm = factory.content_query_manager()
        type_ids = cqm.list_content_types()
        for type_id in type_ids:
            link = {'_href': reverse('content_type_resource', kwargs={'type_id': type_id})}
            link.update({'content_type': type_id})
            collection.append(link)
        return generate_json_response(collection)
Beispiel #35
0
    def GET(self, type_id, unit_id):
        """
        Return information about a content unit.

        :param type_id: The Unit's type id.
        :type  type_id: basestring
        :param unit_id: The id of the unit that you wish to set the pulp_user_metadata field on
        :type  unit_id: basestring
        """
        cqm = factory.content_query_manager()
        try:
            unit = cqm.get_content_unit_by_id(type_id, unit_id)
        except MissingResource:
            return self.not_found(_('No content unit resource: %(r)s') % {'r': unit_id})

        resource = serialization.content.content_unit_obj(
            unit[constants.PULP_USER_METADATA_FIELDNAME])

        return self.ok(resource)
Beispiel #36
0
    def PUT(self, type_id, unit_id):
        """
        Set the pulp_user_metadata field on the existing unit.

        :param type_id: The Unit's type id.
        :type  type_id: basestring
        :param unit_id: The id of the unit that you wish to set the pulp_user_metadata field on
        :type  unit_id: basestring
        """
        cqm = factory.content_query_manager()
        try:
            cqm.get_content_unit_by_id(type_id, unit_id)
        except MissingResource:
            return self.not_found(_('No content unit resource: %(r)s') % {'r': unit_id})

        cm = factory.content_manager()
        delta = {constants.PULP_USER_METADATA_FIELDNAME: self.params()}
        cm.update_content_unit(type_id, unit_id, delta)
        return self.ok(None)
Beispiel #37
0
    def save_unit(self, unit):
        """
        Performs two distinct steps on the Pulp server:
        - Creates or updates Pulp's knowledge of the content unit.
        - Associates the unit to the repository being synchronized.

        If a unit with the provided unit key already exists, it is updated with
        the attributes on the passed-in unit.

        A reference to the provided unit is returned from this call. This call
        will populate the unit's id field with the UUID for the unit.

        @param unit: unit object returned from the init_unit call
        @type  unit: L{Unit}

        @return: object reference to the provided unit, its state updated from the call
        @rtype:  L{Unit}
        """
        try:
            content_query_manager = manager_factory.content_query_manager()
            content_manager = manager_factory.content_manager()
            association_manager = manager_factory.repo_unit_association_manager()

            # Save or update the unit
            pulp_unit = common_utils.to_pulp_unit(unit)
            try:
                existing_unit = content_query_manager.get_content_unit_by_keys_dict(unit.type_id, unit.unit_key)
                unit.id = existing_unit['_id']
                content_manager.update_content_unit(unit.type_id, unit.id, pulp_unit)
                self._updated_count += 1
            except MissingResource:
                unit.id = content_manager.add_content_unit(unit.type_id, None, pulp_unit)
                self._added_count += 1

            # Associate it with the repo
            association_manager.associate_unit_by_id(self.repo_id, unit.type_id, unit.id, self.association_owner_type, self.association_owner_id)

            return unit
        except Exception, e:
            _LOG.exception(_('Content unit association failed [%s]' % str(unit)))
            raise ImporterConduitException(e), None, sys.exc_info()[2]
Beispiel #38
0
    def __init__(self, importer_type, unit_type, unit_key_fields, working_dir):
        """
        :param importer_type:   unique identifier for the type of importer
        :type  importer_type:   basestring
        :param unit_type:       unique identifier for the unit type in use
        :type  unit_type:       basestring
        :param unit_key_fields: a list of field names in the unit type's unit key.
        :type  unit_key_fields: list
        :param working_dir:     full path to a working directory
        :type  working_dir:     basestring
        """
        super(GetLocalUnitsStep, self).__init__(step_type=reporting_constants.SYNC_STEP_GET_LOCAL,
                                                plugin_type=importer_type,
                                                working_dir=working_dir)
        self.description = _('Copying units already in pulp')

        self.unit_type = unit_type
        self.unit_key_fields = unit_key_fields
        self.content_query_manager = manager_factory.content_query_manager()
        # list of unit keys
        self.units_to_download = []
Beispiel #39
0
    def __init__(self, importer_type, unit_type, unit_key_fields, working_dir):
        """
        :param importer_type:   unique identifier for the type of importer
        :type  importer_type:   basestring
        :param unit_type:       unique identifier for the unit type in use
        :type  unit_type:       basestring
        :param unit_key_fields: a list of field names in the unit type's unit key.
        :type  unit_key_fields: list
        :param working_dir:     full path to a working directory
        :type  working_dir:     basestring
        """
        super(GetLocalUnitsStep, self).__init__(step_type=reporting_constants.SYNC_STEP_GET_LOCAL,
                                                plugin_type=importer_type,
                                                working_dir=working_dir)
        self.description = _('Copying units already in pulp')

        self.unit_type = unit_type
        self.unit_key_fields = unit_key_fields
        self.content_query_manager = manager_factory.content_query_manager()
        # list of unit keys
        self.units_to_download = []
Beispiel #40
0
    def get(self, request):
        """
        Returns a response continaing a list of dicts, one for each available content type.

        :param request: WSGI request object
        :type  request: django.core.handlers.wsgi.WSGIRequest

        :return: response containing a serialized list dicts, one for each content type
        :rtype : django.http.HttpResponse
        """
        collection = []
        cqm = factory.content_query_manager()
        type_ids = cqm.list_content_types()
        for type_id in type_ids:
            link = {
                '_href':
                reverse('content_type_resource', kwargs={'type_id': type_id})
            }
            link.update({'content_type': type_id})
            collection.append(link)
        return generate_json_response(collection)
Beispiel #41
0
    def get(self, request, type_id, unit_id):
        """
        Return user metadata for a content unit.

        :param type_id: The Unit's type id.
        :type  type_id: basestring
        :param unit_id: The id of the unit that you wish to set the pulp_user_metadata field on
        :type  unit_id: basestring

        :return: response containing pulp user metadata field
        :rtype: django.http.HttpResponse or HttpResponseNotFound
        """
        cqm = factory.content_query_manager()
        try:
            unit = cqm.get_content_unit_by_id(type_id, unit_id)
        except MissingResource:
            msg = _("No content unit resource: %(r)s") % {"r": unit_id}
            return generate_json_response(msg, HttpResponseNotFound)

        resource = serial_content.content_unit_obj(unit[constants.PULP_USER_METADATA_FIELDNAME])
        return generate_json_response(resource)
Beispiel #42
0
    def _proxy_query_method(self, criteria):
        """
        Normally the constructor passes a manager's query method to the
        super-class constructor. Since our manager's query method takes an extra
        parameter to tell it what content type to look in, we have this proxy
        query method that will make the correct call at the time.

        Also, at the time of instantiation, we don't know what the content
        type_id will be, so each request handler method will set self._type_id
        to the correct value, and this method will use it at the time of being
        called.

        This sounds like it's asking for a race condition, I know, but web.py
        instantiates a new controller for each and every request, so that isn't
        a concern.

        @param criteria:    Criteria representing a search
        @type  criteria:    models.db.criteria.Criteria

        @return:    same as PulpCollection.query
        """
        return factory.content_query_manager().find_by_criteria(self._type_id, criteria)
Beispiel #43
0
    def find_unit_by_unit_key(self, type_id, unit_key):
        """
        Finds a unit based on its unit key. If more than one unit comes back,
        an exception will be raised.

        @param type_id: indicates the type of units being retrieved
        @type  type_id: str
        @param unit_key: the unit key for the unit
        @type  unit_key: dict

        @return: a single unit
        @rtype:  L{Unit}
        """
        content_query_manager = manager_factory.content_query_manager()
        try:
            # this call returns a unit or raises MissingResource
            existing_unit = content_query_manager.get_content_unit_by_keys_dict(type_id, unit_key)
            unit_key_fields = units_controller.get_unit_key_fields_for_type(type_id)
            plugin_unit = common_utils.to_plugin_unit(existing_unit, type_id, unit_key_fields)
            return plugin_unit
        except pulp_exceptions.MissingResource:
            return None
Beispiel #44
0
    def get(self, request, type_id, unit_id):
        """
        Return user metadata for a content unit.

        :param type_id: The Unit's type id.
        :type  type_id: basestring
        :param unit_id: The id of the unit that you wish to set the pulp_user_metadata field on
        :type  unit_id: basestring

        :return: response containing pulp user metadata field
        :rtype: django.http.HttpResponse or HttpResponseNotFound
        """
        cqm = factory.content_query_manager()
        try:
            unit = cqm.get_content_unit_by_id(type_id, unit_id)
        except MissingResource:
            msg = _('No content unit resource: %(r)s') % {'r': unit_id}
            return generate_json_response(msg, HttpResponseNotFound)

        resource = serial_content.content_unit_obj(
            unit[constants.PULP_USER_METADATA_FIELDNAME])
        return generate_json_response(resource)
Beispiel #45
0
    def get(self, request, type_id):
        """
        Return a response with a serialized list of the content units of the specified type.

        :param request: WSGI request object
        :type  request: django.core.handlers.wsgi.WSGIRequest
        :param type_id: the list of content units will be limited to this type
        :type  type_id: str

        :return: response with a serialized list of dicts, one for each unit of the type.
        :rtype: django.http.HttpResponse
        """
        cqm = factory.content_query_manager()
        all_units = cqm.find_by_criteria(type_id, Criteria())
        all_processed_units = []
        for unit in all_units:
            unit = serialization.content.content_unit_obj(unit)
            unit.update({'_href': '/'.join([request.get_full_path().rstrip('/'),
                                            unit['_id'], ''])})
            unit.update({'children': serialization.content.content_unit_child_link_objs(unit)})
            all_processed_units.append(unit)

        return generate_json_response_with_pulp_encoder(all_processed_units)
Beispiel #46
0
    def _update_unit(self, unit, pulp_unit):
        """
        Update a unit. If it is not found, add it.

        :param unit:        the unit to be updated
        :type  unit:        pulp.plugins.model.Unit
        :param pulp_unit:   the unit to be updated, as a dict
        :type  pulp_unit:   dict

        :return:    id of the updated unit
        :rtype:     basestring
        """
        content_query_manager = manager_factory.content_query_manager()
        content_manager = manager_factory.content_manager()
        try:
            existing_unit = content_query_manager.get_content_unit_by_keys_dict(unit.type_id, unit.unit_key)
            unit_id = existing_unit['_id']
            content_manager.update_content_unit(unit.type_id, unit_id, pulp_unit)
            self._updated_count += 1
            return unit_id
        except MissingResource:
            logger.debug(_('cannot update unit; does not exist. adding instead.'))
            return self._add_unit(unit, pulp_unit)
Beispiel #47
0
    def _proxy_query_method(self, criteria):
        """
        Normally the constructor passes a manager's query method to the
        super-class constructor. Since our manager's query method takes an extra
        parameter to tell it what content type to look in, we have this proxy
        query method that will make the correct call at the time.

        Also, at the time of instantiation, we don't know what the content
        type_id will be, so each request handler method will set self._type_id
        to the correct value, and this method will use it at the time of being
        called.

        This sounds like it's asking for a race condition, I know, but web.py
        instantiates a new controller for each and every request, so that isn't
        a concern.

        @param criteria:    Criteria representing a search
        @type  criteria:    models.db.criteria.Criteria

        @return:    same as PulpCollection.query
        """
        return factory.content_query_manager().find_by_criteria(
            self._type_id, criteria)
Beispiel #48
0
    def get(self, request, type_id, unit_id):
        """
        Return a response containing information about the requested content unit.

        :param request: WSGI request object
        :type  request: django.core.handlers.wsgi.WSGIRequest
        :param type_id: type of content contained in the repo
        :type  type_id: str
        :param unit_id: unique id of a unit
        :type  unit_id: str

        :return: response containing a dict with data about requested unit
        :rtype : django.http.HttpResponse
        """
        cqm = factory.content_query_manager()
        try:
            unit = cqm.get_content_unit_by_id(type_id, unit_id)
        except MissingResource:
            msg = _("No content unit resource: %(r)s") % {"r": unit_id}
            return generate_json_response(msg, response_class=HttpResponseNotFound)

        resource = serial_content.content_unit_obj(unit)
        resource.update({"children": serial_content.content_unit_child_link_objs(resource)})
        return generate_json_response_with_pulp_encoder(resource)
Beispiel #49
0
    def put(self, request, type_id, unit_id):
        """
        Set the pulp_user_metadata field on a content unit.

        :param type_id: The Unit's type id.
        :type  type_id: basestring
        :param unit_id: The id of the unit that you wish to set the pulp_user_metadata field on
        :type  unit_id: basestring

        :return: response containing pulp user metadata_field
        :rtype: django.http.HttpResponse or HttpResponseNotFound
        """
        params = request.body_as_json
        cqm = factory.content_query_manager()
        try:
            cqm.get_content_unit_by_id(type_id, unit_id)
        except MissingResource:
            msg = _("No content unit resource: %(r)s") % {"r": unit_id}
            return generate_json_response(msg, HttpResponseNotFound)

        cm = factory.content_manager()
        delta = {constants.PULP_USER_METADATA_FIELDNAME: params}
        cm.update_content_unit(type_id, unit_id, delta)
        return generate_json_response(None)
Beispiel #50
0
    def put(self, request, type_id, unit_id):
        """
        Set the pulp_user_metadata field on a content unit.

        :param type_id: The Unit's type id.
        :type  type_id: basestring
        :param unit_id: The id of the unit that you wish to set the pulp_user_metadata field on
        :type  unit_id: basestring

        :return: response containing pulp user metadata_field
        :rtype: django.http.HttpResponse or HttpResponseNotFound
        """
        params = request.body_as_json
        cqm = factory.content_query_manager()
        try:
            cqm.get_content_unit_by_id(type_id, unit_id)
        except MissingResource:
            msg = _('No content unit resource: %(r)s') % {'r': unit_id}
            return generate_json_response(msg, HttpResponseNotFound)

        cm = factory.content_manager()
        delta = {constants.PULP_USER_METADATA_FIELDNAME: params}
        cm.update_content_unit(type_id, unit_id, delta)
        return generate_json_response(None)
Beispiel #51
0
    def find_applicable_units(self, consumer_criteria=None, repo_criteria=None, unit_criteria=None, override_config=None):
        """
        Determine and report which of the content units specified by the unit_criteria
        are applicable to consumers specified by the consumer_criteria
        with repos specified by repo_criteria. If consumer_criteria is None,
        all consumers registered to the Pulp server are checked for applicability.
        If repo_criteria is None, all repos bound to the consumer are taken
        into consideration. If unit_criteria contains an empty list for a specific type,
        all units with specific type in the repos bound to the consumer
        are taken into consideration. 

        :param consumer_criteria: The consumer selection criteria.
        :type consumer_criteria: dict

        :param repo_criteria: The repo selection criteria.
        :type repo_criteria: dict

        :param unit_criteria: A dictionary of type_id : unit selection criteria
        :type units: dict
                {<type_id1> : <unit_criteria_for_type_id1>,
                 <type_id2> : <unit_criteria_for_type_id2>}
      
        :param override_config: Additional configuration options to be accepted from user
        :type override_config: dict

        :return: applicability reports dictionary keyed by content type id
        :rtype: dict
        """
        result = {}
        conduit = ProfilerConduit()
        consumer_query_manager = managers.consumer_query_manager()
        bind_manager = managers.consumer_bind_manager()

        # Process Repo Criteria
        if repo_criteria:
            # Get repo ids satisfied by specified repo criteria
            repo_query_manager = managers.repo_query_manager()
            repo_criteria_ids = [r['id'] for r in repo_query_manager.find_by_criteria(repo_criteria)]
            # if repo_criteria is specified and there are no repos satisfying the criteria, return empty result
            if not repo_criteria_ids:
                return result
        else:
            repo_criteria_ids = None

        # Process Consumer Criteria
        if consumer_criteria:
            # Get consumer ids satisfied by specified consumer criteria
            consumer_ids = [c['id'] for c in consumer_query_manager.find_by_criteria(consumer_criteria)]
        else:
            if repo_criteria_ids:
                # If repo_criteria is specified, get all the consumers bound to the repos
                # satisfied by repo_criteria
                bind_criteria = Criteria(filters={"repo_id": {"$in": repo_criteria_ids}})
                consumer_ids = [b['consumer_id'] for b in bind_manager.find_by_criteria(bind_criteria)]
                # Remove duplicate consumer ids
                consumer_ids = list(set(consumer_ids))
            else:
                # Get all consumer ids registered to the Pulp server
                consumer_ids = [c['id'] for c in consumer_query_manager.find_all()]
        # if there are no relevant consumers, return empty result
        if not consumer_ids:
            return result
        else:
            # Based on the consumers, get all the repos bound to the consumers in consideration
            # and find intersection of repo_criteria_ids and consumer_repo_ids
            bind_criteria = Criteria(filters={"consumer_id": {"$in": consumer_ids}})
            consumer_repo_ids = [b['repo_id'] for b in bind_manager.find_by_criteria(bind_criteria)]
            if not repo_criteria_ids:
                repo_criteria_ids = list(set(consumer_repo_ids))
            else:
                repo_criteria_ids = list(set(consumer_repo_ids) & set(repo_criteria_ids))
            if not repo_criteria_ids:
                return result

        # Process Unit Criteria
        if unit_criteria:
            # If unit_criteria is specified, get unit ids satisfied by the criteria for each content type
            # and save them in a dictionary keyed by the content type
            unit_ids_by_type = {}
            content_query_manager = managers.content_query_manager()
            for type_id, criteria in unit_criteria.items():
                if criteria:
                    criteria_ids = [u['_id'] for u in content_query_manager.find_by_criteria(type_id, criteria)]
                    # If there are no units satisfied by the criteria, skip adding it to the dictionary
                    if criteria_ids:
                        unit_ids_by_type[type_id] = criteria_ids
                else:
                    # If criteria for a content type id is None or empty dictionary, add it to the dictionary
                    # with empty list as a value. This will be interpreted as all units of that specific type
                    unit_ids_by_type[type_id] = []
        else:
            # If unit_criteria is not specified set unit_ids_by_type to None to differentiate between
            # considering all units vs considering 0 units since no units were found satisfying given criteria
            unit_ids_by_type = None

        # Create a dictionary with consumer profile and repo_ids bound to the consumer keyed by consumer id
        consumer_profile_and_repo_ids = {}
        all_relevant_repo_ids = set()
        for consumer_id in consumer_ids:
            # Find repos bound to the consumer in consideration and find an intersection of bound repos to the
            # repos specified by repo_criteria
            consumer_bound_repo_ids = [b['repo_id'] for b in bind_manager.find_by_consumer(consumer_id)]
            consumer_bound_repo_ids = list(set(consumer_bound_repo_ids))
            # If repo_criteria is not specified, use repos bound to the consumer, else take intersection 
            # of repos specified in the criteria and repos bound to the consumer.
            if repo_criteria_ids is None:
                repo_ids = consumer_bound_repo_ids
            else:
                repo_ids = list(set(consumer_bound_repo_ids) & set(repo_criteria_ids))

            if repo_ids:
                # Save all eligible repo ids to get relevant plugin unit keys when unit_criteria is not specified
                all_relevant_repo_ids = (all_relevant_repo_ids | set(repo_ids))
                consumer_profile_and_repo_ids[consumer_id] = {'repo_ids': repo_ids}
                consumer_profile_and_repo_ids[consumer_id]['profiled_consumer'] = self.__profiled_consumer(consumer_id)

        # Get relevant units if unit_criteria is not specified
        units = self.__populate_units(unit_ids_by_type, list(all_relevant_repo_ids))

        if units:
            # Call respective profiler api according to the unit type to check for applicability
            for typeid, unit_ids in units.items():
                # if unit_ids is None or empty, continue to the next type id
                if not unit_ids:
                    continue
                # Find a profiler for each type id and find units applicable using that profiler.
                profiler, cfg = self.__profiler(typeid)
                call_config = PluginCallConfiguration(plugin_config=cfg, repo_plugin_config=None,
                                                      override_config=override_config)
                try:
                    report_list = profiler.find_applicable_units(consumer_profile_and_repo_ids, typeid, unit_ids, call_config, conduit)
                except PulpExecutionException:
                    report_list = None

                if report_list is None:
                    _LOG.warn("Profiler for unit type [%s] is not returning applicability reports" % typeid)
                else:
                    result[typeid] = report_list

        return result
Beispiel #52
0
    def find_applicable_units(self,
                              consumer_criteria=None,
                              repo_criteria=None,
                              unit_criteria=None,
                              override_config=None):
        """
        Determine and report which of the content units specified by the unit_criteria
        are applicable to consumers specified by the consumer_criteria
        with repos specified by repo_criteria. If consumer_criteria is None,
        all consumers registered to the Pulp server are checked for applicability.
        If repo_criteria is None, all repos bound to the consumer are taken
        into consideration. If unit_criteria contains an empty list for a specific type,
        all units with specific type in the repos bound to the consumer
        are taken into consideration. 

        :param consumer_criteria: The consumer selection criteria.
        :type consumer_criteria: dict

        :param repo_criteria: The repo selection criteria.
        :type repo_criteria: dict

        :param unit_criteria: A dictionary of type_id : unit selection criteria
        :type units: dict
                {<type_id1> : <unit_criteria_for_type_id1>,
                 <type_id2> : <unit_criteria_for_type_id2>}
      
        :param override_config: Additional configuration options to be accepted from user
        :type override_config: dict

        :return: applicability reports dictionary keyed by content type id
        :rtype: dict
        """
        result = {}
        conduit = ProfilerConduit()
        consumer_query_manager = managers.consumer_query_manager()
        bind_manager = managers.consumer_bind_manager()

        # Process Repo Criteria
        if repo_criteria:
            # Get repo ids satisfied by specified repo criteria
            repo_query_manager = managers.repo_query_manager()
            repo_criteria_ids = [
                r['id']
                for r in repo_query_manager.find_by_criteria(repo_criteria)
            ]
            # if repo_criteria is specified and there are no repos satisfying the criteria, return empty result
            if not repo_criteria_ids:
                return result
        else:
            repo_criteria_ids = None

        # Process Consumer Criteria
        if consumer_criteria:
            # Get consumer ids satisfied by specified consumer criteria
            consumer_ids = [
                c['id'] for c in consumer_query_manager.find_by_criteria(
                    consumer_criteria)
            ]
        else:
            if repo_criteria_ids:
                # If repo_criteria is specified, get all the consumers bound to the repos
                # satisfied by repo_criteria
                bind_criteria = Criteria(
                    filters={"repo_id": {
                        "$in": repo_criteria_ids
                    }})
                consumer_ids = [
                    b['consumer_id']
                    for b in bind_manager.find_by_criteria(bind_criteria)
                ]
                # Remove duplicate consumer ids
                consumer_ids = list(set(consumer_ids))
            else:
                # Get all consumer ids registered to the Pulp server
                consumer_ids = [
                    c['id'] for c in consumer_query_manager.find_all()
                ]
        # if there are no relevant consumers, return empty result
        if not consumer_ids:
            return result
        else:
            # Based on the consumers, get all the repos bound to the consumers in consideration
            # and find intersection of repo_criteria_ids and consumer_repo_ids
            bind_criteria = Criteria(
                filters={"consumer_id": {
                    "$in": consumer_ids
                }})
            consumer_repo_ids = [
                b['repo_id']
                for b in bind_manager.find_by_criteria(bind_criteria)
            ]
            if not repo_criteria_ids:
                repo_criteria_ids = list(set(consumer_repo_ids))
            else:
                repo_criteria_ids = list(
                    set(consumer_repo_ids) & set(repo_criteria_ids))
            if not repo_criteria_ids:
                return result

        # Process Unit Criteria
        if unit_criteria:
            # If unit_criteria is specified, get unit ids satisfied by the criteria for each content type
            # and save them in a dictionary keyed by the content type
            unit_ids_by_type = {}
            content_query_manager = managers.content_query_manager()
            for type_id, criteria in unit_criteria.items():
                if criteria:
                    criteria_ids = [
                        u['_id']
                        for u in content_query_manager.find_by_criteria(
                            type_id, criteria)
                    ]
                    # If there are no units satisfied by the criteria, skip adding it to the dictionary
                    if criteria_ids:
                        unit_ids_by_type[type_id] = criteria_ids
                else:
                    # If criteria for a content type id is None or empty dictionary, add it to the dictionary
                    # with empty list as a value. This will be interpreted as all units of that specific type
                    unit_ids_by_type[type_id] = []
        else:
            # If unit_criteria is not specified set unit_ids_by_type to None to differentiate between
            # considering all units vs considering 0 units since no units were found satisfying given criteria
            unit_ids_by_type = None

        # Create a dictionary with consumer profile and repo_ids bound to the consumer keyed by consumer id
        consumer_profile_and_repo_ids = {}
        all_relevant_repo_ids = set()
        for consumer_id in consumer_ids:
            # Find repos bound to the consumer in consideration and find an intersection of bound repos to the
            # repos specified by repo_criteria
            consumer_bound_repo_ids = [
                b['repo_id']
                for b in bind_manager.find_by_consumer(consumer_id)
            ]
            consumer_bound_repo_ids = list(set(consumer_bound_repo_ids))
            # If repo_criteria is not specified, use repos bound to the consumer, else take intersection
            # of repos specified in the criteria and repos bound to the consumer.
            if repo_criteria_ids is None:
                repo_ids = consumer_bound_repo_ids
            else:
                repo_ids = list(
                    set(consumer_bound_repo_ids) & set(repo_criteria_ids))

            if repo_ids:
                # Save all eligible repo ids to get relevant plugin unit keys when unit_criteria is not specified
                all_relevant_repo_ids = (all_relevant_repo_ids | set(repo_ids))
                consumer_profile_and_repo_ids[consumer_id] = {
                    'repo_ids': repo_ids
                }
                consumer_profile_and_repo_ids[consumer_id][
                    'profiled_consumer'] = self.__profiled_consumer(
                        consumer_id)

        # Get relevant units if unit_criteria is not specified
        units = self.__populate_units(unit_ids_by_type,
                                      list(all_relevant_repo_ids))

        if units:
            # Call respective profiler api according to the unit type to check for applicability
            for typeid, unit_ids in units.items():
                # if unit_ids is None or empty, continue to the next type id
                if not unit_ids:
                    continue
                # Find a profiler for each type id and find units applicable using that profiler.
                profiler, cfg = self.__profiler(typeid)
                call_config = PluginCallConfiguration(
                    plugin_config=cfg,
                    repo_plugin_config=None,
                    override_config=override_config)
                try:
                    report_list = profiler.find_applicable_units(
                        consumer_profile_and_repo_ids, typeid, unit_ids,
                        call_config, conduit)
                except PulpExecutionException:
                    report_list = None

                if report_list is None:
                    _LOG.warn(
                        "Profiler for unit type [%s] is not returning applicability reports"
                        % typeid)
                else:
                    result[typeid] = report_list

        return result