Example #1
0
 def test_get_bindings(self):
     # Setup
     self.populate()
     # Test
     conduit = ProfilerConduit()
     binds = conduit.get_bindings(self.CONSUMER_ID)
     # Verify
     self.assertEquals(1, len(binds))
     self.assertTrue(binds[0], self.REPO_ID)
Example #2
0
 def test_get_units(self):
     # Setup
     self.populate()
     # Test
     conduit = ProfilerConduit()
     criteria = UnitAssociationCriteria(type_ids=[self.TYPE_1_DEF.id])
     units = conduit.get_units(self.REPO_ID, criteria)
     # Verify
     self.assertEquals(len(units), 9)
Example #3
0
 def test_get_units(self):
     # Setup
     self.populate()
     # Test
     conduit = ProfilerConduit()
     criteria = UnitAssociationCriteria(type_ids=[self.TYPE_1_DEF.id])
     units = conduit.get_units(self.REPO_ID, criteria)
     # Verify
     self.assertEquals(len(units), 9)
Example #4
0
 def test_get_bindings(self):
     # Setup
     self.populate()
     # Test
     conduit = ProfilerConduit()
     binds = conduit.get_bindings(self.CONSUMER_ID)
     # Verify
     self.assertEquals(1, len(binds))
     self.assertTrue(binds[0], self.REPO_ID)
Example #5
0
    def test_get_repo_units_additional_field(self):
        # Setup
        self.populate(additional_key='extra_field')
        # Test
        conduit = ProfilerConduit()
        units = conduit.get_repo_units(self.REPO_ID, content_type_id=self.TYPE_1_DEF.id, 
                                       additional_unit_fields=['extra_field'])

        # Verify that all the units in the repo with given type are returned along with unit_key and extra field
        self.assertEquals(len(units), 9)
        for u in units:
            self.assertTrue('key-1' in u.unit_key)
            self.assertTrue('extra_field' in u.metadata)
Example #6
0
    def test_get_repo_units_additional_field(self):
        # Setup
        self.populate(additional_key='extra_field')
        # Test
        conduit = ProfilerConduit()
        units = conduit.get_repo_units(self.REPO_ID,
                                       content_type_id=self.TYPE_1_DEF.id,
                                       additional_unit_fields=['extra_field'])

        # Verify that all the units in the repo with given type are returned along with unit_key and extra field
        self.assertEquals(len(units), 9)
        for u in units:
            self.assertTrue('key-1' in u.unit_key)
            self.assertTrue('extra_field' in u.metadata)
 def setUp(self):
     self.profiler = wholerepo.WholeRepoProfiler()
     self.consumer = Consumer('consumer1', {})
     self.units = [{
         'type_id': constants.TYPE_PUPPET_MODULE,
         'unit_key': {
             'name': 'gcc',
             'author': 'puppetlabs'
         }
     }, {
         'type_id': constants.TYPE_PUPPET_MODULE,
         'unit_key': {
             'name': 'stdlib',
             'author': 'puppetlabs',
             'version': '3.1.1'
         }
     }, {
         'type_id': constants.TYPE_PUPPET_MODULE,
         'unit_key': {
             'name': 'stdlib',
             'author': 'puppetlabs',
             'version': '3.2.0'
         }
     }]
     self.conduit = mock.MagicMock(spec=ProfilerConduit())
     self.conduit.get_units.return_value = [
         AssociatedUnit(constants.TYPE_PUPPET_MODULE, unit['unit_key'], {},
                        '', '', '', '', '') for unit in self.units
     ]
Example #8
0
    def test_get_repo_units(self):
        # Setup
        self.populate()
        # Test
        conduit = ProfilerConduit()
        units1 = conduit.get_repo_units(self.REPO_ID, content_type_id=self.TYPE_1_DEF.id, additional_unit_fields=[])
        units2 = conduit.get_repo_units(self.REPO_ID, content_type_id=self.TYPE_2_DEF.id, additional_unit_fields=[])

        # Verify that all the units in the repo with given type are returned along with unit_key
        self.assertEquals(len(units1), 9)
        for u in units1:
            self.assertTrue('key-1' in u.unit_key)
            self.assertFalse('key-2' in u.unit_key)
        self.assertEquals(len(units2), 9)
        for u in units2:
            self.assertTrue('key-2' in u.unit_key)
            self.assertFalse('key-1' in u.unit_key)
Example #9
0
    def test_get_repo_units(self):
        # Setup
        self.populate()
        # Test
        conduit = ProfilerConduit()
        units1 = conduit.get_repo_units(self.REPO_ID,
                                        content_type_id=self.TYPE_1_DEF.id,
                                        additional_unit_fields=[])
        units2 = conduit.get_repo_units(self.REPO_ID,
                                        content_type_id=self.TYPE_2_DEF.id,
                                        additional_unit_fields=[])

        # Verify that all the units in the repo with given type are returned along with unit_key
        self.assertEquals(len(units1), 9)
        for u in units1:
            self.assertTrue('key-1' in u.unit_key)
            self.assertFalse('key-2' in u.unit_key)
        self.assertEquals(len(units2), 9)
        for u in units2:
            self.assertTrue('key-2' in u.unit_key)
            self.assertFalse('key-1' in u.unit_key)
Example #10
0
    def uninstall_content(consumer_id, units, options):
        """
        Uninstall content units on a consumer.
        :param consumer_id: The consumer ID.
        :type consumer_id: str
        :param units: A list of content units to be uninstalled.
        :type units: list of:
            { type_id:<str>, type_id:<dict> }
        :param options: Uninstall options; based on unit type.
        :type options: dict
        :return: A task ID that may be used to track the agent request.
        :rtype: dict
        """
        # track agent operations using a pseudo task
        task_id = str(uuid4())
        task_tags = [
            tags.resource_tag(tags.RESOURCE_CONSUMER_TYPE, consumer_id),
            tags.action_tag(tags.ACTION_AGENT_UNIT_UNINSTALL)
        ]
        task = TaskStatus(task_id, 'agent', tags=task_tags).save()

        # agent request
        manager = managers.consumer_manager()
        consumer = manager.get_consumer(consumer_id)
        conduit = ProfilerConduit()
        collated = Units(units)
        for typeid, units in collated.items():
            pc = AgentManager._profiled_consumer(consumer_id)
            profiler, cfg = AgentManager._profiler(typeid)
            units = AgentManager._invoke_plugin(
                profiler.uninstall_units,
                pc,
                units,
                options,
                cfg,
                conduit)
            collated[typeid] = units
        units = collated.join()
        context = Context(consumer, task_id=task_id, consumer_id=consumer_id)
        agent = PulpAgent()
        agent.content.uninstall(context, units, options)
        return task
Example #11
0
 def uninstall_content(self, consumer_id, units, options):
     """
     Uninstall content units on a consumer.
     :param consumer_id: The consumer ID.
     :type consumer_id: str
     :param units: A list of content units to be uninstalled.
     :type units: list of:
         { type_id:<str>, type_id:<dict> }
     :param options: Uninstall options; based on unit type.
     :type options: dict
     """
     manager = managers.consumer_manager()
     consumer = manager.get_consumer(consumer_id)
     conduit = ProfilerConduit()
     collated = Units(units)
     for typeid, units in collated.items():
         pc = self.__profiled_consumer(consumer_id)
         profiler, cfg = self.__profiler(typeid)
         units = self.__invoke_plugin(profiler.uninstall_units, pc, units,
                                      options, cfg, conduit)
         collated[typeid] = units
     units = collated.join()
     agent = PulpAgent(consumer)
     agent.content.uninstall(units, options)
Example #12
0
    def regenerate_applicability(profile_hash,
                                 content_type,
                                 profile_id,
                                 bound_repo_id,
                                 existing_applicability=None):
        """
        Regenerate and save applicability data for given profile and bound repo id.
        If existing_applicability is not None, replace it with the new applicability data.

        :param profile_hash: hash of the unit profile
        :type profile_hash: basestring

        :param content_type: profile (unit) type ID
        :type content_type: str

        :param profile_id: unique id of the unit profile
        :type profile_id: str

        :param bound_repo_id: repo id to be used to calculate applicability
                              against the given unit profile
        :type bound_repo_id: str

        :param existing_applicability: existing RepoProfileApplicability object to be replaced
        :type existing_applicability: pulp.server.db.model.consumer.RepoProfileApplicability
        """
        profiler_conduit = ProfilerConduit()
        # Get the profiler for content_type of given unit_profile
        profiler, profiler_cfg = ApplicabilityRegenerationManager._profiler(
            content_type)

        # Check if the profiler supports applicability, else return
        if profiler.calculate_applicable_units == Profiler.calculate_applicable_units:
            # If base class calculate_applicable_units method is called,
            # skip applicability regeneration
            return

        # Find out which content types have unit counts greater than zero in the bound repo
        repo_content_types = ApplicabilityRegenerationManager._get_existing_repo_content_types(
            bound_repo_id)
        # Get the intersection of existing types in the repo and the types that the profiler
        # handles. If the intersection is not empty, regenerate applicability
        if (set(repo_content_types) & set(profiler.metadata()['types'])):
            # Get the actual profile for existing_applicability or lookup using profile_id
            if existing_applicability:
                profile = existing_applicability.profile
            else:
                unit_profile = UnitProfile.get_collection().find_one(
                    {'id': profile_id}, projection=['profile'])
                profile = unit_profile['profile']
            call_config = PluginCallConfiguration(plugin_config=profiler_cfg,
                                                  repo_plugin_config=None)
            try:
                applicability = profiler.calculate_applicable_units(
                    profile, bound_repo_id, call_config, profiler_conduit)
            except NotImplementedError:
                msg = "Profiler for content type [%s] does not support applicability" % content_type
                _logger.debug(msg)
                return

            if existing_applicability:
                # Update existing applicability object
                existing_applicability.applicability = applicability
                existing_applicability.save()
            else:
                # Create a new RepoProfileApplicability object and save it in the db
                RepoProfileApplicability.objects.create(
                    profile_hash, bound_repo_id, unit_profile['profile'],
                    applicability)
Example #13
0
    def find_applicable_units(self,
                              consumer_criteria=None,
                              repo_criteria=None,
                              unit_criteria=None,
                              override_config=None):
        """
        Determine and report which of the content units specified by the unit_criteria
        are applicable to consumers specified by the consumer_criteria
        with repos specified by repo_criteria. If consumer_criteria is None,
        all consumers registered to the Pulp server are checked for applicability.
        If repo_criteria is None, all repos bound to the consumer are taken
        into consideration. If unit_criteria contains an empty list for a specific type,
        all units with specific type in the repos bound to the consumer
        are taken into consideration. 

        :param consumer_criteria: The consumer selection criteria.
        :type consumer_criteria: dict

        :param repo_criteria: The repo selection criteria.
        :type repo_criteria: dict

        :param unit_criteria: A dictionary of type_id : unit selection criteria
        :type units: dict
                {<type_id1> : <unit_criteria_for_type_id1>,
                 <type_id2> : <unit_criteria_for_type_id2>}
      
        :param override_config: Additional configuration options to be accepted from user
        :type override_config: dict

        :return: applicability reports dictionary keyed by content type id
        :rtype: dict
        """
        result = {}
        conduit = ProfilerConduit()
        consumer_query_manager = managers.consumer_query_manager()
        bind_manager = managers.consumer_bind_manager()

        # Process Repo Criteria
        if repo_criteria:
            # Get repo ids satisfied by specified repo criteria
            repo_query_manager = managers.repo_query_manager()
            repo_criteria_ids = [
                r['id']
                for r in repo_query_manager.find_by_criteria(repo_criteria)
            ]
            # if repo_criteria is specified and there are no repos satisfying the criteria, return empty result
            if not repo_criteria_ids:
                return result
        else:
            repo_criteria_ids = None

        # Process Consumer Criteria
        if consumer_criteria:
            # Get consumer ids satisfied by specified consumer criteria
            consumer_ids = [
                c['id'] for c in consumer_query_manager.find_by_criteria(
                    consumer_criteria)
            ]
        else:
            if repo_criteria_ids:
                # If repo_criteria is specified, get all the consumers bound to the repos
                # satisfied by repo_criteria
                bind_criteria = Criteria(
                    filters={"repo_id": {
                        "$in": repo_criteria_ids
                    }})
                consumer_ids = [
                    b['consumer_id']
                    for b in bind_manager.find_by_criteria(bind_criteria)
                ]
                # Remove duplicate consumer ids
                consumer_ids = list(set(consumer_ids))
            else:
                # Get all consumer ids registered to the Pulp server
                consumer_ids = [
                    c['id'] for c in consumer_query_manager.find_all()
                ]
        # if there are no relevant consumers, return empty result
        if not consumer_ids:
            return result
        else:
            # Based on the consumers, get all the repos bound to the consumers in consideration
            # and find intersection of repo_criteria_ids and consumer_repo_ids
            bind_criteria = Criteria(
                filters={"consumer_id": {
                    "$in": consumer_ids
                }})
            consumer_repo_ids = [
                b['repo_id']
                for b in bind_manager.find_by_criteria(bind_criteria)
            ]
            if not repo_criteria_ids:
                repo_criteria_ids = list(set(consumer_repo_ids))
            else:
                repo_criteria_ids = list(
                    set(consumer_repo_ids) & set(repo_criteria_ids))
            if not repo_criteria_ids:
                return result

        # Create a dictionary with consumer profile and repo_ids bound to the consumer keyed by consumer id
        consumer_profile_and_repo_ids = {}
        all_relevant_repo_ids = set()
        for consumer_id in consumer_ids:
            # Find repos bound to the consumer in consideration and find an intersection of bound repos to the
            # repos specified by repo_criteria
            consumer_bound_repo_ids = [
                b['repo_id']
                for b in bind_manager.find_by_consumer(consumer_id)
            ]
            consumer_bound_repo_ids = list(set(consumer_bound_repo_ids))
            # If repo_criteria is not specified, use repos bound to the consumer, else take intersection
            # of repos specified in the criteria and repos bound to the consumer.
            if repo_criteria_ids is None:
                repo_ids = consumer_bound_repo_ids
            else:
                repo_ids = list(
                    set(consumer_bound_repo_ids) & set(repo_criteria_ids))

            if repo_ids:
                # Save all eligible repo ids to get relevant plugin unit keys when unit_criteria is not specified
                all_relevant_repo_ids = (all_relevant_repo_ids | set(repo_ids))
                consumer_profile_and_repo_ids[consumer_id] = {
                    'repo_ids': repo_ids
                }
                consumer_profile_and_repo_ids[consumer_id][
                    'profiled_consumer'] = self.__profiled_consumer(
                        consumer_id)

        if not unit_criteria:
            return result

        # Call respective profiler api according to the unit type to check for applicability
        for unit_type_id, criteria in unit_criteria.items():
            # Find a profiler for each type id and find units applicable using that profiler.
            profiler, cfg = self.__profiler(unit_type_id)
            call_config = PluginCallConfiguration(
                plugin_config=cfg,
                repo_plugin_config=None,
                override_config=override_config)
            try:
                report_list = profiler.find_applicable_units(
                    consumer_profile_and_repo_ids, unit_type_id, criteria,
                    call_config, conduit)
            except PulpExecutionException:
                report_list = None

            if report_list is None:
                _LOG.warn(
                    "Profiler for unit type [%s] is not returning applicability reports"
                    % unit_type_id)
            else:
                result[unit_type_id] = report_list

        return result
Example #14
0
    def regenerate_applicability(all_profiles_hash, profiles, bound_repo_id):
        """
        Regenerate and save applicability data for given set of profiles and bound repo id.

        :param all_profiles_hash: hash of the consumer profiles
        :type  all_profiles_hash: basestring

        :param profiles: profiles data: (profile_hash, content_type, profile_id)
        :type  profiles: list of tuples

        :param bound_repo_id: repo id to be used to calculate applicability
                              against the given unit profile
        :type  bound_repo_id: str
        """
        profiler_conduit = ProfilerConduit()

        # Get the profiler for content_type of given profiles.
        # The assumption is that the same profiler is used for all the content types, so different
        # profilers are not supported at the moment.
        # Take the content type from the first profile.
        content_type = profiles[0][1]
        profiler, profiler_cfg = ApplicabilityRegenerationManager._profiler(
            content_type)

        # Check if the profiler supports applicability, else return
        if profiler.calculate_applicable_units == Profiler.calculate_applicable_units:
            # If base class calculate_applicable_units method is called,
            # skip applicability regeneration
            return

        # Find out which content types have unit counts greater than zero in the bound repo
        repo_content_types = ApplicabilityRegenerationManager._get_existing_repo_content_types(
            bound_repo_id)

        # Get the intersection of existing types in the repo and the types that the profiler
        # handles. If the intersection is not empty, regenerate applicability
        if (set(repo_content_types) & set(profiler.metadata()['types'])):
            profile_ids = [p_id for _, _, p_id in profiles]
            unit_profiles = UnitProfile.get_collection().find(
                {'id': {
                    '$in': profile_ids
                }},
                projection=['profile', 'content_type', 'profile_hash'])
            try:
                profiles = [(p['profile_hash'], p['content_type'],
                             p['profile']) for p in unit_profiles]
            except TypeError:
                # It means that p = None.
                # Consumer can be removed during applicability regeneration,
                # so it is possible that its profile no longer exists. It is harmless.
                return

            call_config = PluginCallConfiguration(plugin_config=profiler_cfg,
                                                  repo_plugin_config=None)
            try:
                applicability = profiler.calculate_applicable_units(
                    profiles, bound_repo_id, call_config, profiler_conduit)
            except NotImplementedError:
                msg = "Profiler for content type [%s] does not support applicability" % content_type
                _logger.debug(msg)
                return

            # Save applicability results on each of the profiles. The results are duplicated.
            # It's a compromise to have applicability data available in any applicability profile
            # record in the DB.
            for profile in profiles:
                profile_hash = profile[0]
                try:
                    # Create a new RepoProfileApplicability object and save it in the db
                    RepoProfileApplicability.objects.create(
                        profile_hash=profile_hash,
                        repo_id=bound_repo_id,
                        # profiles can be large, the one in
                        # repo_profile_applicability collection
                        # is no longer used,
                        # it's a duplicated data
                        # from the consumer_unit_profiles
                        # collection.
                        profile=[],
                        applicability=applicability,
                        all_profiles_hash=all_profiles_hash)
                except DuplicateKeyError:
                    applicability_dict = RepoProfileApplicability.get_collection(
                    ).find_one({
                        'repo_id': bound_repo_id,
                        'all_profiles_hash': all_profiles_hash,
                        'profile_hash': profile_hash
                    })
                    existing_applicability = RepoProfileApplicability(
                        **applicability_dict)
                    existing_applicability.applicability = applicability
                    existing_applicability.save()