def batch_regenerate_applicability(repo_id, existing_applicability_ids): """ Regenerate and save applicability data for a batch of existing applicabilities :param repo_id: Repository id for which applicability is being calculated :type repo_id: str :param existing_applicability_ids: Tuple of Object Ids for applicability profiles :type existing_applicability_ids: tuple of dicts in form of {"_id": ObjectID('mongo-id')} """ id_list = [id['_id'] for id in existing_applicability_ids] existing_applicabilities = RepoProfileApplicability.get_collection().find( {"_id": {"$in": id_list}}) for existing_applicability in existing_applicabilities: # Convert cursor to RepoProfileApplicability object existing_applicability = RepoProfileApplicability(**dict(existing_applicability)) profile_hash = existing_applicability['profile_hash'] unit_profile = UnitProfile.get_collection().find_one({'profile_hash': profile_hash}, fields=['id', 'content_type']) if unit_profile is None: # Unit profiles change whenever packages are installed or removed on consumers, # and it is possible that existing_applicability references a UnitProfile # that no longer exists. This is harmless, as Pulp has a monthly cleanup task # that will identify these dangling references and remove them. continue # Regenerate applicability data for given unit_profile and repo id ApplicabilityRegenerationManager.regenerate_applicability( profile_hash, unit_profile['content_type'], unit_profile['id'], repo_id, existing_applicability)
def create(self, profile_hash, repo_id, profile, applicability, all_profiles_hash): """ Create and return a RepoProfileApplicability object. :param profile_hash: The hash of the profile that is a part of the profile set of a consumer :type profile_hash: basestring :param repo_id: The repo ID that this applicability data is for :type repo_id: basestring :param profile: The entire profile that resulted in the profile_hash :type profile: object :param applicability: A dictionary structure mapping unit type IDs to lists of applicable Unit IDs. :type applicability: dict :param all_profiles_hash: The hash of the set of the profiles that this applicability data is for :type all_profiles_hash: basestring :return: A new RepoProfileApplicability object :rtype: pulp.server.db.model.consumer.RepoProfileApplicability """ applicability = RepoProfileApplicability( profile_hash=profile_hash, repo_id=repo_id, profile=profile, applicability=applicability, all_profiles_hash=all_profiles_hash) applicability.save() return applicability
def filter(self, query_params): """ Get a list of RepoProfileApplicability objects with the given MongoDB query dict. :param query_params: A MongoDB query dictionary that selects RepoProfileApplicability documents :type query_params: dict :return: A list of RepoProfileApplicability objects that match the given query :rtype: list """ collection = RepoProfileApplicability.get_collection() mongo_applicabilities = collection.find(query_params) applicabilities = [RepoProfileApplicability(**dict(applicability)) \ for applicability in mongo_applicabilities] return applicabilities
def regenerate_applicability_for_repos(repo_criteria): """ Regenerate and save applicability data affected by given updated repositories. :param repo_criteria: The repo selection criteria :type repo_criteria: dict """ repo_criteria = Criteria.from_dict(repo_criteria) # Process repo criteria repo_criteria.fields = ['id'] repo_ids = [ r.repo_id for r in model.Repository.objects.find_by_criteria(repo_criteria) ] for repo_id in repo_ids: # Find all existing applicabilities for given repo_id. Setting batch size of 5 ensures # the MongoDB cursor does not time out. See https://pulp.plan.io/issues/998#note-6 for # more details. existing_applicabilities = RepoProfileApplicability.get_collection( ).find({ 'repo_id': repo_id }).batch_size(5) for existing_applicability in existing_applicabilities: existing_applicability = RepoProfileApplicability( **dict(existing_applicability)) profile_hash = existing_applicability['profile_hash'] unit_profile = UnitProfile.get_collection().find_one( {'profile_hash': profile_hash}, projection=['id', 'content_type']) if unit_profile is None: # Unit profiles change whenever packages are installed or removed on consumers, # and it is possible that existing_applicability references a UnitProfile # that no longer exists. This is harmless, as Pulp has a monthly cleanup task # that will identify these dangling references and remove them. continue # Regenerate applicability data for given unit_profile and repo id ApplicabilityRegenerationManager.regenerate_applicability( profile_hash, unit_profile['content_type'], unit_profile['id'], repo_id, existing_applicability)
def regenerate_applicability_for_repos(repo_criteria): """ Regenerate and save applicability data affected by given updated repositories. :param repo_criteria: The repo selection criteria :type repo_criteria: dict """ repo_criteria = Criteria.from_dict(repo_criteria) repo_query_manager = managers.repo_query_manager() # Process repo criteria repo_criteria.fields = ['id'] repo_ids = [ r['id'] for r in repo_query_manager.find_by_criteria(repo_criteria) ] for repo_id in repo_ids: # Find all existing applicabilities for given repo_id existing_applicabilities = RepoProfileApplicability.get_collection( ).find({'repo_id': repo_id}) for existing_applicability in existing_applicabilities: # Convert cursor to RepoProfileApplicability object existing_applicability = RepoProfileApplicability( **dict(existing_applicability)) profile_hash = existing_applicability['profile_hash'] unit_profile = UnitProfile.get_collection().find_one( {'profile_hash': profile_hash}, fields=['id', 'content_type']) if unit_profile is None: # Unit profiles change whenever packages are installed or removed on consumers, # and it is possible that existing_applicability references a UnitProfile # that no longer exists. This is harmless, as Pulp has a monthly cleanup task # that will identify these dangling references and remove them. continue # Regenerate applicability data for given unit_profile and repo id ApplicabilityRegenerationManager.regenerate_applicability( profile_hash, unit_profile['content_type'], unit_profile['id'], repo_id, existing_applicability)
def batch_regenerate_applicability(repo_id, profile_hashes): """ Regenerate and save applicability data for a batch of existing applicabilities :param repo_id: Repository id for which applicability is being calculated :type repo_id: str :param profile_hashes: Tuple of consumer profile hashes for applicability profiles. Don't pass too much of these, all the profile data associated with these hashes is loaded into the memory. :type profile_hashes: tuple of dicts in form of {'profile_hash': str} """ profile_hash_list = [phash['profile_hash'] for phash in profile_hashes] existing_applicabilities = RepoProfileApplicability.get_collection( ).find({ "repo_id": repo_id, "profile_hash": { "$in": profile_hash_list } }) for existing_applicability in list(existing_applicabilities): # Convert cursor to RepoProfileApplicability object existing_applicability = RepoProfileApplicability( **dict(existing_applicability)) profile_hash = existing_applicability['profile_hash'] unit_profile = UnitProfile.get_collection().find_one( {'profile_hash': profile_hash}, projection=['id', 'content_type']) if unit_profile is None: # Unit profiles change whenever packages are installed or removed on consumers, # and it is possible that existing_applicability references a UnitProfile # that no longer exists. This is harmless, as Pulp has a monthly cleanup task # that will identify these dangling references and remove them. continue # Regenerate applicability data for given unit_profile and repo id ApplicabilityRegenerationManager.regenerate_applicability( profile_hash, unit_profile['content_type'], unit_profile['id'], repo_id, existing_applicability)
def regenerate_applicability(profile_hash, content_type, profile_id, bound_repo_id, existing_applicability=None): """ Regenerate and save applicability data for given profile and bound repo id. If existing_applicability is not None, replace it with the new applicability data. :param profile_hash: hash of the unit profile :type profile_hash: basestring :param content_type: profile (unit) type ID :type content_type: str :param profile_id: unique id of the unit profile :type profile_id: str :param bound_repo_id: repo id to be used to calculate applicability against the given unit profile :type bound_repo_id: str :param existing_applicability: existing RepoProfileApplicability object to be replaced :type existing_applicability: pulp.server.db.model.consumer.RepoProfileApplicability """ profiler_conduit = ProfilerConduit() # Get the profiler for content_type of given unit_profile profiler, profiler_cfg = ApplicabilityRegenerationManager._profiler( content_type) # Check if the profiler supports applicability, else return if profiler.calculate_applicable_units == Profiler.calculate_applicable_units: # If base class calculate_applicable_units method is called, # skip applicability regeneration return # Find out which content types have unit counts greater than zero in the bound repo repo_content_types = ApplicabilityRegenerationManager._get_existing_repo_content_types( bound_repo_id) # Get the intersection of existing types in the repo and the types that the profiler # handles. If the intersection is not empty, regenerate applicability if (set(repo_content_types) & set(profiler.metadata()['types'])): # Get the actual profile for existing_applicability or lookup using profile_id if existing_applicability: profile = existing_applicability.profile else: unit_profile = UnitProfile.get_collection().find_one( {'id': profile_id}, projection=['profile']) profile = unit_profile['profile'] call_config = PluginCallConfiguration(plugin_config=profiler_cfg, repo_plugin_config=None) try: applicability = profiler.calculate_applicable_units( profile, bound_repo_id, call_config, profiler_conduit) except NotImplementedError: msg = "Profiler for content type [%s] does not support applicability" % content_type _logger.debug(msg) return try: # Create a new RepoProfileApplicability object and save it in the db RepoProfileApplicability.objects.create( profile_hash, bound_repo_id, profile, applicability) except DuplicateKeyError: # Update existing applicability if not existing_applicability: applicability_dict = RepoProfileApplicability.get_collection( ).find_one({ 'repo_id': bound_repo_id, 'profile_hash': profile_hash }) existing_applicability = RepoProfileApplicability( **applicability_dict) existing_applicability.applicability = applicability existing_applicability.save()
def regenerate_applicability(all_profiles_hash, profiles, bound_repo_id): """ Regenerate and save applicability data for given set of profiles and bound repo id. :param all_profiles_hash: hash of the consumer profiles :type all_profiles_hash: basestring :param profiles: profiles data: (profile_hash, content_type, profile_id) :type profiles: list of tuples :param bound_repo_id: repo id to be used to calculate applicability against the given unit profile :type bound_repo_id: str """ profiler_conduit = ProfilerConduit() # Get the profiler for content_type of given profiles. # The assumption is that the same profiler is used for all the content types, so different # profilers are not supported at the moment. # Take the content type from the first profile. content_type = profiles[0][1] profiler, profiler_cfg = ApplicabilityRegenerationManager._profiler( content_type) # Check if the profiler supports applicability, else return if profiler.calculate_applicable_units == Profiler.calculate_applicable_units: # If base class calculate_applicable_units method is called, # skip applicability regeneration return # Find out which content types have unit counts greater than zero in the bound repo repo_content_types = ApplicabilityRegenerationManager._get_existing_repo_content_types( bound_repo_id) # Get the intersection of existing types in the repo and the types that the profiler # handles. If the intersection is not empty, regenerate applicability if (set(repo_content_types) & set(profiler.metadata()['types'])): profile_ids = [p_id for _, _, p_id in profiles] unit_profiles = UnitProfile.get_collection().find( {'id': { '$in': profile_ids }}, projection=['profile', 'content_type', 'profile_hash']) try: profiles = [(p['profile_hash'], p['content_type'], p['profile']) for p in unit_profiles] except TypeError: # It means that p = None. # Consumer can be removed during applicability regeneration, # so it is possible that its profile no longer exists. It is harmless. return call_config = PluginCallConfiguration(plugin_config=profiler_cfg, repo_plugin_config=None) try: applicability = profiler.calculate_applicable_units( profiles, bound_repo_id, call_config, profiler_conduit) except NotImplementedError: msg = "Profiler for content type [%s] does not support applicability" % content_type _logger.debug(msg) return # Save applicability results on each of the profiles. The results are duplicated. # It's a compromise to have applicability data available in any applicability profile # record in the DB. for profile in profiles: profile_hash = profile[0] try: # Create a new RepoProfileApplicability object and save it in the db RepoProfileApplicability.objects.create( profile_hash=profile_hash, repo_id=bound_repo_id, # profiles can be large, the one in # repo_profile_applicability collection # is no longer used, # it's a duplicated data # from the consumer_unit_profiles # collection. profile=[], applicability=applicability, all_profiles_hash=all_profiles_hash) except DuplicateKeyError: applicability_dict = RepoProfileApplicability.get_collection( ).find_one({ 'repo_id': bound_repo_id, 'all_profiles_hash': all_profiles_hash, 'profile_hash': profile_hash }) existing_applicability = RepoProfileApplicability( **applicability_dict) existing_applicability.applicability = applicability existing_applicability.save()