def batch_regenerate_applicability(repo_id, existing_applicability_ids):
"""
Regenerate and save applicability data for a batch of existing applicabilities
:param repo_id: Repository id for which applicability is being calculated
:type repo_id: str
:param existing_applicability_ids: Tuple of Object Ids for applicability profiles
:type existing_applicability_ids: tuple of dicts in form of {"_id": ObjectID('mongo-id')}
"""
id_list = [id['_id'] for id in existing_applicability_ids]
existing_applicabilities = RepoProfileApplicability.get_collection().find(
{"_id": {"$in": id_list}})
for existing_applicability in existing_applicabilities:
# Convert cursor to RepoProfileApplicability object
existing_applicability = RepoProfileApplicability(**dict(existing_applicability))
profile_hash = existing_applicability['profile_hash']
unit_profile = UnitProfile.get_collection().find_one({'profile_hash': profile_hash},
fields=['id', 'content_type'])
if unit_profile is None:
# Unit profiles change whenever packages are installed or removed on consumers,
# and it is possible that existing_applicability references a UnitProfile
# that no longer exists. This is harmless, as Pulp has a monthly cleanup task
# that will identify these dangling references and remove them.
continue
# Regenerate applicability data for given unit_profile and repo id
ApplicabilityRegenerationManager.regenerate_applicability(
profile_hash, unit_profile['content_type'], unit_profile['id'], repo_id,
existing_applicability)
def create(self, profile_hash, repo_id, profile, applicability,
all_profiles_hash):
"""
Create and return a RepoProfileApplicability object.
:param profile_hash: The hash of the profile that is a part of the profile set of a
consumer
:type profile_hash: basestring
:param repo_id: The repo ID that this applicability data is for
:type repo_id: basestring
:param profile: The entire profile that resulted in the profile_hash
:type profile: object
:param applicability: A dictionary structure mapping unit type IDs to lists of applicable
Unit IDs.
:type applicability: dict
:param all_profiles_hash: The hash of the set of the profiles that this applicability
data is for
:type all_profiles_hash: basestring
:return: A new RepoProfileApplicability object
:rtype: pulp.server.db.model.consumer.RepoProfileApplicability
"""
applicability = RepoProfileApplicability(
profile_hash=profile_hash,
repo_id=repo_id,
profile=profile,
applicability=applicability,
all_profiles_hash=all_profiles_hash)
applicability.save()
return applicability
def filter(self, query_params):
"""
Get a list of RepoProfileApplicability objects with the given MongoDB query dict.
:param query_params: A MongoDB query dictionary that selects RepoProfileApplicability
documents
:type query_params: dict
:return: A list of RepoProfileApplicability objects that match the given query
:rtype: list
"""
collection = RepoProfileApplicability.get_collection()
mongo_applicabilities = collection.find(query_params)
applicabilities = [RepoProfileApplicability(**dict(applicability)) \
for applicability in mongo_applicabilities]
return applicabilities
def regenerate_applicability_for_repos(repo_criteria):
"""
Regenerate and save applicability data affected by given updated repositories.
:param repo_criteria: The repo selection criteria
:type repo_criteria: dict
"""
repo_criteria = Criteria.from_dict(repo_criteria)
# Process repo criteria
repo_criteria.fields = ['id']
repo_ids = [
r.repo_id
for r in model.Repository.objects.find_by_criteria(repo_criteria)
]
for repo_id in repo_ids:
# Find all existing applicabilities for given repo_id. Setting batch size of 5 ensures
# the MongoDB cursor does not time out. See https://pulp.plan.io/issues/998#note-6 for
# more details.
existing_applicabilities = RepoProfileApplicability.get_collection(
).find({
'repo_id': repo_id
}).batch_size(5)
for existing_applicability in existing_applicabilities:
existing_applicability = RepoProfileApplicability(
**dict(existing_applicability))
profile_hash = existing_applicability['profile_hash']
unit_profile = UnitProfile.get_collection().find_one(
{'profile_hash': profile_hash},
projection=['id', 'content_type'])
if unit_profile is None:
# Unit profiles change whenever packages are installed or removed on consumers,
# and it is possible that existing_applicability references a UnitProfile
# that no longer exists. This is harmless, as Pulp has a monthly cleanup task
# that will identify these dangling references and remove them.
continue
# Regenerate applicability data for given unit_profile and repo id
ApplicabilityRegenerationManager.regenerate_applicability(
profile_hash, unit_profile['content_type'],
unit_profile['id'], repo_id, existing_applicability)
def regenerate_applicability_for_repos(repo_criteria):
"""
Regenerate and save applicability data affected by given updated repositories.
:param repo_criteria: The repo selection criteria
:type repo_criteria: dict
"""
repo_criteria = Criteria.from_dict(repo_criteria)
repo_query_manager = managers.repo_query_manager()
# Process repo criteria
repo_criteria.fields = ['id']
repo_ids = [
r['id'] for r in repo_query_manager.find_by_criteria(repo_criteria)
]
for repo_id in repo_ids:
# Find all existing applicabilities for given repo_id
existing_applicabilities = RepoProfileApplicability.get_collection(
).find({'repo_id': repo_id})
for existing_applicability in existing_applicabilities:
# Convert cursor to RepoProfileApplicability object
existing_applicability = RepoProfileApplicability(
**dict(existing_applicability))
profile_hash = existing_applicability['profile_hash']
unit_profile = UnitProfile.get_collection().find_one(
{'profile_hash': profile_hash},
fields=['id', 'content_type'])
if unit_profile is None:
# Unit profiles change whenever packages are installed or removed on consumers,
# and it is possible that existing_applicability references a UnitProfile
# that no longer exists. This is harmless, as Pulp has a monthly cleanup task
# that will identify these dangling references and remove them.
continue
# Regenerate applicability data for given unit_profile and repo id
ApplicabilityRegenerationManager.regenerate_applicability(
profile_hash, unit_profile['content_type'],
unit_profile['id'], repo_id, existing_applicability)
def batch_regenerate_applicability(repo_id, profile_hashes):
"""
Regenerate and save applicability data for a batch of existing applicabilities
:param repo_id: Repository id for which applicability is being calculated
:type repo_id: str
:param profile_hashes: Tuple of consumer profile hashes for applicability profiles.
Don't pass too much of these, all the profile data
associated with these hashes is loaded into the memory.
:type profile_hashes: tuple of dicts in form of {'profile_hash': str}
"""
profile_hash_list = [phash['profile_hash'] for phash in profile_hashes]
existing_applicabilities = RepoProfileApplicability.get_collection(
).find({
"repo_id": repo_id,
"profile_hash": {
"$in": profile_hash_list
}
})
for existing_applicability in list(existing_applicabilities):
# Convert cursor to RepoProfileApplicability object
existing_applicability = RepoProfileApplicability(
**dict(existing_applicability))
profile_hash = existing_applicability['profile_hash']
unit_profile = UnitProfile.get_collection().find_one(
{'profile_hash': profile_hash},
projection=['id', 'content_type'])
if unit_profile is None:
# Unit profiles change whenever packages are installed or removed on consumers,
# and it is possible that existing_applicability references a UnitProfile
# that no longer exists. This is harmless, as Pulp has a monthly cleanup task
# that will identify these dangling references and remove them.
continue
# Regenerate applicability data for given unit_profile and repo id
ApplicabilityRegenerationManager.regenerate_applicability(
profile_hash, unit_profile['content_type'], unit_profile['id'],
repo_id, existing_applicability)
def regenerate_applicability(profile_hash,
content_type,
profile_id,
bound_repo_id,
existing_applicability=None):
"""
Regenerate and save applicability data for given profile and bound repo id.
If existing_applicability is not None, replace it with the new applicability data.
:param profile_hash: hash of the unit profile
:type profile_hash: basestring
:param content_type: profile (unit) type ID
:type content_type: str
:param profile_id: unique id of the unit profile
:type profile_id: str
:param bound_repo_id: repo id to be used to calculate applicability
against the given unit profile
:type bound_repo_id: str
:param existing_applicability: existing RepoProfileApplicability object to be replaced
:type existing_applicability: pulp.server.db.model.consumer.RepoProfileApplicability
"""
profiler_conduit = ProfilerConduit()
# Get the profiler for content_type of given unit_profile
profiler, profiler_cfg = ApplicabilityRegenerationManager._profiler(
content_type)
# Check if the profiler supports applicability, else return
if profiler.calculate_applicable_units == Profiler.calculate_applicable_units:
# If base class calculate_applicable_units method is called,
# skip applicability regeneration
return
# Find out which content types have unit counts greater than zero in the bound repo
repo_content_types = ApplicabilityRegenerationManager._get_existing_repo_content_types(
bound_repo_id)
# Get the intersection of existing types in the repo and the types that the profiler
# handles. If the intersection is not empty, regenerate applicability
if (set(repo_content_types) & set(profiler.metadata()['types'])):
# Get the actual profile for existing_applicability or lookup using profile_id
if existing_applicability:
profile = existing_applicability.profile
else:
unit_profile = UnitProfile.get_collection().find_one(
{'id': profile_id}, projection=['profile'])
profile = unit_profile['profile']
call_config = PluginCallConfiguration(plugin_config=profiler_cfg,
repo_plugin_config=None)
try:
applicability = profiler.calculate_applicable_units(
profile, bound_repo_id, call_config, profiler_conduit)
except NotImplementedError:
msg = "Profiler for content type [%s] does not support applicability" % content_type
_logger.debug(msg)
return
try:
# Create a new RepoProfileApplicability object and save it in the db
RepoProfileApplicability.objects.create(
profile_hash, bound_repo_id, profile, applicability)
except DuplicateKeyError:
# Update existing applicability
if not existing_applicability:
applicability_dict = RepoProfileApplicability.get_collection(
).find_one({
'repo_id': bound_repo_id,
'profile_hash': profile_hash
})
existing_applicability = RepoProfileApplicability(
**applicability_dict)
existing_applicability.applicability = applicability
existing_applicability.save()
def regenerate_applicability(all_profiles_hash, profiles, bound_repo_id):
"""
Regenerate and save applicability data for given set of profiles and bound repo id.
:param all_profiles_hash: hash of the consumer profiles
:type all_profiles_hash: basestring
:param profiles: profiles data: (profile_hash, content_type, profile_id)
:type profiles: list of tuples
:param bound_repo_id: repo id to be used to calculate applicability
against the given unit profile
:type bound_repo_id: str
"""
profiler_conduit = ProfilerConduit()
# Get the profiler for content_type of given profiles.
# The assumption is that the same profiler is used for all the content types, so different
# profilers are not supported at the moment.
# Take the content type from the first profile.
content_type = profiles[0][1]
profiler, profiler_cfg = ApplicabilityRegenerationManager._profiler(
content_type)
# Check if the profiler supports applicability, else return
if profiler.calculate_applicable_units == Profiler.calculate_applicable_units:
# If base class calculate_applicable_units method is called,
# skip applicability regeneration
return
# Find out which content types have unit counts greater than zero in the bound repo
repo_content_types = ApplicabilityRegenerationManager._get_existing_repo_content_types(
bound_repo_id)
# Get the intersection of existing types in the repo and the types that the profiler
# handles. If the intersection is not empty, regenerate applicability
if (set(repo_content_types) & set(profiler.metadata()['types'])):
profile_ids = [p_id for _, _, p_id in profiles]
unit_profiles = UnitProfile.get_collection().find(
{'id': {
'$in': profile_ids
}},
projection=['profile', 'content_type', 'profile_hash'])
try:
profiles = [(p['profile_hash'], p['content_type'],
p['profile']) for p in unit_profiles]
except TypeError:
# It means that p = None.
# Consumer can be removed during applicability regeneration,
# so it is possible that its profile no longer exists. It is harmless.
return
call_config = PluginCallConfiguration(plugin_config=profiler_cfg,
repo_plugin_config=None)
try:
applicability = profiler.calculate_applicable_units(
profiles, bound_repo_id, call_config, profiler_conduit)
except NotImplementedError:
msg = "Profiler for content type [%s] does not support applicability" % content_type
_logger.debug(msg)
return
# Save applicability results on each of the profiles. The results are duplicated.
# It's a compromise to have applicability data available in any applicability profile
# record in the DB.
for profile in profiles:
profile_hash = profile[0]
try:
# Create a new RepoProfileApplicability object and save it in the db
RepoProfileApplicability.objects.create(
profile_hash=profile_hash,
repo_id=bound_repo_id,
# profiles can be large, the one in
# repo_profile_applicability collection
# is no longer used,
# it's a duplicated data
# from the consumer_unit_profiles
# collection.
profile=[],
applicability=applicability,
all_profiles_hash=all_profiles_hash)
except DuplicateKeyError:
applicability_dict = RepoProfileApplicability.get_collection(
).find_one({
'repo_id': bound_repo_id,
'all_profiles_hash': all_profiles_hash,
'profile_hash': profile_hash
})
existing_applicability = RepoProfileApplicability(
**applicability_dict)
existing_applicability.applicability = applicability
existing_applicability.save()
